diff --git a/.github/.github/pull_request_template.md b/.github/.github/pull_request_template.md index 8ce224e8..62c0286c 100644 --- a/.github/.github/pull_request_template.md +++ b/.github/.github/pull_request_template.md @@ -1,7 +1,7 @@ -## What type of PR is this? +## What type of PR is this? - [ ] Refactor @@ -13,8 +13,8 @@ ## How is this tested? -- [ ] Unit tests -- [ ] E2E Tests +- [ ] Unit tests +- [ ] E2E Tests - [ ] Manually - [ ] N/A diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aa9b23ef..961a7638 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,4 @@ # the repo. Unless a later match takes precedence, these # users will be requested for review when someone opens a # pull request. -* @susodapop @arikfr @yunbodeng-db @andrefurlan-db +* @rcypher-databricks @yunbodeng-db @andrefurlan-db @jackyhu-db @benc-db @kravets-levko diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index fe47eb15..843d09ac 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -1,5 +1,5 @@ name: Code Quality Checks -on: +on: push: branches: - main @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.9, "3.10", "3.11", "3.12"] steps: #---------------------------------------------- # check-out repo and set-up python @@ -58,11 +58,62 @@ jobs: #---------------------------------------------- - name: Run tests run: poetry run python -m pytest tests/unit + run-unit-tests-with-arrow: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.9, "3.10", "3.11", "3.12"] + steps: + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Check out repository + uses: actions/checkout@v2 + - name: Set up python ${{ matrix.python-version }} + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v2 + with: + path: .venv-pyarrow + key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: poetry install --no-interaction --all-extras + #---------------------------------------------- + # run test suite + #---------------------------------------------- + - name: Run tests + run: poetry run python -m pytest tests/unit check-linting: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: [3.9, "3.10", "3.11", "3.12"] steps: #---------------------------------------------- # check-out repo and set-up python @@ -114,7 +165,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: [3.9, "3.10", "3.11", "3.12"] steps: #---------------------------------------------- # check-out repo and set-up python @@ -157,7 +208,9 @@ jobs: - name: Install library run: poetry install --no-interaction #---------------------------------------------- - # black the code + # mypy the code #---------------------------------------------- - name: Mypy - run: poetry run mypy --install-types --non-interactive src + run: | + mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 + poetry run mypy --install-types --non-interactive src diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 00000000..aef7b7f2 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,57 @@ +name: Integration Tests +on: + push: + paths-ignore: + - "**.MD" + - "**.md" + +jobs: + run-e2e-tests: + runs-on: ubuntu-latest + environment: azure-prod + env: + DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_CATALOG: peco + DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} + steps: + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Check out repository + uses: actions/checkout@v3 + - name: Set up python + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v2 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + run: poetry install --no-interaction --all-extras + #---------------------------------------------- + # run test suite + #---------------------------------------------- + - name: Run e2e tests + run: poetry run python -m pytest tests/e2e diff --git a/.github/workflows/publish-manual.yml b/.github/workflows/publish-manual.yml new file mode 100644 index 00000000..ecad71a2 --- /dev/null +++ b/.github/workflows/publish-manual.yml @@ -0,0 +1,78 @@ +name: Publish to PyPI Manual [Production] + +# Allow manual triggering of the workflow +on: + workflow_dispatch: {} + +jobs: + publish: + name: Publish + runs-on: ubuntu-latest + + steps: + #---------------------------------------------- + # Step 1: Check out the repository code + #---------------------------------------------- + - name: Check out repository + uses: actions/checkout@v2 # Check out the repository to access the code + + #---------------------------------------------- + # Step 2: Set up Python environment + #---------------------------------------------- + - name: Set up python + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: 3.9 # Specify the Python version to be used + + #---------------------------------------------- + # Step 3: Install and configure Poetry + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 # Install Poetry, the Python package manager + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + +# #---------------------------------------------- +# # Step 4: Load cached virtual environment (if available) +# #---------------------------------------------- +# - name: Load cached venv +# id: cached-poetry-dependencies +# uses: actions/cache@v2 +# with: +# path: .venv # Path to the virtual environment +# key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} +# # Cache key is generated based on OS, Python version, repo name, and the `poetry.lock` file hash + +# #---------------------------------------------- +# # Step 5: Install dependencies if the cache is not found +# #---------------------------------------------- +# - name: Install dependencies +# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' # Only run if the cache was not hit +# run: poetry install --no-interaction --no-root # Install dependencies without interaction + +# #---------------------------------------------- +# # Step 6: Update the version to the manually provided version +# #---------------------------------------------- +# - name: Update pyproject.toml with the specified version +# run: poetry version ${{ github.event.inputs.version }} # Use the version provided by the user input + + #---------------------------------------------- + # Step 7: Build and publish the first package to PyPI + #---------------------------------------------- + - name: Build and publish databricks sql connector to PyPI + working-directory: ./databricks_sql_connector + run: | + poetry build + poetry publish -u __token__ -p ${{ secrets.PROD_PYPI_TOKEN }} # Publish with PyPI token + #---------------------------------------------- + # Step 7: Build and publish the second package to PyPI + #---------------------------------------------- + + - name: Build and publish databricks sql connector core to PyPI + working-directory: ./databricks_sql_connector_core + run: | + poetry build + poetry publish -u __token__ -p ${{ secrets.PROD_PYPI_TOKEN }} # Publish with PyPI token \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9ea751d0..324575ff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -61,4 +61,4 @@ jobs: - name: Build and publish to pypi uses: JRubics/poetry-publish@v1.10 with: - pypi_token: ${{ secrets.PROD_PYPI_TOKEN }} \ No newline at end of file + pypi_token: ${{ secrets.PROD_PYPI_TOKEN }} diff --git a/.gitignore b/.gitignore index 66c94734..2ae38dbc 100644 --- a/.gitignore +++ b/.gitignore @@ -195,7 +195,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # End of https://www.toptal.com/developers/gitignore/api/python,macos @@ -204,4 +204,7 @@ dist/ build/ # vs code stuff -.vscode \ No newline at end of file +.vscode + +# don't commit authentication info to source control +test.env diff --git a/CHANGELOG.md b/CHANGELOG.md index d424c7b3..9d64f4d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,175 @@ # Release History -## 2.5.x (Unreleased) +# 4.0.0 (2025-01-19) + +- Split the connector into two separate packages: `databricks-sql-connector` and `databricks-sqlalchemy`. The `databricks-sql-connector` package contains the core functionality of the connector, while the `databricks-sqlalchemy` package contains the SQLAlchemy dialect for the connector. +- Pyarrow dependency is now optional in `databricks-sql-connector`. Users needing arrow are supposed to explicitly install pyarrow + +# 3.7.1 (2025-01-07) + +- Relaxed the number of Http retry attempts (databricks/databricks-sql-python#486 by @jprakash-db) + +# 3.7.0 (2024-12-23) + +- Fix: Incorrect number of rows fetched in inline results when fetching results with FETCH_NEXT orientation (databricks/databricks-sql-python#479 by @jprakash-db) +- Updated the doc to specify native parameters are not supported in PUT operation (databricks/databricks-sql-python#477 by @jprakash-db) +- Relax `pyarrow` and `numpy` pin (databricks/databricks-sql-python#452 by @arredond) +- Feature: Support for async execute has been added (databricks/databricks-sql-python#463 by @jprakash-db) +- Updated the HTTP retry logic to be similar to the other Databricks drivers (databricks/databricks-sql-python#467 by @jprakash-db) + +# 3.6.0 (2024-10-25) + +- Support encryption headers in the cloud fetch request (https://github.com/databricks/databricks-sql-python/pull/460 by @jackyhu-db) + +# 3.5.0 (2024-10-18) + +- Create a non pyarrow flow to handle small results for the column set (databricks/databricks-sql-python#440 by @jprakash-db) +- Fix: On non-retryable error, ensure PySQL includes useful information in error (databricks/databricks-sql-python#447 by @shivam2680) + +# 3.4.0 (2024-08-27) + +- Unpin pandas to support v2.2.2 (databricks/databricks-sql-python#416 by @kfollesdal) +- Make OAuth as the default authenticator if no authentication setting is provided (databricks/databricks-sql-python#419 by @jackyhu-db) +- Fix (regression): use SSL options with HTTPS connection pool (databricks/databricks-sql-python#425 by @kravets-levko) + +# 3.3.0 (2024-07-18) + +- Don't retry requests that fail with HTTP code 401 (databricks/databricks-sql-python#408 by @Hodnebo) +- Remove username/password (aka "basic") auth option (databricks/databricks-sql-python#409 by @jackyhu-db) +- Refactor CloudFetch handler to fix numerous issues with it (databricks/databricks-sql-python#405 by @kravets-levko) +- Add option to disable SSL verification for CloudFetch links (databricks/databricks-sql-python#414 by @kravets-levko) + +Databricks-managed passwords reached end of life on July 10, 2024. Therefore, Basic auth support was removed from +the library. See https://docs.databricks.com/en/security/auth-authz/password-deprecation.html + +The existing option `_tls_no_verify=True` of `sql.connect(...)` will now also disable SSL cert verification +(but not the SSL itself) for CloudFetch links. This option should be used as a workaround only, when other ways +to fix SSL certificate errors didn't work. + +# 3.2.0 (2024-06-06) + +- Update proxy authentication (databricks/databricks-sql-python#354 by @amir-haroun) +- Relax `pyarrow` pin (databricks/databricks-sql-python#389 by @dhirschfeld) +- Fix error logging in OAuth manager (databricks/databricks-sql-python#269 by @susodapop) +- SQLAlchemy: enable delta.feature.allowColumnDefaults for all tables (databricks/databricks-sql-python#343 by @dhirschfeld) +- Update `thrift` dependency (databricks/databricks-sql-python#397 by @m1n0) + +# 3.1.2 (2024-04-18) + +- Remove broken cookie code (#379) +- Small typing fixes (#382, #384 thanks @wyattscarpenter) + +# 3.1.1 (2024-03-19) + +- Don't retry requests that fail with code 403 (#373) +- Assume a default retry-after for 429/503 (#371) +- Fix boolean literals (#357) + +# 3.1.0 (2024-02-16) + +- Revert retry-after behavior to be exponential backoff (#349) +- Support Databricks OAuth on Azure (#351) +- Support Databricks OAuth on GCP (#338) + +# 3.0.3 (2024-02-02) + +- Revised docstrings and examples for OAuth (#339) +- Redact the URL query parameters from the urllib3.connectionpool logs (#341) + +# 3.0.2 (2024-01-25) + +- SQLAlchemy dialect now supports table and column comments (thanks @cbornet!) +- Fix: SQLAlchemy dialect now correctly reflects TINYINT types (thanks @TimTheinAtTabs!) +- Fix: `server_hostname` URIs that included `https://` would raise an exception +- Other: pinned to `pandas<=2.1` and `urllib3>=1.26` to avoid runtime errors in dbt-databricks (#330) + +## 3.0.1 (2023-12-01) + +- Other: updated docstring comment about default parameterization approach (#287) +- Other: added tests for reading complex types and revised docstrings and type hints (#293) +- Fix: SQLAlchemy dialect raised DeprecationWarning due to `dbapi` classmethod (#294) +- Fix: SQLAlchemy dialect could not reflect TIMESTAMP_NTZ columns (#296) + +## 3.0.0 (2023-11-17) + +- Remove support for Python 3.7 +- Add support for native parameterized SQL queries. Requires DBR 14.2 and above. See docs/parameters.md for more info. +- Completely rewritten SQLAlchemy dialect + - Adds support for SQLAlchemy >= 2.0 and drops support for SQLAlchemy 1.x + - Full e2e test coverage of all supported features + - Detailed usage notes in `README.sqlalchemy.md` + - Adds support for: + - New types: `TIME`, `TIMESTAMP`, `TIMESTAMP_NTZ`, `TINYINT` + - `Numeric` type scale and precision, like `Numeric(10,2)` + - Reading and writing `PrimaryKeyConstraint` and `ForeignKeyConstraint` + - Reading and writing composite keys + - Reading and writing from views + - Writing `Identity` to tables (i.e. autoincrementing primary keys) + - `LIMIT` and `OFFSET` for paging through results + - Caching metadata calls +- Enable cloud fetch by default. To disable, set `use_cloud_fetch=False` when building `databricks.sql.client`. +- Add integration tests for Databricks UC Volumes ingestion queries +- Retries: + - Add `_retry_max_redirects` config + - Set `_enable_v3_retries=True` and warn if users override it +- Security: bump minimum pyarrow version to 14.0.1 (CVE-2023-47248) + +## 2.9.3 (2023-08-24) + +- Fix: Connections failed when urllib3~=1.0.0 is installed (#206) + +## 2.9.2 (2023-08-17) + +**Note: this release was yanked from Pypi on 13 September 2023 due to compatibility issues with environments where `urllib3<=2.0.0` were installed. The log changes are incorporated into version 2.9.3 and greater.** + +- Other: Add `examples/v3_retries_query_execute.py` (#199) +- Other: suppress log message when `_enable_v3_retries` is not `True` (#199) +- Other: make this connector backwards compatible with `urllib3>=1.0.0` (#197) + +## 2.9.1 (2023-08-11) + +**Note: this release was yanked from Pypi on 13 September 2023 due to compatibility issues with environments where `urllib3<=2.0.0` were installed.** + +- Other: Explicitly pin urllib3 to ^2.0.0 (#191) + +## 2.9.0 (2023-08-10) + +- Replace retry handling with DatabricksRetryPolicy. This is disabled by default. To enable, set `_enable_v3_retries=True` when creating `databricks.sql.client` (#182) +- Other: Fix typo in README quick start example (#186) +- Other: Add autospec to Client mocks and tidy up `make_request` (#188) + +## 2.8.0 (2023-07-21) + +- Add support for Cloud Fetch. Disabled by default. Set `use_cloud_fetch=True` when building `databricks.sql.client` to enable it (#146, #151, #154) +- SQLAlchemy has_table function now honours schema= argument and adds catalog= argument (#174) +- SQLAlchemy set non_native_boolean_check_constraint False as it's not supported by Databricks (#120) +- Fix: Revised SQLAlchemy dialect and examples for compatibility with SQLAlchemy==1.3.x (#173) +- Fix: oauth would fail if expired credentials appeared in ~/.netrc (#122) +- Fix: Python HTTP proxies were broken after switch to urllib3 (#158) +- Other: remove unused import in SQLAlchemy dialect +- Other: Relax pandas dependency constraint to allow ^2.0.0 (#164) +- Other: Connector now logs operation handle guids as hexadecimal instead of bytes (#170) +- Other: test_socket_timeout_user_defined e2e test was broken (#144) + +## 2.7.0 (2023-06-26) + +- Fix: connector raised exception when calling close() on a closed Thrift session +- Improve e2e test development ergonomics +- Redact logged thrift responses by default +- Add support for OAuth on Databricks Azure + +## 2.6.2 (2023-06-14) + +- Fix: Retry GetOperationStatus requests for http errors + +## 2.6.1 (2023-06-08) + +- Fix: http.client would raise a BadStatusLine exception in some cases + +## 2.6.0 (2023-06-07) + +- Add support for HTTP 1.1 connections (connection pools) +- Add a default socket timeout for thrift RPCs ## 2.5.2 (2023-05-08) @@ -12,6 +181,7 @@ - Other: Relax sqlalchemy required version as it was unecessarily strict. ## 2.5.0 (2023-04-14) + - Add support for External Auth providers - Fix: Python HTTP proxies were broken - Other: All Thrift requests that timeout during connection will be automatically retried @@ -33,8 +203,8 @@ ## 2.2.2 (2023-01-03) -- Support custom oauth client id and redirect port -- Fix: Add none check on _oauth_persistence in DatabricksOAuthProvider +- Support custom oauth client id and redirect port +- Fix: Add none check on \_oauth_persistence in DatabricksOAuthProvider ## 2.2.1 (2022-11-29) @@ -66,57 +236,71 @@ Huge thanks to @dbaxa for contributing this change! - Add retry logic for `GetOperationStatus` requests that fail with an `OSError` - Reorganised code to use Poetry for dependency management. + ## 2.0.2 (2022-05-04) + - Better exception handling in automatic connection close ## 2.0.1 (2022-04-21) + - Fixed Pandas dependency in setup.cfg to be >= 1.2.0 ## 2.0.0 (2022-04-19) + - Initial stable release of V2 -- Added better support for complex types, so that in Databricks runtime 10.3+, Arrays, Maps and Structs will get +- Added better support for complex types, so that in Databricks runtime 10.3+, Arrays, Maps and Structs will get deserialized as lists, lists of tuples and dicts, respectively. - Changed the name of the metadata arg to http_headers ## 2.0.b2 (2022-04-04) + - Change import of collections.Iterable to collections.abc.Iterable to make the library compatible with Python 3.10 - Fixed bug with .tables method so that .tables works as expected with Unity-Catalog enabled endpoints ## 2.0.0b1 (2022-03-04) + - Fix packaging issue (dependencies were not being installed properly) - Fetching timestamp results will now return aware instead of naive timestamps - The client will now default to using simplified error messages ## 2.0.0b (2022-02-08) + - Initial beta release of V2. V2 is an internal re-write of large parts of the connector to use Databricks edge features. All public APIs from V1 remain. -- Added Unity Catalog support (pass catalog and / or schema key word args to the .connect method to select initial schema and catalog) +- Added Unity Catalog support (pass catalog and / or schema key word args to the .connect method to select initial schema and catalog) --- **Note**: The code for versions prior to `v2.0.0b` is not contained in this repository. The below entries are included for reference only. --- + ## 1.0.0 (2022-01-20) + - Add operations for retrieving metadata - Add the ability to access columns by name on result rows - Add the ability to provide configuration settings on connect ## 0.9.4 (2022-01-10) + - Improved logging and error messages. ## 0.9.3 (2021-12-08) + - Add retries for 429 and 503 HTTP responses. ## 0.9.2 (2021-12-02) + - (Bug fix) Increased Thrift requirement from 0.10.0 to 0.13.0 as 0.10.0 was in fact incompatible - (Bug fix) Fixed error message after query execution failed -SQLSTATE and Error message were misplaced ## 0.9.1 (2021-09-01) + - Public Preview release, Experimental tag removed - minor updates in internal build/packaging - no functional changes ## 0.9.0 (2021-08-04) + - initial (Experimental) release of pyhive-forked connector - Python DBAPI 2.0 (PEP-0249), thrift based - see docs for more info: https://docs.databricks.com/dev-tools/python-sql-connector.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aea830eb..0cb25876 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,7 +74,7 @@ If you set your `user.name` and `user.email` git configs, you can sign your comm This project uses [Poetry](https://python-poetry.org/) for dependency management, tests, and linting. 1. Clone this respository -2. Run `poetry install` +2. Run `poetry install` ### Run tests @@ -107,8 +107,21 @@ End-to-end tests require a Databricks account. Before you can run them, you must export host="" export http_path="" export access_token="" +export catalog="" +export schema="" ``` +Or you can write these into a file called `test.env` in the root of the repository: + +``` +host="****.cloud.databricks.com" +http_path="/sql/1.0/warehouses/***" +access_token="dapi***" +staging_ingestion_user="***@example.com" +``` + +To see logging output from pytest while running tests, set `log_cli = "true"` under `tool.pytest.ini_options` in `pyproject.toml`. You can also set `log_cli_level` to any of the default Python log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` + There are several e2e test suites available: - `PySQLCoreTestSuite` - `PySQLLargeQueriesSuite` @@ -130,6 +143,8 @@ The `PySQLLargeQueriesSuite` namespace contains long-running query tests and is The `PySQLStagingIngestionTestSuite` namespace requires a cluster running DBR version > 12.x which supports staging ingestion commands. The suites marked `[not documented]` require additional configuration which will be documented at a later time. + + ### Code formatting This project uses [Black](https://pypi.org/project/black/). @@ -149,5 +164,4 @@ Modify the dependency specification (syntax can be found [here](https://python-p - `poetry update` - `rm poetry.lock && poetry install` -Sometimes `poetry update` can freeze or run forever. Deleting the `poetry.lock` file and calling `poetry install` is guaranteed to update everything but is usually _slower_ than `poetry update` **if `poetry update` works at all**. - +Sometimes `poetry update` can freeze or run forever. Deleting the `poetry.lock` file and calling `poetry install` is guaranteed to update everything but is usually _slower_ than `poetry update` **if `poetry update` works at all**. diff --git a/README.md b/README.md index 60c9081c..a4c5a130 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,15 @@ [![PyPI](https://img.shields.io/pypi/v/databricks-sql-connector?style=flat-square)](https://pypi.org/project/databricks-sql-connector/) [![Downloads](https://pepy.tech/badge/databricks-sql-connector)](https://pepy.tech/project/databricks-sql-connector) -The Databricks SQL Connector for Python allows you to develop Python applications that connect to Databricks clusters and SQL warehouses. It is a Thrift-based client with no dependencies on ODBC or JDBC. It conforms to the [Python DB API 2.0 specification](https://www.python.org/dev/peps/pep-0249/) and exposes a [SQLAlchemy](https://www.sqlalchemy.org/) dialect for use with tools like `pandas` and `alembic` which use SQLAlchemy to execute DDL. +The Databricks SQL Connector for Python allows you to develop Python applications that connect to Databricks clusters and SQL warehouses. It is a Thrift-based client with no dependencies on ODBC or JDBC. It conforms to the [Python DB API 2.0 specification](https://www.python.org/dev/peps/pep-0249/). -This connector uses Arrow as the data-exchange format, and supports APIs to directly fetch Arrow tables. Arrow tables are wrapped in the `ArrowQueue` class to provide a natural API to get several rows at a time. +This connector uses Arrow as the data-exchange format, and supports APIs (e.g. `fetchmany_arrow`) to directly fetch Arrow tables. Arrow tables are wrapped in the `ArrowQueue` class to provide a natural API to get several rows at a time. [PyArrow](https://arrow.apache.org/docs/python/index.html) is required to enable this and use these APIs, you can install it via `pip install pyarrow` or `pip install databricks-sql-connector[pyarrow]`. You are welcome to file an issue here for general use cases. You can also contact Databricks Support [here](help.databricks.com). ## Requirements -Python 3.7 or above is required. +Python 3.8 or above is required. ## Documentation @@ -22,14 +22,16 @@ For the latest documentation, see ## Quickstart -Install the library with `pip install databricks-sql-connector` +### Installing the core library +Install using `pip install databricks-sql-connector` + +### Installing the core library with PyArrow +Install using `pip install databricks-sql-connector[pyarrow]` -Note: Don't hard-code authentication secrets into your Python. Use environment variables ```bash export DATABRICKS_HOST=********.databricks.com export DATABRICKS_HTTP_PATH=/sql/1.0/endpoints/**************** -export DATABRICKS_TOKEN=dapi******************************** ``` Example usage: @@ -39,16 +41,13 @@ from databricks import sql host = os.getenv("DATABRICKS_HOST") http_path = os.getenv("DATABRICKS_HTTP_PATH") -access_token = os.getenv("DATABRICKS_ACCESS_TOKEN") connection = sql.connect( server_hostname=host, - http_path=http_path, - access_token=access_token) + http_path=http_path) cursor = connection.cursor() - -cursor.execute('SELECT * FROM RANGE(10)') +cursor.execute('SELECT :param `p`, * FROM RANGE(10)', {"param": "foo"}) result = cursor.fetchall() for row in result: print(row) @@ -61,7 +60,22 @@ In the above example: - `server-hostname` is the Databricks instance host name. - `http-path` is the HTTP Path either to a Databricks SQL endpoint (e.g. /sql/1.0/endpoints/1234567890abcdef), or to a Databricks Runtime interactive cluster (e.g. /sql/protocolv1/o/1234567890123456/1234-123456-slid123) -- `personal-access-token` is the Databricks Personal Access Token for the account that will execute commands and queries + +> Note: This example uses [Databricks OAuth U2M](https://docs.databricks.com/en/dev-tools/auth/oauth-u2m.html) +> to authenticate the target Databricks user account and needs to open the browser for authentication. So it +> can only run on the user's machine. + +## SQLAlchemy +Starting from `databricks-sql-connector` version 4.0.0 SQLAlchemy support has been extracted to a new library `databricks-sqlalchemy`. + +- Github repository [databricks-sqlalchemy github](https://github.com/databricks/databricks-sqlalchemy) +- PyPI [databricks-sqlalchemy pypi](https://pypi.org/project/databricks-sqlalchemy/) + +### Quick SQLAlchemy guide +Users can now choose between using the SQLAlchemy v1 or SQLAlchemy v2 dialects with the connector core + +- Install the latest SQLAlchemy v1 using `pip install databricks-sqlalchemy~=1.0` +- Install SQLAlchemy v2 using `pip install databricks-sqlalchemy` ## Contributing diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..c8b350be --- /dev/null +++ b/conftest.py @@ -0,0 +1,44 @@ +import os +import pytest + + +@pytest.fixture(scope="session") +def host(): + return os.getenv("DATABRICKS_SERVER_HOSTNAME") + + +@pytest.fixture(scope="session") +def http_path(): + return os.getenv("DATABRICKS_HTTP_PATH") + + +@pytest.fixture(scope="session") +def access_token(): + return os.getenv("DATABRICKS_TOKEN") + + +@pytest.fixture(scope="session") +def ingestion_user(): + return os.getenv("DATABRICKS_USER") + + +@pytest.fixture(scope="session") +def catalog(): + return os.getenv("DATABRICKS_CATALOG") + + +@pytest.fixture(scope="session") +def schema(): + return os.getenv("DATABRICKS_SCHEMA", "default") + + +@pytest.fixture(scope="session", autouse=True) +def connection_details(host, http_path, access_token, ingestion_user, catalog, schema): + return { + "host": host, + "http_path": http_path, + "access_token": access_token, + "ingestion_user": ingestion_user, + "catalog": catalog, + "schema": schema, + } diff --git a/docs/parameters.md b/docs/parameters.md new file mode 100644 index 00000000..f9f4c5ff --- /dev/null +++ b/docs/parameters.md @@ -0,0 +1,256 @@ +# Using Native Parameters + +This connector supports native parameterized query execution. When you execute a query that includes variable markers, then you can pass a collection of parameters which are sent separately to Databricks Runtime for safe execution. This prevents SQL injection and can improve query performance. + +This behaviour is distinct from legacy "inline" parameterized execution in versions below 3.0.0. The legacy behavior is preserved behind a flag called `use_inline_params`, which will be removed in a future release. See [Using Inline Parameters](#using-inline-parameters) for more information. + +See **[below](#migrating-to-native-parameters)** for details about updating your client code to use native parameters. + +See `examples/parameters.py` in this repository for a runnable demo. + +## Requirements + +- `databricks-sql-connector>=3.0.0` +- A SQL warehouse or all-purpose cluster running Databricks Runtime >=14.2 + +## Limitations + +- A query executed with native parameters can contain at most 255 parameter markers +- The maximum size of all parameterized values cannot exceed 1MB +- For volume operations such as PUT, native parameters are not supported + +## SQL Syntax + +Variables in your SQL query can use one of three PEP-249 [paramstyles](https://peps.python.org/pep-0249/#paramstyle). A parameterized query can use exactly one paramstyle. + +|paramstyle|example|comment| +|-|-|-| +|`named`|`:param`|Parameters must be named| +|`qmark`|`?`|Parameter names are ignored| +|`pyformat`|`%(param)s`|Legacy syntax. Will be deprecated. Parameters must be named.| + +#### Example + +```sql +-- named paramstyle +SELECT * FROM table WHERE field = :value + +-- qmark paramstyle +SELECT * FROM table WHERE field = ? + +-- pyformat paramstyle (legacy) +SELECT * FROM table WHERE field = %(value)s +``` + +## Python Syntax + +This connector follows the [PEP-249 interface](https://peps.python.org/pep-0249/#id20). The expected structure of the parameter collection follows the paramstyle of the variables in your query. + +### `named` paramstyle Usage Example + +When your SQL query uses `named` paramstyle variable markers, you need specify a name for each value that corresponds to a variable marker in your query. + +Generally, you do this by passing `parameters` as a dictionary whose keys match the variables in your query. The length of the dictionary must exactly match the count of variable markers or an exception will be raised. + +```python +from databricks import sql + +with sql.connect(...) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = :value1 AND another_field = :value2" + parameters = {"value1": "foo", "value2": 20} + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +This paramstyle is a drop-in replacement for the `pyformat` paramstyle which was used in connector versions below 3.0.0. It should be used going forward. + +### `qmark` paramstyle Usage Example + +When your SQL query uses `qmark` paramstyle variable markers, you only need to specify a value for each variable marker in your query. + +You do this by passing `parameters` as a list. The order of values in the list corresponds to the order of `qmark` variables in your query. The length of the list must exactly match the count of variable markers in your query or an exception will be raised. + +```python +from databricks import sql + +with sql.connect(...) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = ? AND another_field = ?" + parameters = ["foo", 20] + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +The result of the above two examples is identical. + +### Legacy `pyformat` paramstyle Usage Example + +Databricks Runtime expects variable markers to use either `named` or `qmark` paramstyles. Historically, this connector used `pyformat` which Databricks Runtime does not support. So to assist assist customers transitioning their codebases from `pyformat` → `named`, we can dynamically rewrite the variable markers before sending the query to Databricks. This happens only when `use_inline_params=False`. + + This dynamic rewrite will be deprecated in a future release. New queries should be written using the `named` paramstyle instead. And users should update their client code to replace `pyformat` markers with `named` markers. + +For example: + +```sql +-- a query written for databricks-sql-connector==2.9.3 and below + +SELECT field1, field2, %(param1)s FROM table WHERE field4 = %(param2)s + +-- rewritten for databricks-sql-connector==3.0.0 and above + +SELECT field1, field2, :param1 FROM table WHERE field4 = :param2 +``` + + +**Note:** While named `pyformat` markers are transparently replaced when `use_inline_params=False`, un-named inline `%s`-style markers are ignored. If your client code makes extensive use of `%s` markers, these queries will need to be updated to use `?` markers before you can execute them when `use_inline_params=False`. See [When to use inline parameters](#when-to-use-inline-parameters) for more information. + +### Type inference + +Under the covers, parameter values are annotated with a valid Databricks SQL type. As shown in the examples above, this connector accepts primitive Python types like `int`, `str`, and `Decimal`. When this happens, the connector infers the corresponding Databricks SQL type (e.g. `INT`, `STRING`, `DECIMAL`) automatically. This means that the parameters passed to `cursor.execute()` are always wrapped in a `TDbsqlParameter` subtype prior to execution. + +Automatic inferrence is sufficient for most usages. But you can bypass the inference by explicitly setting the Databricks SQL type in your client code. All supported Databricks SQL types have `TDbsqlParameter` implementations which you can import from `databricks.sql.parameters`. + +`TDbsqlParameter` objects must always be passed within a list. Either paramstyle (`:named` or `?`) may be used. However, if your query uses the `named` paramstyle, all `TDbsqlParameter` objects must be provided a `name` when they are constructed. + +```python +from databricks import sql +from databricks.sql.parameters import StringParameter, IntegerParameter + +# with `named` markers +with sql.connect(...) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = :value1 AND another_field = :value2" + parameters = [ + StringParameter(name="value1", value="foo"), + IntegerParameter(name="value2", value=20) + ] + result = cursor.execute(query, parameters=parameters).fetchone() + +# with `?` markers +with sql.connect(...) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = ? AND another_field = ?" + parameters = [ + StringParameter(value="foo"), + IntegerParameter(value=20) + ] + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +In general, we recommend using `?` markers when passing `TDbsqlParameter`'s directly. + +**Note**: When using `?` markers, you can bypass inference for _some_ parameters by passing a list containing both primitive Python types and `TDbsqlParameter` objects. `TDbsqlParameter` objects can never be passed in a dictionary. + +# Using Inline Parameters + +Since its initial release, this connector's `cursor.execute()` method has supported passing a sequence or mapping of parameter values. Prior to Databricks Runtime introducing native parameter support, however, "parameterized" queries could not be executed in a guaranteed safe manner. Instead, the connector made a best effort to escape parameter values and and render those strings inline with the query. + +This approach has several drawbacks: + +- It's not guaranteed to be safe from SQL injection +- The server could not boost performance by caching prepared statements +- The parameter marker syntax conflicted with SQL syntax in some cases + +Nevertheless, this behaviour is preserved in version 3.0.0 and above for legacy purposes. It will be removed in a subsequent major release. To enable this legacy code path, you must now construct your connection with `use_inline_params=True`. + +## Requirements + +Rendering parameters inline is supported on all versions of DBR since these queries are indistinguishable from ad-hoc query text. + + +## SQL Syntax + +Variables in your SQL query can look like `%(param)s` or like `%s`. + +#### Example + +```sql +-- pyformat paramstyle is used for named parameters +SELECT * FROM table WHERE field = %(value)s + +-- %s is used for positional parameters +SELECT * FROM table WHERE field = %s +``` + +## Python Syntax + +This connector follows the [PEP-249 interface](https://peps.python.org/pep-0249/#id20). The expected structure of the parameter collection follows the paramstyle of the variables in your query. + +### `pyformat` paramstyle Usage Example + +Parameters must be passed as a dictionary. + +```python +from databricks import sql + +with sql.connect(..., use_inline_params=True) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = %(value1)s AND another_field = %(value2)s" + parameters = {"value1": "foo", "value2": 20} + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +The above query would be rendered into the following SQL: + +```sql +SELECT field FROM table WHERE field = 'foo' AND another_field = 20 +``` + +### `%s` paramstyle Usage Example + +Parameters must be passed as a list. + +```python +from databricks import sql + +with sql.connect(..., use_inline_params=True) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = %s AND another_field = %s" + parameters = ["foo", 20] + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +The result of the above two examples is identical. + +**Note**: `%s` is not compliant with PEP-249 and only works due to the specific implementation of our inline renderer. + +**Note:** This `%s` syntax overlaps with valid SQL syntax around the usage of `LIKE` DML. For example if your query includes a clause like `WHERE field LIKE '%sequence'`, the parameter inlining function will raise an exception because this string appears to include an inline marker but none is provided. This means that connector versions below 3.0.0 it has been impossible to execute a query that included both parameters and LIKE wildcards. When `use_inline_params=False`, we will pass `%s` occurrences along to the database, allowing it to be used as expected in `LIKE` statements. + +### Passing sequences as parameter values + +Parameter values can also be passed as a sequence. This is typically used when writing `WHERE ... IN` clauses: + +```python +from databricks import sql + +with sql.connect(..., use_inline_params=True) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field IN %(value_list)s" + parameters = {"value_list": [1,2,3,4,5]} + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +Output: + +```sql +SELECT field FROM table WHERE field IN (1,2,3,4,5) +``` + +**Note**: this behavior is not specified by PEP-249 and only works due to the specific implementation of our inline renderer. + +### Migrating to native parameters + +Native parameters are meant to be a drop-in replacement for inline parameters. In most use-cases, upgrading to `databricks-sql-connector>=3.0.0` will grant an immediate improvement to safety. Plus, native parameters allow you to use SQL LIKE wildcards (`%`) in your queries which is impossible with inline parameters. Future improvements to parameterization (such as support for binding complex types like `STRUCT`, `MAP`, and `ARRAY`) will only be available when `use_inline_params=False`. + +To completely migrate, you need to [revise your SQL queries](#legacy-pyformat-paramstyle-usage-example) to use the new paramstyles. + + +### When to use inline parameters + +You should only set `use_inline_params=True` in the following cases: + +1. Your client code passes more than 255 parameters in a single query execution +2. Your client code passes parameter values greater than 1MB in a single query execution +3. Your client code makes extensive use of [`%s` positional parameter markers](#s-paramstyle-usage-example) +4. Your client code uses [sequences as parameter values](#passing-sequences-as-parameter-values) + +We expect limitations (1) and (2) to be addressed in a future Databricks Runtime release. diff --git a/examples/README.md b/examples/README.md index 4fbe8527..43d248da 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ We provide example scripts so you can see the connector in action for basic usag - DATABRICKS_TOKEN Follow the quick start in our [README](../README.md) to install `databricks-sql-connector` and see -how to find the hostname, http path, and access token. Note that for the OAuth examples below a +how to find the hostname, http path, and access token. Note that for the OAuth examples below a personal access token is not needed. @@ -33,9 +33,12 @@ To run all of these examples you can clone the entire repository to your disk. O - **`insert_data.py`** adds a tables called `squares` to your default catalog and inserts one hundred rows of example data. Then it fetches this data and prints it to the screen. - **`query_cancel.py`** shows how to cancel a query assuming that you can access the `Cursor` executing that query from a different thread. This is necessary because `databricks-sql-connector` does not yet implement an asynchronous API; calling `.execute()` blocks the current thread until execution completes. Therefore, the connector can't cancel queries from the same thread where they began. - **`interactive_oauth.py`** shows the simplest example of authenticating by OAuth (no need for a PAT generated in the DBSQL UI) while Bring Your Own IDP is in public preview. When you run the script it will open a browser window so you can authenticate. Afterward, the script fetches some sample data from Databricks and prints it to the screen. For this script, the OAuth token is not persisted which means you need to authenticate every time you run the script. +- **`m2m_oauth.py`** shows the simplest example of authenticating by using OAuth M2M (machine-to-machine) for service principal. - **`persistent_oauth.py`** shows a more advanced example of authenticating by OAuth while Bring Your Own IDP is in public preview. In this case, it shows how to use a sublcass of `OAuthPersistence` to reuse an OAuth token across script executions. - **`set_user_agent.py`** shows how to customize the user agent header used for Thrift commands. In this example the string `ExamplePartnerTag` will be added to the the user agent on every request. - **`staging_ingestion.py`** shows how the connector handles Databricks' experimental staging ingestion commands `GET`, `PUT`, and `REMOVE`. -- **`sqlalchemy.py`** shows a basic example of connecting to Databricks with [SQLAlchemy](https://www.sqlalchemy.org/). -- **`custom_cred_provider.py`** shows how to pass a custom credential provider to bypass connector authentication. Please install databricks-sdk prior to running this example. \ No newline at end of file +- **`sqlalchemy.py`** shows a basic example of connecting to Databricks with [SQLAlchemy 2.0](https://www.sqlalchemy.org/). +- **`custom_cred_provider.py`** shows how to pass a custom credential provider to bypass connector authentication. Please install databricks-sdk prior to running this example. +- **`v3_retries_query_execute.py`** shows how to enable v3 retries in connector version 2.9.x including how to enable retries for non-default retry cases. +- **`parameters.py`** shows how to use parameters in native and inline modes. diff --git a/examples/custom_cred_provider.py b/examples/custom_cred_provider.py index 4c43280f..67945f23 100644 --- a/examples/custom_cred_provider.py +++ b/examples/custom_cred_provider.py @@ -4,23 +4,27 @@ from databricks.sdk.oauth import OAuthClient import os -oauth_client = OAuthClient(host=os.getenv("DATABRICKS_SERVER_HOSTNAME"), - client_id=os.getenv("DATABRICKS_CLIENT_ID"), - client_secret=os.getenv("DATABRICKS_CLIENT_SECRET"), - redirect_url=os.getenv("APP_REDIRECT_URL"), - scopes=['all-apis', 'offline_access']) +oauth_client = OAuthClient( + host=os.getenv("DATABRICKS_SERVER_HOSTNAME"), + client_id=os.getenv("DATABRICKS_CLIENT_ID"), + client_secret=os.getenv("DATABRICKS_CLIENT_SECRET"), + redirect_url=os.getenv("APP_REDIRECT_URL"), + scopes=["all-apis", "offline_access"], +) consent = oauth_client.initiate_consent() creds = consent.launch_external_browser() -with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"), - http_path = os.getenv("DATABRICKS_HTTP_PATH"), - credentials_provider=creds) as connection: +with sql.connect( + server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), + http_path=os.getenv("DATABRICKS_HTTP_PATH"), + credentials_provider=creds, +) as connection: for x in range(1, 5): cursor = connection.cursor() - cursor.execute('SELECT 1+1') + cursor.execute("SELECT 1+1") result = cursor.fetchall() for row in result: print(row) diff --git a/examples/insert_data.py b/examples/insert_data.py index 511986aa..053ed158 100644 --- a/examples/insert_data.py +++ b/examples/insert_data.py @@ -1,21 +1,23 @@ from databricks import sql import os -with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"), - http_path = os.getenv("DATABRICKS_HTTP_PATH"), - access_token = os.getenv("DATABRICKS_TOKEN")) as connection: +with sql.connect( + server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), + http_path=os.getenv("DATABRICKS_HTTP_PATH"), + access_token=os.getenv("DATABRICKS_TOKEN"), +) as connection: - with connection.cursor() as cursor: - cursor.execute("CREATE TABLE IF NOT EXISTS squares (x int, x_squared int)") + with connection.cursor() as cursor: + cursor.execute("CREATE TABLE IF NOT EXISTS squares (x int, x_squared int)") - squares = [(i, i * i) for i in range(100)] - values = ",".join([f"({x}, {y})" for (x, y) in squares]) + squares = [(i, i * i) for i in range(100)] + values = ",".join([f"({x}, {y})" for (x, y) in squares]) - cursor.execute(f"INSERT INTO squares VALUES {values}") + cursor.execute(f"INSERT INTO squares VALUES {values}") - cursor.execute("SELECT * FROM squares LIMIT 10") + cursor.execute("SELECT * FROM squares LIMIT 10") - result = cursor.fetchall() + result = cursor.fetchall() - for row in result: - print(row) \ No newline at end of file + for row in result: + print(row) diff --git a/examples/interactive_oauth.py b/examples/interactive_oauth.py index c520d96a..8dbc8c47 100644 --- a/examples/interactive_oauth.py +++ b/examples/interactive_oauth.py @@ -1,38 +1,26 @@ from databricks import sql import os -"""Bring Your Own Identity Provider with fined grained OAuth scopes is currently public preview on -Databricks in AWS. databricks-sql-connector supports user to machine OAuth login which means the -end user has to be present to login in a browser which will be popped up by the Python process. You -must enable OAuth in your Databricks account to run this example. More information on how to enable -OAuth in your Databricks Account in AWS can be found here: - -https://docs.databricks.com/administration-guide/account-settings-e2/single-sign-on.html +"""databricks-sql-connector supports user to machine OAuth login which means the +end user has to be present to login in a browser which will be popped up by the Python process. Pre-requisites: -- You have a Databricks account in AWS. -- You have configured OAuth in Databricks account in AWS using the link above. - You have installed a browser (Chrome, Firefox, Safari, Internet Explorer, etc) that will be accessible on the machine for performing OAuth login. This code does not persist the auth token. Hence after the Python process terminates the end user will have to login again. See examples/persistent_oauth.py to learn about persisting the token across script executions. - -Bring Your Own Identity Provider is in public preview. The API may change prior to becoming GA. -You can monitor these two links to find out when it will become generally available: - - 1. https://docs.databricks.com/administration-guide/account-settings-e2/single-sign-on.html - 2. https://docs.databricks.com/dev-tools/python-sql-connector.html """ -with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"), - http_path = os.getenv("DATABRICKS_HTTP_PATH"), - auth_type="databricks-oauth") as connection: +with sql.connect( + server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), + http_path=os.getenv("DATABRICKS_HTTP_PATH"), +) as connection: for x in range(1, 100): cursor = connection.cursor() - cursor.execute('SELECT 1+1') + cursor.execute("SELECT 1+1") result = cursor.fetchall() for row in result: print(row) diff --git a/examples/m2m_oauth.py b/examples/m2m_oauth.py new file mode 100644 index 00000000..1c8c7278 --- /dev/null +++ b/examples/m2m_oauth.py @@ -0,0 +1,43 @@ +import os + +from databricks.sdk.core import oauth_service_principal, Config +from databricks import sql + +""" +This example shows how to use OAuth M2M (machine-to-machine) for service principal + +Pre-requisites: +- Create service principal and OAuth secret in Account Console +- Assign the service principal to the workspace + +See more https://docs.databricks.com/en/dev-tools/authentication-oauth.html) +""" + +server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME") + + +def credential_provider(): + config = Config( + host=f"https://{server_hostname}", + # Service Principal UUID + client_id=os.getenv("DATABRICKS_CLIENT_ID"), + # Service Principal Secret + client_secret=os.getenv("DATABRICKS_CLIENT_SECRET"), + ) + return oauth_service_principal(config) + + +with sql.connect( + server_hostname=server_hostname, + http_path=os.getenv("DATABRICKS_HTTP_PATH"), + credentials_provider=credential_provider, +) as connection: + for x in range(1, 100): + cursor = connection.cursor() + cursor.execute("SELECT 1+1") + result = cursor.fetchall() + for row in result: + print(row) + cursor.close() + + connection.close() diff --git a/examples/parameters.py b/examples/parameters.py new file mode 100644 index 00000000..93136ec7 --- /dev/null +++ b/examples/parameters.py @@ -0,0 +1,121 @@ +""" +This example demonstrates how to use parameters in both native (default) and inline (legacy) mode. +""" + +from decimal import Decimal +from databricks import sql +from databricks.sql.parameters import * + +import os +from databricks import sql +from datetime import datetime +import pytz + +host = os.getenv("DATABRICKS_SERVER_HOSTNAME") +http_path = os.getenv("DATABRICKS_HTTP_PATH") +access_token = os.getenv("DATABRICKS_TOKEN") + + +native_connection = sql.connect( + server_hostname=host, http_path=http_path, access_token=access_token +) + +inline_connection = sql.connect( + server_hostname=host, + http_path=http_path, + access_token=access_token, + use_inline_params="silent", +) + +# Example 1 demonstrates how in most cases, queries written for databricks-sql-connector<3.0.0 will work +# with databricks-sql-connector>=3.0.0. This is because the default mode is native mode, which is backwards +# compatible with the legacy inline mode. + +LEGACY_NAMED_QUERY = "SELECT %(name)s `name`, %(age)s `age`, %(active)s `active`" +EX1_PARAMS = {"name": "Jane", "age": 30, "active": True} + +with native_connection.cursor() as cursor: + ex1_native_result = cursor.execute(LEGACY_NAMED_QUERY, EX1_PARAMS).fetchone() + +with inline_connection.cursor() as cursor: + ex1_inline_result = cursor.execute(LEGACY_NAMED_QUERY, EX1_PARAMS).fetchone() + +print("\nEXAMPLE 1") +print("Example 1 result in native mode\t→\t", ex1_native_result) +print("Example 1 result in inline mode\t→\t", ex1_inline_result) + + +# Example 2 shows how to update example 1 to use the new `named` parameter markers. +# This query would fail in inline mode. + +# This is an example of the automatic transformation from pyformat → named. +# The output looks like this: +# SELECT :name `name`, :age `age`, :active `active` +NATIVE_NAMED_QUERY = LEGACY_NAMED_QUERY % { + "name": ":name", + "age": ":age", + "active": ":active", +} +EX2_PARAMS = EX1_PARAMS + +with native_connection.cursor() as cursor: + ex2_named_result = cursor.execute(NATIVE_NAMED_QUERY, EX1_PARAMS).fetchone() + +with native_connection.cursor() as cursor: + ex2_pyformat_result = cursor.execute(LEGACY_NAMED_QUERY, EX1_PARAMS).fetchone() + +print("\nEXAMPLE 2") +print("Example 2 result with pyformat \t→\t", ex2_named_result) +print("Example 2 result with named \t→\t", ex2_pyformat_result) + + +# Example 3 shows how to use positional parameters. Notice the syntax is different between native and inline modes. +# No automatic transformation is done here. So the LEGACY_POSITIONAL_QUERY will not work in native mode. + +NATIVE_POSITIONAL_QUERY = "SELECT ? `name`, ? `age`, ? `active`" +LEGACY_POSITIONAL_QUERY = "SELECT %s `name`, %s `age`, %s `active`" + +EX3_PARAMS = ["Jane", 30, True] + +with native_connection.cursor() as cursor: + ex3_native_result = cursor.execute(NATIVE_POSITIONAL_QUERY, EX3_PARAMS).fetchone() + +with inline_connection.cursor() as cursor: + ex3_inline_result = cursor.execute(LEGACY_POSITIONAL_QUERY, EX3_PARAMS).fetchone() + +print("\nEXAMPLE 3") +print("Example 3 result in native mode\t→\t", ex3_native_result) +print("Example 3 result in inline mode\t→\t", ex3_inline_result) + +# Example 4 shows how to bypass type inference and set an exact Databricks SQL type for a parameter. +# This is only possible when use_inline_params=False + + +moment = datetime(2012, 10, 15, 12, 57, 18) +chicago_timezone = pytz.timezone("America/Chicago") + +# For this parameter value, we don't bypass inference. So we know that the connector +# will infer the datetime object to be a TIMESTAMP, which preserves the timezone info. +ex4_p1 = chicago_timezone.localize(moment) + +# For this parameter value, we bypass inference and set the type to TIMESTAMP_NTZ, +# which does not preserve the timezone info. Therefore we expect the timezone +# will be dropped in the roundtrip. +ex4_p2 = TimestampNTZParameter(value=ex4_p1) + +# For this parameter, we don't bypass inference. So we know that the connector +# will infer the Decimal to be a DECIMAL and will preserve its current precision and scale. +ex4_p3 = Decimal("12.3456") + +# For this parameter value, we bind a decimal with custom scale and precision +# that will result in the decimal being truncated. +ex4_p4 = DecimalParameter(value=ex4_p3, scale=4, precision=2) + +EX4_QUERY = "SELECT ? `p1`, ? `p2`, ? `p3`, ? `p4`" +EX4_PARAMS = [ex4_p1, ex4_p2, ex4_p3, ex4_p4] +with native_connection.cursor() as cursor: + result = cursor.execute(EX4_QUERY, EX4_PARAMS).fetchone() + +print("\nEXAMPLE 4") +print("Example 4 inferred result\t→\t {}\t{}".format(result.p1, result.p3)) +print("Example 4 explicit result\t→\t {}\t\t{}".format(result.p2, result.p4)) diff --git a/examples/persistent_oauth.py b/examples/persistent_oauth.py index b5b14d15..1a2eded2 100644 --- a/examples/persistent_oauth.py +++ b/examples/persistent_oauth.py @@ -1,14 +1,7 @@ -"""Bring Your Own Identity Provider with fined grained OAuth scopes is currently public preview on -Databricks in AWS. databricks-sql-connector supports user to machine OAuth login which means the -end user has to be present to login in a browser which will be popped up by the Python process. You -must enable OAuth in your Databricks account to run this example. More information on how to enable -OAuth in your Databricks Account in AWS can be found here: - -https://docs.databricks.com/administration-guide/account-settings-e2/single-sign-on.html +"""databricks-sql-connector supports user to machine OAuth login which means the +end user has to be present to login in a browser which will be popped up by the Python process. Pre-requisites: -- You have a Databricks account in AWS. -- You have configured OAuth in Databricks account in AWS using the link above. - You have installed a browser (Chrome, Firefox, Safari, Internet Explorer, etc) that will be accessible on the machine for performing OAuth login. @@ -18,49 +11,50 @@ shows which methods you may implement. For this example, the DevOnlyFilePersistence class is provided. Do not use this in production. - -Bring Your Own Identity Provider is in public preview. The API may change prior to becoming GA. -You can monitor these two links to find out when it will become generally available: - - 1. https://docs.databricks.com/administration-guide/account-settings-e2/single-sign-on.html - 2. https://docs.databricks.com/dev-tools/python-sql-connector.html """ import os from typing import Optional from databricks import sql -from databricks.sql.experimental.oauth_persistence import OAuthPersistence, OAuthToken, DevOnlyFilePersistence +from databricks.sql.experimental.oauth_persistence import ( + OAuthPersistence, + OAuthToken, + DevOnlyFilePersistence, +) class SampleOAuthPersistence(OAuthPersistence): - def persist(self, hostname: str, oauth_token: OAuthToken): - """To be implemented by the end user to persist in the preferred storage medium. - - OAuthToken has two properties: - 1. OAuthToken.access_token - 2. OAuthToken.refresh_token + def persist(self, hostname: str, oauth_token: OAuthToken): + """To be implemented by the end user to persist in the preferred storage medium. + + OAuthToken has two properties: + 1. OAuthToken.access_token + 2. OAuthToken.refresh_token + + Both should be persisted. + """ + pass - Both should be persisted. - """ - pass + def read(self, hostname: str) -> Optional[OAuthToken]: + """To be implemented by the end user to fetch token from the preferred storage - def read(self, hostname: str) -> Optional[OAuthToken]: - """To be implemented by the end user to fetch token from the preferred storage + Fetch the access_token and refresh_token for the given hostname. + Return OAuthToken(access_token, refresh_token) + """ + pass - Fetch the access_token and refresh_token for the given hostname. - Return OAuthToken(access_token, refresh_token) - """ - pass -with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"), - http_path = os.getenv("DATABRICKS_HTTP_PATH"), - auth_type="databricks-oauth", - experimental_oauth_persistence=DevOnlyFilePersistence("./sample.json")) as connection: +with sql.connect( + server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), + http_path=os.getenv("DATABRICKS_HTTP_PATH"), + auth_type="databricks-oauth", + experimental_oauth_persistence=DevOnlyFilePersistence("./sample.json"), +) as connection: for x in range(1, 100): cursor = connection.cursor() - cursor.execute('SELECT 1+1') + cursor.execute("SELECT 1+1") result = cursor.fetchall() for row in result: print(row) diff --git a/examples/query_cancel.py b/examples/query_cancel.py index 59202088..b67fc085 100644 --- a/examples/query_cancel.py +++ b/examples/query_cancel.py @@ -5,47 +5,52 @@ The current operation of a cursor may be cancelled by calling its `.cancel()` method as shown in the example below. """ -with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"), - http_path = os.getenv("DATABRICKS_HTTP_PATH"), - access_token = os.getenv("DATABRICKS_TOKEN")) as connection: - - with connection.cursor() as cursor: - def execute_really_long_query(): - try: - cursor.execute("SELECT SUM(A.id - B.id) " + - "FROM range(1000000000) A CROSS JOIN range(100000000) B " + - "GROUP BY (A.id - B.id)") - except sql.exc.RequestError: - print("It looks like this query was cancelled.") - - exec_thread = threading.Thread(target=execute_really_long_query) - - print("\n Beginning to execute long query") - exec_thread.start() - - # Make sure the query has started before cancelling - print("\n Waiting 15 seconds before canceling", end="", flush=True) - - seconds_waited = 0 - while seconds_waited < 15: - seconds_waited += 1 - print(".", end="", flush=True) - time.sleep(1) - - print("\n Cancelling the cursor's operation. This can take a few seconds.") - cursor.cancel() - - print("\n Now checking the cursor status:") - exec_thread.join(5) - - assert not exec_thread.is_alive() - print("\n The previous command was successfully canceled") - - print("\n Now reusing the cursor to run a separate query.") - - # We can still execute a new command on the cursor - cursor.execute("SELECT * FROM range(3)") - - print("\n Execution was successful. Results appear below:") - - print(cursor.fetchall()) +with sql.connect( + server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), + http_path=os.getenv("DATABRICKS_HTTP_PATH"), + access_token=os.getenv("DATABRICKS_TOKEN"), +) as connection: + + with connection.cursor() as cursor: + + def execute_really_long_query(): + try: + cursor.execute( + "SELECT SUM(A.id - B.id) " + + "FROM range(1000000000) A CROSS JOIN range(100000000) B " + + "GROUP BY (A.id - B.id)" + ) + except sql.exc.RequestError: + print("It looks like this query was cancelled.") + + exec_thread = threading.Thread(target=execute_really_long_query) + + print("\n Beginning to execute long query") + exec_thread.start() + + # Make sure the query has started before cancelling + print("\n Waiting 15 seconds before canceling", end="", flush=True) + + seconds_waited = 0 + while seconds_waited < 15: + seconds_waited += 1 + print(".", end="", flush=True) + time.sleep(1) + + print("\n Cancelling the cursor's operation. This can take a few seconds.") + cursor.cancel() + + print("\n Now checking the cursor status:") + exec_thread.join(5) + + assert not exec_thread.is_alive() + print("\n The previous command was successfully canceled") + + print("\n Now reusing the cursor to run a separate query.") + + # We can still execute a new command on the cursor + cursor.execute("SELECT * FROM range(3)") + + print("\n Execution was successful. Results appear below:") + + print(cursor.fetchall()) diff --git a/examples/query_execute.py b/examples/query_execute.py index ec79fd0e..38d2f17a 100644 --- a/examples/query_execute.py +++ b/examples/query_execute.py @@ -1,13 +1,15 @@ from databricks import sql import os -with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"), - http_path = os.getenv("DATABRICKS_HTTP_PATH"), - access_token = os.getenv("DATABRICKS_TOKEN")) as connection: +with sql.connect( + server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), + http_path=os.getenv("DATABRICKS_HTTP_PATH"), + access_token=os.getenv("DATABRICKS_TOKEN"), +) as connection: - with connection.cursor() as cursor: - cursor.execute("SELECT * FROM default.diamonds LIMIT 2") - result = cursor.fetchall() + with connection.cursor() as cursor: + cursor.execute("SELECT * FROM default.diamonds LIMIT 2") + result = cursor.fetchall() - for row in result: - print(row) \ No newline at end of file + for row in result: + print(row) diff --git a/examples/set_user_agent.py b/examples/set_user_agent.py index 449692cf..93eb2e0b 100644 --- a/examples/set_user_agent.py +++ b/examples/set_user_agent.py @@ -1,14 +1,16 @@ from databricks import sql import os -with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"), - http_path = os.getenv("DATABRICKS_HTTP_PATH"), - access_token = os.getenv("DATABRICKS_TOKEN"), - _user_agent_entry="ExamplePartnerTag") as connection: +with sql.connect( + server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), + http_path=os.getenv("DATABRICKS_HTTP_PATH"), + access_token=os.getenv("DATABRICKS_TOKEN"), + _user_agent_entry="ExamplePartnerTag", +) as connection: - with connection.cursor() as cursor: - cursor.execute("SELECT * FROM default.diamonds LIMIT 2") - result = cursor.fetchall() + with connection.cursor() as cursor: + cursor.execute("SELECT * FROM default.diamonds LIMIT 2") + result = cursor.fetchall() - for row in result: - print(row) + for row in result: + print(row) diff --git a/examples/sqlalchemy.py b/examples/sqlalchemy.py deleted file mode 100644 index 2c0b693a..00000000 --- a/examples/sqlalchemy.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -databricks-sql-connector includes a SQLAlchemy dialect compatible with Databricks SQL. -It aims to be a drop-in replacement for the crflynn/sqlalchemy-databricks project, that implements -more of the Databricks API, particularly around table reflection, Alembic usage, and data -ingestion with pandas. - -Expected URI format is: databricks+thrift://token:dapi***@***.cloud.databricks.com?http_path=/sql/*** - -Because of the extent of SQLAlchemy's capabilities it isn't feasible to provide examples of every -usage in a single script, so we only provide a basic one here. More examples are found in our test -suite at tests/e2e/sqlalchemy/test_basic.py and in the PR that implements this change: - -https://github.com/databricks/databricks-sql-python/pull/57 - -# What's already supported - -Most of the functionality is demonstrated in the e2e tests mentioned above. The below list we -derived from those test method names: - - - Create and drop tables with SQLAlchemy Core - - Create and drop tables with SQLAlchemy ORM - - Read created tables via reflection - - Modify column nullability - - Insert records manually - - Insert records with pandas.to_sql (note that this does not work for DataFrames with indexes) - -This connector also aims to support Alembic for programmatic delta table schema maintenance. This -behaviour is not yet backed by integration tests, which will follow in a subsequent PR as we learn -more about customer use cases there. That said, the following behaviours have been tested manually: - - - Autogenerate revisions with alembic revision --autogenerate - - Upgrade and downgrade between revisions with `alembic upgrade ` and - `alembic downgrade ` - -# Known Gaps - - MAP, ARRAY, and STRUCT types: this dialect can read these types out as strings. But you cannot - define a SQLAlchemy model with databricks.sqlalchemy.dialect.types.DatabricksMap (e.g.) because - we haven't implemented them yet. - - Constraints: with the addition of information_schema to Unity Catalog, Databricks SQL supports - foreign key and primary key constraints. This dialect can write these constraints but the ability - for alembic to reflect and modify them programmatically has not been tested. -""" - -import os -from sqlalchemy.orm import declarative_base, Session -from sqlalchemy import Column, String, Integer, BOOLEAN, create_engine, select - -host = os.getenv("DATABRICKS_SERVER_HOSTNAME") -http_path = os.getenv("DATABRICKS_HTTP_PATH") -access_token = os.getenv("DATABRICKS_TOKEN") -catalog = os.getenv("DATABRICKS_CATALOG") -schema = os.getenv("DATABRICKS_SCHEMA") - - -# Extra arguments are passed untouched to the driver -# See thrift_backend.py for complete list -extra_connect_args = { - "_tls_verify_hostname": True, - "_user_agent_entry": "PySQL Example Script", -} - -engine = create_engine( - f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}", - connect_args=extra_connect_args, -) -session = Session(bind=engine) -base = declarative_base(bind=engine) - - -class SampleObject(base): - - __tablename__ = "mySampleTable" - - name = Column(String(255), primary_key=True) - episodes = Column(Integer) - some_bool = Column(BOOLEAN) - - -base.metadata.create_all() - -sample_object_1 = SampleObject(name="Bim Adewunmi", episodes=6, some_bool=True) -sample_object_2 = SampleObject(name="Miki Meek", episodes=12, some_bool=False) - -session.add(sample_object_1) -session.add(sample_object_2) - -session.commit() - -stmt = select(SampleObject).where(SampleObject.name.in_(["Bim Adewunmi", "Miki Meek"])) - -output = [i for i in session.scalars(stmt)] -assert len(output) == 2 - -base.metadata.drop_all() diff --git a/examples/staging_ingestion.py b/examples/staging_ingestion.py index 2980506d..a55be477 100644 --- a/examples/staging_ingestion.py +++ b/examples/staging_ingestion.py @@ -24,7 +24,7 @@ Additionally, the connection can only manipulate files within the cloud storage location of the authenticated user. -To run this script: +To run this script: 1. Set the INGESTION_USER constant to the account email address of the authenticated user 2. Set the FILEPATH constant to the path of a file that will be uploaded (this example assumes its a CSV file) diff --git a/examples/v3_retries_query_execute.py b/examples/v3_retries_query_execute.py new file mode 100644 index 00000000..aaab47d1 --- /dev/null +++ b/examples/v3_retries_query_execute.py @@ -0,0 +1,45 @@ +from databricks import sql +import os + +# Users of connector versions >= 2.9.0 and <= 3.0.0 can use the v3 retry behaviour by setting _enable_v3_retries=True +# This flag will be deprecated in databricks-sql-connector~=3.0.0 as it will become the default. +# +# The new retry behaviour is defined in src/databricks/sql/auth/retry.py +# +# The new retry behaviour allows users to force the connector to automatically retry requests that fail with codes +# that are not retried by default (in most cases only codes 429 and 503 are retried by default). Additional HTTP +# codes to retry are specified as a list passed to `_retry_dangerous_codes`. +# +# Note that, as implied in the name, doing this is *dangerous* and should not be configured in all usages. +# With the default behaviour, ExecuteStatement Thrift commands are only retried for codes 429 and 503 because +# we can be certain at run-time that the statement never reached Databricks compute. These codes are returned by +# the SQL gateway / load balancer. So there is no risk that retrying the request would result in a doubled +# (or tripled etc) command execution. These codes are always accompanied by a Retry-After header, which we honour. +# +# However, if your use-case emits idempotent queries such as SELECT statements, it can be helpful to retry +# for 502 (Bad Gateway) codes etc. In these cases, there is a possibility that the initial command _did_ reach +# Databricks compute and retrying it could result in additional executions. Retrying under these conditions uses +# an exponential back-off since a Retry-After header is not present. +# +# This new retry behaviour allows you to configure the maximum number of redirects that the connector will follow. +# Just set `_retry_max_redirects` to the integer number of redirects you want to allow. The default is None, +# which means all redirects will be followed. In this case, a redirect will count toward the +# _retry_stop_after_attempts_count which means that by default the connector will not enter an endless retry loop. +# +# For complete information about configuring retries, see the docstring for databricks.sql.thrift_backend.ThriftBackend + +with sql.connect( + server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), + http_path=os.getenv("DATABRICKS_HTTP_PATH"), + access_token=os.getenv("DATABRICKS_TOKEN"), + _enable_v3_retries=True, + _retry_dangerous_codes=[502, 400], + _retry_max_redirects=2, +) as connection: + + with connection.cursor() as cursor: + cursor.execute("SELECT * FROM default.diamonds LIMIT 2") + result = cursor.fetchall() + + for row in result: + print(row) diff --git a/poetry.lock b/poetry.lock index 3c95a628..2b63f135 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,43 +1,39 @@ -[[package]] -name = "alembic" -version = "1.10.4" -description = "A database migration tool for SQLAlchemy." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.9\""} -importlib-resources = {version = "*", markers = "python_version < \"3.9\""} -Mako = "*" -SQLAlchemy = ">=1.3.0" -typing-extensions = ">=4" - -[package.extras] -tz = ["python-dateutil"] +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "astroid" -version = "2.11.7" +version = "3.2.4" description = "An abstract syntax tree for Python with inference support." -category = "dev" optional = false -python-versions = ">=3.6.2" +python-versions = ">=3.8.0" +files = [ + {file = "astroid-3.2.4-py3-none-any.whl", hash = "sha256:413658a61eeca6202a59231abb473f932038fbcbf1666587f66d482083413a25"}, + {file = "astroid-3.2.4.tar.gz", hash = "sha256:0e14202810b30da1b735827f78f5157be2bbd4a7a59b7707ca0bfc2fb4c0063a"}, +] [package.dependencies] -lazy-object-proxy = ">=1.4.0" -setuptools = ">=20.0" -typed-ast = {version = ">=1.4.0,<2.0", markers = "implementation_name == \"cpython\" and python_version < \"3.8\""} -typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""} -wrapt = ">=1.11,<2" +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} [[package]] name = "black" version = "22.12.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, + {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, + {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, + {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, + {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, + {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, + {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, + {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, + {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, + {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, + {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, + {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, +] [package.dependencies] click = ">=8.0.0" @@ -45,7 +41,6 @@ mypy-extensions = ">=0.4.3" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} -typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] @@ -56,248 +51,450 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "certifi" -version = "2023.5.7" +version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, +] [[package]] name = "charset-normalizer" -version = "3.1.0" +version = "3.4.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" +files = [ + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, + {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, + {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, +] [[package]] name = "click" -version = "8.1.3" +version = "8.1.7" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "dill" -version = "0.3.6" -description = "serialize all of python" -category = "dev" +version = "0.3.9" +description = "serialize all of Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "dill-0.3.9-py3-none-any.whl", hash = "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a"}, + {file = "dill-0.3.9.tar.gz", hash = "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c"}, +] [package.extras] graph = ["objgraph (>=1.7.2)"] +profile = ["gprof2dot (>=2022.7.29)"] [[package]] name = "et-xmlfile" -version = "1.1.0" +version = "2.0.0" description = "An implementation of lxml.xmlfile for the standard library" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" +files = [ + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, +] [[package]] name = "exceptiongroup" -version = "1.1.1" +version = "1.2.2" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] [package.extras] test = ["pytest (>=6)"] -[[package]] -name = "greenlet" -version = "2.0.2" -description = "Lightweight in-process concurrent programming" -category = "main" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" - -[package.extras] -docs = ["Sphinx", "docutils (<0.18)"] -test = ["objgraph", "psutil"] - [[package]] name = "idna" -version = "3.4" +version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false -python-versions = ">=3.5" - -[[package]] -name = "importlib-metadata" -version = "6.6.0" -description = "Read metadata from Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "importlib-resources" -version = "5.12.0" -description = "Read resources from Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} +python-versions = ">=3.6" +files = [ + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, +] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] [[package]] name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] [[package]] name = "isort" -version = "5.11.5" +version = "5.13.2" description = "A Python utility / library to sort Python imports." -category = "dev" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] [package.extras] -colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - -[[package]] -name = "lazy-object-proxy" -version = "1.9.0" -description = "A fast and thorough lazy object proxy." -category = "dev" -optional = false -python-versions = ">=3.7" +colors = ["colorama (>=0.4.6)"] [[package]] name = "lz4" -version = "4.3.2" +version = "4.3.3" description = "LZ4 Bindings for Python" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "lz4-4.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b891880c187e96339474af2a3b2bfb11a8e4732ff5034be919aa9029484cd201"}, + {file = "lz4-4.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:222a7e35137d7539c9c33bb53fcbb26510c5748779364014235afc62b0ec797f"}, + {file = "lz4-4.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7"}, + {file = "lz4-4.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1d18718f9d78182c6b60f568c9a9cec8a7204d7cb6fad4e511a2ef279e4cb05"}, + {file = "lz4-4.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6cdc60e21ec70266947a48839b437d46025076eb4b12c76bd47f8e5eb8a75dcc"}, + {file = "lz4-4.3.3-cp310-cp310-win32.whl", hash = "sha256:c81703b12475da73a5d66618856d04b1307e43428a7e59d98cfe5a5d608a74c6"}, + {file = "lz4-4.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:43cf03059c0f941b772c8aeb42a0813d68d7081c009542301637e5782f8a33e2"}, + {file = "lz4-4.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:30e8c20b8857adef7be045c65f47ab1e2c4fabba86a9fa9a997d7674a31ea6b6"}, + {file = "lz4-4.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2f7b1839f795315e480fb87d9bc60b186a98e3e5d17203c6e757611ef7dcef61"}, + {file = "lz4-4.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edfd858985c23523f4e5a7526ca6ee65ff930207a7ec8a8f57a01eae506aaee7"}, + {file = "lz4-4.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e9c410b11a31dbdc94c05ac3c480cb4b222460faf9231f12538d0074e56c563"}, + {file = "lz4-4.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d2507ee9c99dbddd191c86f0e0c8b724c76d26b0602db9ea23232304382e1f21"}, + {file = "lz4-4.3.3-cp311-cp311-win32.whl", hash = "sha256:f180904f33bdd1e92967923a43c22899e303906d19b2cf8bb547db6653ea6e7d"}, + {file = "lz4-4.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:b14d948e6dce389f9a7afc666d60dd1e35fa2138a8ec5306d30cd2e30d36b40c"}, + {file = "lz4-4.3.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e36cd7b9d4d920d3bfc2369840da506fa68258f7bb176b8743189793c055e43d"}, + {file = "lz4-4.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:31ea4be9d0059c00b2572d700bf2c1bc82f241f2c3282034a759c9a4d6ca4dc2"}, + {file = "lz4-4.3.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33c9a6fd20767ccaf70649982f8f3eeb0884035c150c0b818ea660152cf3c809"}, + {file = "lz4-4.3.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca8fccc15e3add173da91be8f34121578dc777711ffd98d399be35487c934bf"}, + {file = "lz4-4.3.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d84b479ddf39fe3ea05387f10b779155fc0990125f4fb35d636114e1c63a2e"}, + {file = "lz4-4.3.3-cp312-cp312-win32.whl", hash = "sha256:337cb94488a1b060ef1685187d6ad4ba8bc61d26d631d7ba909ee984ea736be1"}, + {file = "lz4-4.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:5d35533bf2cee56f38ced91f766cd0038b6abf46f438a80d50c52750088be93f"}, + {file = "lz4-4.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:363ab65bf31338eb364062a15f302fc0fab0a49426051429866d71c793c23394"}, + {file = "lz4-4.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0a136e44a16fc98b1abc404fbabf7f1fada2bdab6a7e970974fb81cf55b636d0"}, + {file = "lz4-4.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abc197e4aca8b63f5ae200af03eb95fb4b5055a8f990079b5bdf042f568469dd"}, + {file = "lz4-4.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56f4fe9c6327adb97406f27a66420b22ce02d71a5c365c48d6b656b4aaeb7775"}, + {file = "lz4-4.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0e822cd7644995d9ba248cb4b67859701748a93e2ab7fc9bc18c599a52e4604"}, + {file = "lz4-4.3.3-cp38-cp38-win32.whl", hash = "sha256:24b3206de56b7a537eda3a8123c644a2b7bf111f0af53bc14bed90ce5562d1aa"}, + {file = "lz4-4.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:b47839b53956e2737229d70714f1d75f33e8ac26e52c267f0197b3189ca6de24"}, + {file = "lz4-4.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6756212507405f270b66b3ff7f564618de0606395c0fe10a7ae2ffcbbe0b1fba"}, + {file = "lz4-4.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee9ff50557a942d187ec85462bb0960207e7ec5b19b3b48949263993771c6205"}, + {file = "lz4-4.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b901c7784caac9a1ded4555258207d9e9697e746cc8532129f150ffe1f6ba0d"}, + {file = "lz4-4.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d9ec061b9eca86e4dcc003d93334b95d53909afd5a32c6e4f222157b50c071"}, + {file = "lz4-4.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0"}, + {file = "lz4-4.3.3-cp39-cp39-win32.whl", hash = "sha256:054b4631a355606e99a42396f5db4d22046a3397ffc3269a348ec41eaebd69d2"}, + {file = "lz4-4.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:eac9af361e0d98335a02ff12fb56caeb7ea1196cf1a49dbf6f17828a131da807"}, + {file = "lz4-4.3.3.tar.gz", hash = "sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e"}, +] [package.extras] docs = ["sphinx (>=1.6.0)", "sphinx-bootstrap-theme"] flake8 = ["flake8"] tests = ["psutil", "pytest (!=3.3.0)", "pytest-cov"] -[[package]] -name = "mako" -version = "1.2.4" -description = "A super-fast templating language that borrows the best ideas from the existing templating languages." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} -MarkupSafe = ">=0.9.2" - -[package.extras] -babel = ["Babel"] -lingua = ["lingua"] -testing = ["pytest"] - -[[package]] -name = "markupsafe" -version = "2.1.2" -description = "Safely add untrusted strings to HTML/XML markup." -category = "main" -optional = false -python-versions = ">=3.7" - [[package]] name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] [[package]] name = "mypy" -version = "0.950" +version = "1.13.0" description = "Optional static typing for Python" -category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" +files = [ + {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, + {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, + {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, + {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, + {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, + {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, + {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, + {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, + {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, + {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, + {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, + {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, + {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, + {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, + {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, + {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, + {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, + {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, + {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, + {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, + {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, + {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, +] [package.dependencies] -mypy-extensions = ">=0.4.3" +mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} -typing-extensions = ">=3.10" +typing-extensions = ">=4.6.0" [package.extras] dmypy = ["psutil (>=4.0)"] -python2 = ["typed-ast (>=1.4.0,<2)"] +faster-cache = ["orjson"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] reports = ["lxml"] [[package]] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] [[package]] name = "numpy" -version = "1.21.6" -description = "NumPy is the fundamental package for array computing with Python." -category = "main" +version = "1.24.4" +description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.7,<3.11" +python-versions = ">=3.8" +files = [ + {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, + {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6"}, + {file = "numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc"}, + {file = "numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5"}, + {file = "numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d"}, + {file = "numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc"}, + {file = "numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2"}, + {file = "numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d"}, + {file = "numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835"}, + {file = "numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2"}, + {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, +] [[package]] name = "numpy" -version = "1.24.3" +version = "1.26.4" description = "Fundamental package for array computing in Python" -category = "main" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] [[package]] name = "oauthlib" version = "3.2.2" description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] [package.extras] rsa = ["cryptography (>=3.0.0)"] @@ -306,77 +503,133 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "openpyxl" -version = "3.1.2" +version = "3.1.5" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] [package.dependencies] et-xmlfile = "*" [[package]] name = "packaging" -version = "23.1" +version = "24.2" description = "Core utilities for Python packages" -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, +] [[package]] name = "pandas" -version = "1.3.5" +version = "2.0.3" description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" optional = false -python-versions = ">=3.7.1" +python-versions = ">=3.8" +files = [ + {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, + {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0"}, + {file = "pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210"}, + {file = "pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df"}, + {file = "pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd"}, + {file = "pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0"}, + {file = "pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02"}, + {file = "pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641"}, + {file = "pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682"}, + {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, + {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, +] [package.dependencies] numpy = [ - {version = ">=1.17.3", markers = "platform_machine != \"aarch64\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, - {version = ">=1.19.2", markers = "platform_machine == \"aarch64\" and python_version < \"3.10\""}, - {version = ">=1.20.0", markers = "platform_machine == \"arm64\" and python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] -python-dateutil = ">=2.7.3" -pytz = ">=2017.3" +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" [package.extras] -test = ["hypothesis (>=3.58)", "pytest (>=6.0)", "pytest-xdist"] +all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] +aws = ["s3fs (>=2021.08.0)"] +clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] +compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] +computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2021.07.0)"] +gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] +hdf5 = ["tables (>=3.6.1)"] +html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] +mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] +spss = ["pyreadstat (>=1.1.2)"] +sql-other = ["SQLAlchemy (>=1.4.16)"] +test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.6.3)"] [[package]] name = "pathspec" -version = "0.11.1" +version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] [[package]] name = "platformdirs" -version = "3.5.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" +version = "4.3.6" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = {version = ">=4.5", markers = "python_version < \"3.8\""} +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, +] [package.extras] -docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.11.2)"] [[package]] name = "pluggy" -version = "1.0.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false -python-versions = ">=3.6" - -[package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] [package.extras] dev = ["pre-commit", "tox"] @@ -384,82 +637,171 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pyarrow" -version = "12.0.0" +version = "17.0.0" description = "Python library for Apache Arrow" -category = "main" -optional = false -python-versions = ">=3.7" +optional = true +python-versions = ">=3.8" +files = [ + {file = "pyarrow-17.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07"}, + {file = "pyarrow-17.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db023dc4c6cae1015de9e198d41250688383c3f9af8f565370ab2b4cb5f62655"}, + {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da1e060b3876faa11cee287839f9cc7cdc00649f475714b8680a05fd9071d545"}, + {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c06d4624c0ad6674364bb46ef38c3132768139ddec1c56582dbac54f2663e2"}, + {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:fa3c246cc58cb5a4a5cb407a18f193354ea47dd0648194e6265bd24177982fe8"}, + {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:f7ae2de664e0b158d1607699a16a488de3d008ba99b3a7aa5de1cbc13574d047"}, + {file = "pyarrow-17.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5984f416552eea15fd9cee03da53542bf4cddaef5afecefb9aa8d1010c335087"}, + {file = "pyarrow-17.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:1c8856e2ef09eb87ecf937104aacfa0708f22dfeb039c363ec99735190ffb977"}, + {file = "pyarrow-17.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e19f569567efcbbd42084e87f948778eb371d308e137a0f97afe19bb860ccb3"}, + {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b244dc8e08a23b3e352899a006a26ae7b4d0da7bb636872fa8f5884e70acf15"}, + {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b72e87fe3e1db343995562f7fff8aee354b55ee83d13afba65400c178ab2597"}, + {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dc5c31c37409dfbc5d014047817cb4ccd8c1ea25d19576acf1a001fe07f5b420"}, + {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e3343cb1e88bc2ea605986d4b94948716edc7a8d14afd4e2c097232f729758b4"}, + {file = "pyarrow-17.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a27532c38f3de9eb3e90ecab63dfda948a8ca859a66e3a47f5f42d1e403c4d03"}, + {file = "pyarrow-17.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22"}, + {file = "pyarrow-17.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1e70de6cb5790a50b01d2b686d54aaf73da01266850b05e3af2a1bc89e16053"}, + {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0071ce35788c6f9077ff9ecba4858108eebe2ea5a3f7cf2cf55ebc1dbc6ee24a"}, + {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:757074882f844411fcca735e39aae74248a1531367a7c80799b4266390ae51cc"}, + {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ba11c4f16976e89146781a83833df7f82077cdab7dc6232c897789343f7891a"}, + {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b0c6ac301093b42d34410b187bba560b17c0330f64907bfa4f7f7f2444b0cf9b"}, + {file = "pyarrow-17.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:392bc9feabc647338e6c89267635e111d71edad5fcffba204425a7c8d13610d7"}, + {file = "pyarrow-17.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:af5ff82a04b2171415f1410cff7ebb79861afc5dae50be73ce06d6e870615204"}, + {file = "pyarrow-17.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:edca18eaca89cd6382dfbcff3dd2d87633433043650c07375d095cd3517561d8"}, + {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c7916bff914ac5d4a8fe25b7a25e432ff921e72f6f2b7547d1e325c1ad9d155"}, + {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f553ca691b9e94b202ff741bdd40f6ccb70cdd5fbf65c187af132f1317de6145"}, + {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0cdb0e627c86c373205a2f94a510ac4376fdc523f8bb36beab2e7f204416163c"}, + {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:d7d192305d9d8bc9082d10f361fc70a73590a4c65cf31c3e6926cd72b76bc35c"}, + {file = "pyarrow-17.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:02dae06ce212d8b3244dd3e7d12d9c4d3046945a5933d28026598e9dbbda1fca"}, + {file = "pyarrow-17.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:13d7a460b412f31e4c0efa1148e1d29bdf18ad1411eb6757d38f8fbdcc8645fb"}, + {file = "pyarrow-17.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9b564a51fbccfab5a04a80453e5ac6c9954a9c5ef2890d1bcf63741909c3f8df"}, + {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32503827abbc5aadedfa235f5ece8c4f8f8b0a3cf01066bc8d29de7539532687"}, + {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a155acc7f154b9ffcc85497509bcd0d43efb80d6f733b0dc3bb14e281f131c8b"}, + {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:dec8d129254d0188a49f8a1fc99e0560dc1b85f60af729f47de4046015f9b0a5"}, + {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:a48ddf5c3c6a6c505904545c25a4ae13646ae1f8ba703c4df4a1bfe4f4006bda"}, + {file = "pyarrow-17.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:42bf93249a083aca230ba7e2786c5f673507fa97bbd9725a1e2754715151a204"}, + {file = "pyarrow-17.0.0.tar.gz", hash = "sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28"}, +] [package.dependencies] numpy = ">=1.16.6" +[package.extras] +test = ["cffi", "hypothesis", "pandas", "pytest", "pytz"] + [[package]] name = "pylint" -version = "2.13.9" +version = "3.2.7" description = "python code static checker" -category = "dev" optional = false -python-versions = ">=3.6.2" +python-versions = ">=3.8.0" +files = [ + {file = "pylint-3.2.7-py3-none-any.whl", hash = "sha256:02f4aedeac91be69fb3b4bea997ce580a4ac68ce58b89eaefeaf06749df73f4b"}, + {file = "pylint-3.2.7.tar.gz", hash = "sha256:1b7a721b575eaeaa7d39db076b6e7743c993ea44f57979127c517c6c572c803e"}, +] [package.dependencies] -astroid = ">=2.11.5,<=2.12.0-dev0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -dill = ">=0.2" -isort = ">=4.2.5,<6" +astroid = ">=3.2.4,<=3.3.0-dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +dill = [ + {version = ">=0.2", markers = "python_version < \"3.11\""}, + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, + {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, +] +isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" mccabe = ">=0.6,<0.8" platformdirs = ">=2.2.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +tomlkit = ">=0.10.1" typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] -testutil = ["gitpython (>3)"] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] [[package]] name = "pytest" -version = "7.3.1" +version = "7.4.4" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, +] [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-dotenv" +version = "0.5.2" +description = "A py.test plugin that parses environment files before running tests" +optional = false +python-versions = "*" +files = [ + {file = "pytest-dotenv-0.5.2.tar.gz", hash = "sha256:2dc6c3ac6d8764c71c6d2804e902d0ff810fa19692e95fe138aefc9b1aa73732"}, + {file = "pytest_dotenv-0.5.2-py3-none-any.whl", hash = "sha256:40a2cece120a213898afaa5407673f6bd924b1fa7eafce6bda0e8abffe2f710f"}, +] + +[package.dependencies] +pytest = ">=5.0.0" +python-dotenv = ">=0.9.1" [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "pytz" -version = "2023.3" +version = "2024.2" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" +files = [ + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, +] [[package]] name = "requests" -version = "2.30.0" +version = "2.32.3" description = "Python HTTP for Humans." -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] [package.dependencies] certifi = ">=2017.4.17" @@ -471,67 +813,26 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] -[[package]] -name = "setuptools" -version = "67.7.2" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - [[package]] name = "six" -version = "1.16.0" +version = "1.17.0" description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "sqlalchemy" -version = "1.4.48" -description = "Database Abstraction Library" -category = "main" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" - -[package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} - -[package.extras] -aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] -aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] -asyncio = ["greenlet (!=0.4.17)"] -asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] -mssql = ["pyodbc"] -mssql-pymssql = ["pymssql"] -mssql-pyodbc = ["pyodbc"] -mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] -mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] -mysql-connector = ["mysql-connector-python"] -oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] -postgresql = ["psycopg2 (>=2.7)"] -postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] -postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] -postgresql-psycopg2binary = ["psycopg2-binary"] -postgresql-psycopg2cffi = ["psycopg2cffi"] -pymysql = ["pymysql", "pymysql (<1)"] -sqlcipher = ["sqlcipher3_binary"] +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] [[package]] name = "thrift" -version = "0.16.0" +version = "0.20.0" description = "Python bindings for the Apache Thrift RPC system" -category = "main" optional = false python-versions = "*" +files = [ + {file = "thrift-0.20.0.tar.gz", hash = "sha256:4dd662eadf6b8aebe8a41729527bd69adf6ceaa2a8681cbef64d1273b3e8feba"}, +] [package.dependencies] six = ">=1.7.2" @@ -543,766 +844,99 @@ twisted = ["twisted"] [[package]] name = "tomli" -version = "2.0.1" +version = "2.2.1" description = "A lil' TOML parser" -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, +] + +[[package]] +name = "tomlkit" +version = "0.13.2" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, +] [[package]] -name = "typed-ast" -version = "1.5.4" -description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] [[package]] -name = "typing-extensions" -version = "4.5.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" +name = "tzdata" +version = "2024.2" +description = "Provider of IANA time zone data" optional = false -python-versions = ">=3.7" +python-versions = ">=2" +files = [ + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, +] [[package]] name = "urllib3" -version = "2.0.2" +version = "2.2.3" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, + {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, +] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] -[[package]] -name = "wrapt" -version = "1.15.0" -description = "Module for decorators, wrappers and monkey patching." -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[[package]] -name = "zipp" -version = "3.15.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +[extras] +pyarrow = ["pyarrow"] [metadata] -lock-version = "1.1" -python-versions = "^3.7.1" -content-hash = "8432ddba9b066e5b1c34ca44918443f1f7566d95e4f0c0a9b630dd95b95bb71e" - -[metadata.files] -alembic = [ - {file = "alembic-1.10.4-py3-none-any.whl", hash = "sha256:43942c3d4bf2620c466b91c0f4fca136fe51ae972394a0cc8b90810d664e4f5c"}, - {file = "alembic-1.10.4.tar.gz", hash = "sha256:295b54bbb92c4008ab6a7dcd1e227e668416d6f84b98b3c4446a2bc6214a556b"}, -] -astroid = [ - {file = "astroid-2.11.7-py3-none-any.whl", hash = "sha256:86b0a340a512c65abf4368b80252754cda17c02cdbbd3f587dddf98112233e7b"}, - {file = "astroid-2.11.7.tar.gz", hash = "sha256:bb24615c77f4837c707669d16907331374ae8a964650a66999da3f5ca68dc946"}, -] -black = [ - {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, - {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, - {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, - {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, - {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, - {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, - {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, - {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, - {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, - {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, - {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, - {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, -] -certifi = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, -] -charset-normalizer = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -dill = [ - {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, - {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, -] -et-xmlfile = [ - {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, - {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, -] -exceptiongroup = [ - {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, - {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, -] -greenlet = [ - {file = "greenlet-2.0.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d"}, - {file = "greenlet-2.0.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9"}, - {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, - {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, - {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, - {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470"}, - {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a"}, - {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, - {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, - {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, - {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19"}, - {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3"}, - {file = "greenlet-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5"}, - {file = "greenlet-2.0.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6"}, - {file = "greenlet-2.0.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43"}, - {file = "greenlet-2.0.2-cp35-cp35m-win32.whl", hash = "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a"}, - {file = "greenlet-2.0.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394"}, - {file = "greenlet-2.0.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75"}, - {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf"}, - {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292"}, - {file = "greenlet-2.0.2-cp36-cp36m-win32.whl", hash = "sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9"}, - {file = "greenlet-2.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f"}, - {file = "greenlet-2.0.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73"}, - {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86"}, - {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33"}, - {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, - {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, - {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857"}, - {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a"}, - {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, - {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, - {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, - {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b"}, - {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8"}, - {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9"}, - {file = "greenlet-2.0.2-cp39-cp39-win32.whl", hash = "sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5"}, - {file = "greenlet-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564"}, - {file = "greenlet-2.0.2.tar.gz", hash = "sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0"}, -] -idna = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] -importlib-metadata = [ - {file = "importlib_metadata-6.6.0-py3-none-any.whl", hash = "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed"}, - {file = "importlib_metadata-6.6.0.tar.gz", hash = "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"}, -] -importlib-resources = [ - {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, - {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, -] -iniconfig = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] -isort = [ - {file = "isort-5.11.5-py3-none-any.whl", hash = "sha256:ba1d72fb2595a01c7895a5128f9585a5cc4b6d395f1c8d514989b9a7eb2a8746"}, - {file = "isort-5.11.5.tar.gz", hash = "sha256:6be1f76a507cb2ecf16c7cf14a37e41609ca082330be4e3436a18ef74add55db"}, -] -lazy-object-proxy = [ - {file = "lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, -] -lz4 = [ - {file = "lz4-4.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1c4c100d99eed7c08d4e8852dd11e7d1ec47a3340f49e3a96f8dfbba17ffb300"}, - {file = "lz4-4.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:edd8987d8415b5dad25e797043936d91535017237f72fa456601be1479386c92"}, - {file = "lz4-4.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7c50542b4ddceb74ab4f8b3435327a0861f06257ca501d59067a6a482535a77"}, - {file = "lz4-4.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f5614d8229b33d4a97cb527db2a1ac81308c6e796e7bdb5d1309127289f69d5"}, - {file = "lz4-4.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f00a9ba98f6364cadda366ae6469b7b3568c0cced27e16a47ddf6b774169270"}, - {file = "lz4-4.3.2-cp310-cp310-win32.whl", hash = "sha256:b10b77dc2e6b1daa2f11e241141ab8285c42b4ed13a8642495620416279cc5b2"}, - {file = "lz4-4.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:86480f14a188c37cb1416cdabacfb4e42f7a5eab20a737dac9c4b1c227f3b822"}, - {file = "lz4-4.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7c2df117def1589fba1327dceee51c5c2176a2b5a7040b45e84185ce0c08b6a3"}, - {file = "lz4-4.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1f25eb322eeb24068bb7647cae2b0732b71e5c639e4e4026db57618dcd8279f0"}, - {file = "lz4-4.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8df16c9a2377bdc01e01e6de5a6e4bbc66ddf007a6b045688e285d7d9d61d1c9"}, - {file = "lz4-4.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f571eab7fec554d3b1db0d666bdc2ad85c81f4b8cb08906c4c59a8cad75e6e22"}, - {file = "lz4-4.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7211dc8f636ca625abc3d4fb9ab74e5444b92df4f8d58ec83c8868a2b0ff643d"}, - {file = "lz4-4.3.2-cp311-cp311-win32.whl", hash = "sha256:867664d9ca9bdfce840ac96d46cd8838c9ae891e859eb98ce82fcdf0e103a947"}, - {file = "lz4-4.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:a6a46889325fd60b8a6b62ffc61588ec500a1883db32cddee9903edfba0b7584"}, - {file = "lz4-4.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3a85b430138882f82f354135b98c320dafb96fc8fe4656573d95ab05de9eb092"}, - {file = "lz4-4.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65d5c93f8badacfa0456b660285e394e65023ef8071142e0dcbd4762166e1be0"}, - {file = "lz4-4.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b50f096a6a25f3b2edca05aa626ce39979d63c3b160687c8c6d50ac3943d0ba"}, - {file = "lz4-4.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:200d05777d61ba1ff8d29cb51c534a162ea0b4fe6d3c28be3571a0a48ff36080"}, - {file = "lz4-4.3.2-cp37-cp37m-win32.whl", hash = "sha256:edc2fb3463d5d9338ccf13eb512aab61937be50aa70734bcf873f2f493801d3b"}, - {file = "lz4-4.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:83acfacab3a1a7ab9694333bcb7950fbeb0be21660d236fd09c8337a50817897"}, - {file = "lz4-4.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a9eec24ec7d8c99aab54de91b4a5a149559ed5b3097cf30249b665689b3d402"}, - {file = "lz4-4.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:31d72731c4ac6ebdce57cd9a5cabe0aecba229c4f31ba3e2c64ae52eee3fdb1c"}, - {file = "lz4-4.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83903fe6db92db0be101acedc677aa41a490b561567fe1b3fe68695b2110326c"}, - {file = "lz4-4.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926b26db87ec8822cf1870efc3d04d06062730ec3279bbbd33ba47a6c0a5c673"}, - {file = "lz4-4.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e05afefc4529e97c08e65ef92432e5f5225c0bb21ad89dee1e06a882f91d7f5e"}, - {file = "lz4-4.3.2-cp38-cp38-win32.whl", hash = "sha256:ad38dc6a7eea6f6b8b642aaa0683253288b0460b70cab3216838747163fb774d"}, - {file = "lz4-4.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:7e2dc1bd88b60fa09b9b37f08553f45dc2b770c52a5996ea52b2b40f25445676"}, - {file = "lz4-4.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:edda4fb109439b7f3f58ed6bede59694bc631c4b69c041112b1b7dc727fffb23"}, - {file = "lz4-4.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ca83a623c449295bafad745dcd399cea4c55b16b13ed8cfea30963b004016c9"}, - {file = "lz4-4.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5ea0e788dc7e2311989b78cae7accf75a580827b4d96bbaf06c7e5a03989bd5"}, - {file = "lz4-4.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a98b61e504fb69f99117b188e60b71e3c94469295571492a6468c1acd63c37ba"}, - {file = "lz4-4.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4931ab28a0d1c133104613e74eec1b8bb1f52403faabe4f47f93008785c0b929"}, - {file = "lz4-4.3.2-cp39-cp39-win32.whl", hash = "sha256:ec6755cacf83f0c5588d28abb40a1ac1643f2ff2115481089264c7630236618a"}, - {file = "lz4-4.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:4caedeb19e3ede6c7a178968b800f910db6503cb4cb1e9cc9221157572139b49"}, - {file = "lz4-4.3.2.tar.gz", hash = "sha256:e1431d84a9cfb23e6773e72078ce8e65cad6745816d4cbf9ae67da5ea419acda"}, -] -mako = [ - {file = "Mako-1.2.4-py3-none-any.whl", hash = "sha256:c97c79c018b9165ac9922ae4f32da095ffd3c4e6872b45eded42926deea46818"}, - {file = "Mako-1.2.4.tar.gz", hash = "sha256:d60a3903dc3bb01a18ad6a89cdbe2e4eadc69c0bc8ef1e3773ba53d44c3f7a34"}, -] -markupsafe = [ - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, - {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, -] -mccabe = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] -mypy = [ - {file = "mypy-0.950-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b"}, - {file = "mypy-0.950-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0"}, - {file = "mypy-0.950-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22"}, - {file = "mypy-0.950-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb"}, - {file = "mypy-0.950-cp310-cp310-win_amd64.whl", hash = "sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334"}, - {file = "mypy-0.950-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f"}, - {file = "mypy-0.950-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc"}, - {file = "mypy-0.950-cp36-cp36m-win_amd64.whl", hash = "sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2"}, - {file = "mypy-0.950-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed"}, - {file = "mypy-0.950-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075"}, - {file = "mypy-0.950-cp37-cp37m-win_amd64.whl", hash = "sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b"}, - {file = "mypy-0.950-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d"}, - {file = "mypy-0.950-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a"}, - {file = "mypy-0.950-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605"}, - {file = "mypy-0.950-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2"}, - {file = "mypy-0.950-cp38-cp38-win_amd64.whl", hash = "sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff"}, - {file = "mypy-0.950-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8"}, - {file = "mypy-0.950-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038"}, - {file = "mypy-0.950-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2"}, - {file = "mypy-0.950-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519"}, - {file = "mypy-0.950-cp39-cp39-win_amd64.whl", hash = "sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef"}, - {file = "mypy-0.950-py3-none-any.whl", hash = "sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb"}, - {file = "mypy-0.950.tar.gz", hash = "sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de"}, -] -mypy-extensions = [ - {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, - {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, -] -numpy = [ - {file = "numpy-1.21.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8737609c3bbdd48e380d463134a35ffad3b22dc56295eff6f79fd85bd0eeeb25"}, - {file = "numpy-1.21.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fdffbfb6832cd0b300995a2b08b8f6fa9f6e856d562800fea9182316d99c4e8e"}, - {file = "numpy-1.21.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3820724272f9913b597ccd13a467cc492a0da6b05df26ea09e78b171a0bb9da6"}, - {file = "numpy-1.21.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f17e562de9edf691a42ddb1eb4a5541c20dd3f9e65b09ded2beb0799c0cf29bb"}, - {file = "numpy-1.21.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f30427731561ce75d7048ac254dbe47a2ba576229250fb60f0fb74db96501a1"}, - {file = "numpy-1.21.6-cp310-cp310-win32.whl", hash = "sha256:d4bf4d43077db55589ffc9009c0ba0a94fa4908b9586d6ccce2e0b164c86303c"}, - {file = "numpy-1.21.6-cp310-cp310-win_amd64.whl", hash = "sha256:d136337ae3cc69aa5e447e78d8e1514be8c3ec9b54264e680cf0b4bd9011574f"}, - {file = "numpy-1.21.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6aaf96c7f8cebc220cdfc03f1d5a31952f027dda050e5a703a0d1c396075e3e7"}, - {file = "numpy-1.21.6-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:67c261d6c0a9981820c3a149d255a76918278a6b03b6a036800359aba1256d46"}, - {file = "numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a6be4cb0ef3b8c9250c19cc122267263093eee7edd4e3fa75395dfda8c17a8e2"}, - {file = "numpy-1.21.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c4068a8c44014b2d55f3c3f574c376b2494ca9cc73d2f1bd692382b6dffe3db"}, - {file = "numpy-1.21.6-cp37-cp37m-win32.whl", hash = "sha256:7c7e5fa88d9ff656e067876e4736379cc962d185d5cd808014a8a928d529ef4e"}, - {file = "numpy-1.21.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bcb238c9c96c00d3085b264e5c1a1207672577b93fa666c3b14a45240b14123a"}, - {file = "numpy-1.21.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:82691fda7c3f77c90e62da69ae60b5ac08e87e775b09813559f8901a88266552"}, - {file = "numpy-1.21.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:643843bcc1c50526b3a71cd2ee561cf0d8773f062c8cbaf9ffac9fdf573f83ab"}, - {file = "numpy-1.21.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:357768c2e4451ac241465157a3e929b265dfac85d9214074985b1786244f2ef3"}, - {file = "numpy-1.21.6-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9f411b2c3f3d76bba0865b35a425157c5dcf54937f82bbeb3d3c180789dd66a6"}, - {file = "numpy-1.21.6-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4aa48afdce4660b0076a00d80afa54e8a97cd49f457d68a4342d188a09451c1a"}, - {file = "numpy-1.21.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a96eef20f639e6a97d23e57dd0c1b1069a7b4fd7027482a4c5c451cd7732f4"}, - {file = "numpy-1.21.6-cp38-cp38-win32.whl", hash = "sha256:5c3c8def4230e1b959671eb959083661b4a0d2e9af93ee339c7dada6759a9470"}, - {file = "numpy-1.21.6-cp38-cp38-win_amd64.whl", hash = "sha256:bf2ec4b75d0e9356edea834d1de42b31fe11f726a81dfb2c2112bc1eaa508fcf"}, - {file = "numpy-1.21.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4391bd07606be175aafd267ef9bea87cf1b8210c787666ce82073b05f202add1"}, - {file = "numpy-1.21.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:67f21981ba2f9d7ba9ade60c9e8cbaa8cf8e9ae51673934480e45cf55e953673"}, - {file = "numpy-1.21.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee5ec40fdd06d62fe5d4084bef4fd50fd4bb6bfd2bf519365f569dc470163ab0"}, - {file = "numpy-1.21.6-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1dbe1c91269f880e364526649a52eff93ac30035507ae980d2fed33aaee633ac"}, - {file = "numpy-1.21.6-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d9caa9d5e682102453d96a0ee10c7241b72859b01a941a397fd965f23b3e016b"}, - {file = "numpy-1.21.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58459d3bad03343ac4b1b42ed14d571b8743dc80ccbf27444f266729df1d6f5b"}, - {file = "numpy-1.21.6-cp39-cp39-win32.whl", hash = "sha256:7f5ae4f304257569ef3b948810816bc87c9146e8c446053539947eedeaa32786"}, - {file = "numpy-1.21.6-cp39-cp39-win_amd64.whl", hash = "sha256:e31f0bb5928b793169b87e3d1e070f2342b22d5245c755e2b81caa29756246c3"}, - {file = "numpy-1.21.6-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dd1c8f6bd65d07d3810b90d02eba7997e32abbdf1277a481d698969e921a3be0"}, - {file = "numpy-1.21.6.zip", hash = "sha256:ecb55251139706669fdec2ff073c98ef8e9a84473e51e716211b41aa0f18e656"}, - {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, - {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, - {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, - {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, - {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, - {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, - {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, - {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, - {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, - {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, - {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, -] -oauthlib = [ - {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, - {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, -] -openpyxl = [ - {file = "openpyxl-3.1.2-py2.py3-none-any.whl", hash = "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"}, - {file = "openpyxl-3.1.2.tar.gz", hash = "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184"}, -] -packaging = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, -] -pandas = [ - {file = "pandas-1.3.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:62d5b5ce965bae78f12c1c0df0d387899dd4211ec0bdc52822373f13a3a022b9"}, - {file = "pandas-1.3.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:adfeb11be2d54f275142c8ba9bf67acee771b7186a5745249c7d5a06c670136b"}, - {file = "pandas-1.3.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:60a8c055d58873ad81cae290d974d13dd479b82cbb975c3e1fa2cf1920715296"}, - {file = "pandas-1.3.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd541ab09e1f80a2a1760032d665f6e032d8e44055d602d65eeea6e6e85498cb"}, - {file = "pandas-1.3.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2651d75b9a167cc8cc572cf787ab512d16e316ae00ba81874b560586fa1325e0"}, - {file = "pandas-1.3.5-cp310-cp310-win_amd64.whl", hash = "sha256:aaf183a615ad790801fa3cf2fa450e5b6d23a54684fe386f7e3208f8b9bfbef6"}, - {file = "pandas-1.3.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:344295811e67f8200de2390093aeb3c8309f5648951b684d8db7eee7d1c81fb7"}, - {file = "pandas-1.3.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:552020bf83b7f9033b57cbae65589c01e7ef1544416122da0c79140c93288f56"}, - {file = "pandas-1.3.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cce0c6bbeb266b0e39e35176ee615ce3585233092f685b6a82362523e59e5b4"}, - {file = "pandas-1.3.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d28a3c65463fd0d0ba8bbb7696b23073efee0510783340a44b08f5e96ffce0c"}, - {file = "pandas-1.3.5-cp37-cp37m-win32.whl", hash = "sha256:a62949c626dd0ef7de11de34b44c6475db76995c2064e2d99c6498c3dba7fe58"}, - {file = "pandas-1.3.5-cp37-cp37m-win_amd64.whl", hash = "sha256:8025750767e138320b15ca16d70d5cdc1886e8f9cc56652d89735c016cd8aea6"}, - {file = "pandas-1.3.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fe95bae4e2d579812865db2212bb733144e34d0c6785c0685329e5b60fcb85dd"}, - {file = "pandas-1.3.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f261553a1e9c65b7a310302b9dbac31cf0049a51695c14ebe04e4bfd4a96f02"}, - {file = "pandas-1.3.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b6dbec5f3e6d5dc80dcfee250e0a2a652b3f28663492f7dab9a24416a48ac39"}, - {file = "pandas-1.3.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3bc49af96cd6285030a64779de5b3688633a07eb75c124b0747134a63f4c05f"}, - {file = "pandas-1.3.5-cp38-cp38-win32.whl", hash = "sha256:b6b87b2fb39e6383ca28e2829cddef1d9fc9e27e55ad91ca9c435572cdba51bf"}, - {file = "pandas-1.3.5-cp38-cp38-win_amd64.whl", hash = "sha256:a395692046fd8ce1edb4c6295c35184ae0c2bbe787ecbe384251da609e27edcb"}, - {file = "pandas-1.3.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bd971a3f08b745a75a86c00b97f3007c2ea175951286cdda6abe543e687e5f2f"}, - {file = "pandas-1.3.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37f06b59e5bc05711a518aa10beaec10942188dccb48918bb5ae602ccbc9f1a0"}, - {file = "pandas-1.3.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c21778a688d3712d35710501f8001cdbf96eb70a7c587a3d5613573299fdca6"}, - {file = "pandas-1.3.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3345343206546545bc26a05b4602b6a24385b5ec7c75cb6059599e3d56831da2"}, - {file = "pandas-1.3.5-cp39-cp39-win32.whl", hash = "sha256:c69406a2808ba6cf580c2255bcf260b3f214d2664a3a4197d0e640f573b46fd3"}, - {file = "pandas-1.3.5-cp39-cp39-win_amd64.whl", hash = "sha256:32e1a26d5ade11b547721a72f9bfc4bd113396947606e00d5b4a5b79b3dcb006"}, - {file = "pandas-1.3.5.tar.gz", hash = "sha256:1e4285f5de1012de20ca46b188ccf33521bff61ba5c5ebd78b4fb28e5416a9f1"}, -] -pathspec = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, -] -platformdirs = [ - {file = "platformdirs-3.5.0-py3-none-any.whl", hash = "sha256:47692bc24c1958e8b0f13dd727307cff1db103fca36399f457da8e05f222fdc4"}, - {file = "platformdirs-3.5.0.tar.gz", hash = "sha256:7954a68d0ba23558d753f73437c55f89027cf8f5108c19844d4b82e5af396335"}, -] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] -pyarrow = [ - {file = "pyarrow-12.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:3b97649c8a9a09e1d8dc76513054f1331bd9ece78ee39365e6bf6bc7503c1e94"}, - {file = "pyarrow-12.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bc4ea634dacb03936f50fcf59574a8e727f90c17c24527e488d8ceb52ae284de"}, - {file = "pyarrow-12.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d568acfca3faa565d663e53ee34173be8e23a95f78f2abfdad198010ec8f745"}, - {file = "pyarrow-12.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b50bb9a82dca38a002d7cbd802a16b1af0f8c50ed2ec94a319f5f2afc047ee9"}, - {file = "pyarrow-12.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3d1733b1ea086b3c101427d0e57e2be3eb964686e83c2363862a887bb5c41fa8"}, - {file = "pyarrow-12.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:a7cd32fe77f967fe08228bc100433273020e58dd6caced12627bcc0a7675a513"}, - {file = "pyarrow-12.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:92fb031e6777847f5c9b01eaa5aa0c9033e853ee80117dce895f116d8b0c3ca3"}, - {file = "pyarrow-12.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:280289ebfd4ac3570f6b776515baa01e4dcbf17122c401e4b7170a27c4be63fd"}, - {file = "pyarrow-12.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:272f147d4f8387bec95f17bb58dcfc7bc7278bb93e01cb7b08a0e93a8921e18e"}, - {file = "pyarrow-12.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:0846ace49998825eda4722f8d7f83fa05601c832549c9087ea49d6d5397d8cec"}, - {file = "pyarrow-12.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:993287136369aca60005ee7d64130f9466489c4f7425f5c284315b0a5401ccd9"}, - {file = "pyarrow-12.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7b6a765ee4f88efd7d8348d9a1f804487d60799d0428b6ddf3344eaef37282"}, - {file = "pyarrow-12.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1c4fce253d5bdc8d62f11cfa3da5b0b34b562c04ce84abb8bd7447e63c2b327"}, - {file = "pyarrow-12.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e6be4d85707fc8e7a221c8ab86a40449ce62559ce25c94321df7c8500245888f"}, - {file = "pyarrow-12.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:ea830d9f66bfb82d30b5794642f83dd0e4a718846462d22328981e9eb149cba8"}, - {file = "pyarrow-12.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7b5b9f60d9ef756db59bec8d90e4576b7df57861e6a3d6a8bf99538f68ca15b3"}, - {file = "pyarrow-12.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b99e559d27db36ad3a33868a475f03e3129430fc065accc839ef4daa12c6dab6"}, - {file = "pyarrow-12.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b0810864a593b89877120972d1f7af1d1c9389876dbed92b962ed81492d3ffc"}, - {file = "pyarrow-12.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:23a77d97f4d101ddfe81b9c2ee03a177f0e590a7e68af15eafa06e8f3cf05976"}, - {file = "pyarrow-12.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2cc63e746221cddb9001f7281dee95fd658085dd5b717b076950e1ccc607059c"}, - {file = "pyarrow-12.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d8c26912607e26c2991826bbaf3cf2b9c8c3e17566598c193b492f058b40d3a4"}, - {file = "pyarrow-12.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d8b90efc290e99a81d06015f3a46601c259ecc81ffb6d8ce288c91bd1b868c9"}, - {file = "pyarrow-12.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2466be046b81863be24db370dffd30a2e7894b4f9823fb60ef0a733c31ac6256"}, - {file = "pyarrow-12.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:0e36425b1c1cbf5447718b3f1751bf86c58f2b3ad299f996cd9b1aa040967656"}, - {file = "pyarrow-12.0.0.tar.gz", hash = "sha256:19c812d303610ab5d664b7b1de4051ae23565f9f94d04cbea9e50569746ae1ee"}, -] -pylint = [ - {file = "pylint-2.13.9-py3-none-any.whl", hash = "sha256:705c620d388035bdd9ff8b44c5bcdd235bfb49d276d488dd2c8ff1736aa42526"}, - {file = "pylint-2.13.9.tar.gz", hash = "sha256:095567c96e19e6f57b5b907e67d265ff535e588fe26b12b5ebe1fc5645b2c731"}, -] -pytest = [ - {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, - {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -pytz = [ - {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, - {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, -] -requests = [ - {file = "requests-2.30.0-py3-none-any.whl", hash = "sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294"}, - {file = "requests-2.30.0.tar.gz", hash = "sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4"}, -] -setuptools = [ - {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, - {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -sqlalchemy = [ - {file = "SQLAlchemy-1.4.48-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:4bac3aa3c3d8bc7408097e6fe8bf983caa6e9491c5d2e2488cfcfd8106f13b6a"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:dbcae0e528d755f4522cad5842f0942e54b578d79f21a692c44d91352ea6d64e"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-win32.whl", hash = "sha256:cbbe8b8bffb199b225d2fe3804421b7b43a0d49983f81dc654d0431d2f855543"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-win_amd64.whl", hash = "sha256:627e04a5d54bd50628fc8734d5fc6df2a1aa5962f219c44aad50b00a6cdcf965"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9af1db7a287ef86e0f5cd990b38da6bd9328de739d17e8864f1817710da2d217"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:ce7915eecc9c14a93b73f4e1c9d779ca43e955b43ddf1e21df154184f39748e5"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5381ddd09a99638f429f4cbe1b71b025bed318f6a7b23e11d65f3eed5e181c33"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:87609f6d4e81a941a17e61a4c19fee57f795e96f834c4f0a30cee725fc3f81d9"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb0808ad34167f394fea21bd4587fc62f3bd81bba232a1e7fbdfa17e6cfa7cd7"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-win32.whl", hash = "sha256:d53cd8bc582da5c1c8c86b6acc4ef42e20985c57d0ebc906445989df566c5603"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-win_amd64.whl", hash = "sha256:4355e5915844afdc5cf22ec29fba1010166e35dd94a21305f49020022167556b"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:066c2b0413e8cb980e6d46bf9d35ca83be81c20af688fedaef01450b06e4aa5e"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c99bf13e07140601d111a7c6f1fc1519914dd4e5228315bbda255e08412f61a4"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ee26276f12614d47cc07bc85490a70f559cba965fb178b1c45d46ffa8d73fda"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-win32.whl", hash = "sha256:49c312bcff4728bffc6fb5e5318b8020ed5c8b958a06800f91859fe9633ca20e"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-win_amd64.whl", hash = "sha256:cef2e2abc06eab187a533ec3e1067a71d7bbec69e582401afdf6d8cad4ba3515"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:3509159e050bd6d24189ec7af373359f07aed690db91909c131e5068176c5a5d"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc2ab4d9f6d9218a5caa4121bdcf1125303482a1cdcfcdbd8567be8518969c0"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e1ddbbcef9bcedaa370c03771ebec7e39e3944782bef49e69430383c376a250b"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f82d8efea1ca92b24f51d3aea1a82897ed2409868a0af04247c8c1e4fef5890"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-win32.whl", hash = "sha256:e3e98d4907805b07743b583a99ecc58bf8807ecb6985576d82d5e8ae103b5272"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-win_amd64.whl", hash = "sha256:25887b4f716e085a1c5162f130b852f84e18d2633942c8ca40dfb8519367c14f"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:0817c181271b0ce5df1aa20949f0a9e2426830fed5ecdcc8db449618f12c2730"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe1dd2562313dd9fe1778ed56739ad5d9aae10f9f43d9f4cf81d65b0c85168bb"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:68413aead943883b341b2b77acd7a7fe2377c34d82e64d1840860247cec7ff7c"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbde5642104ac6e95f96e8ad6d18d9382aa20672008cf26068fe36f3004491df"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-win32.whl", hash = "sha256:11c6b1de720f816c22d6ad3bbfa2f026f89c7b78a5c4ffafb220e0183956a92a"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-win_amd64.whl", hash = "sha256:eb5464ee8d4bb6549d368b578e9529d3c43265007193597ddca71c1bae6174e6"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:92e6133cf337c42bfee03ca08c62ba0f2d9695618c8abc14a564f47503157be9"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44d29a3fc6d9c45962476b470a81983dd8add6ad26fdbfae6d463b509d5adcda"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:005e942b451cad5285015481ae4e557ff4154dde327840ba91b9ac379be3b6ce"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c8cfe951ed074ba5e708ed29c45397a95c4143255b0d022c7c8331a75ae61f3"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-win32.whl", hash = "sha256:2b9af65cc58726129d8414fc1a1a650dcdd594ba12e9c97909f1f57d48e393d3"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-win_amd64.whl", hash = "sha256:2b562e9d1e59be7833edf28b0968f156683d57cabd2137d8121806f38a9d58f4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:a1fc046756cf2a37d7277c93278566ddf8be135c6a58397b4c940abf837011f4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d9b55252d2ca42a09bcd10a697fa041e696def9dfab0b78c0aaea1485551a08"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6dab89874e72a9ab5462997846d4c760cdb957958be27b03b49cf0de5e5c327c"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd8b5ee5a3acc4371f820934b36f8109ce604ee73cc668c724abb054cebcb6e"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-win32.whl", hash = "sha256:eee09350fd538e29cfe3a496ec6f148504d2da40dbf52adefb0d2f8e4d38ccc4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-win_amd64.whl", hash = "sha256:7ad2b0f6520ed5038e795cc2852eb5c1f20fa6831d73301ced4aafbe3a10e1f6"}, - {file = "SQLAlchemy-1.4.48.tar.gz", hash = "sha256:b47bc287096d989a0838ce96f7d8e966914a24da877ed41a7531d44b55cdb8df"}, -] -thrift = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -typed-ast = [ - {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, - {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"}, - {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"}, - {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"}, - {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"}, - {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, - {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"}, - {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"}, - {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, - {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, -] -typing-extensions = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, -] -urllib3 = [ - {file = "urllib3-2.0.2-py3-none-any.whl", hash = "sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e"}, - {file = "urllib3-2.0.2.tar.gz", hash = "sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc"}, -] -wrapt = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, -] -zipp = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, -] +lock-version = "2.0" +python-versions = "^3.8.0" +content-hash = "43ea4a4ca7c8403d2b2033b783fe57743e100354986c723ef1f202cde2ac8881" diff --git a/pyproject.toml b/pyproject.toml index e93dcd1b..168fa9fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,45 +1,44 @@ [tool.poetry] name = "databricks-sql-connector" -version = "2.5.2" +version = "4.0.0" description = "Databricks SQL Connector for Python" authors = ["Databricks "] license = "Apache-2.0" readme = "README.md" -packages = [{include = "databricks", from = "src"}] +packages = [{ include = "databricks", from = "src" }] include = ["CHANGELOG.md"] [tool.poetry.dependencies] -python = "^3.7.1" -thrift = "^0.16.0" -pandas = "^1.2.5" -pyarrow = [ - {version = ">=6.0.0", python = ">=3.7,<3.11"}, - {version = ">=10.0.1", python = ">=3.11"} +python = "^3.8.0" +thrift = ">=0.16.0,<0.21.0" +pandas = [ + { version = ">=1.2.5,<2.3.0", python = ">=3.8" } ] lz4 = "^4.0.2" -requests="^2.18.1" -oauthlib="^3.1.0" +requests = "^2.18.1" +oauthlib = "^3.1.0" numpy = [ - {version = ">=1.16.6", python = ">=3.7,<3.11"}, - {version = ">=1.23.4", python = ">=3.11"} + { version = "^1.16.6", python = ">=3.8,<3.11" }, + { version = "^1.23.4", python = ">=3.11" }, ] -sqlalchemy = "^1.3.24" openpyxl = "^3.0.10" -alembic = "^1.0.11" +urllib3 = ">=1.26" +pyarrow = { version = ">=14.0.1", optional=true } + +[tool.poetry.extras] +pyarrow = ["pyarrow"] [tool.poetry.dev-dependencies] pytest = "^7.1.2" -mypy = "^0.950" +mypy = "^1.10.1" pylint = ">=2.12.0" black = "^22.3.0" +pytest-dotenv = "^0.5.2" [tool.poetry.urls] "Homepage" = "https://github.com/databricks/databricks-sql-python" "Bug Tracker" = "https://github.com/databricks/databricks-sql-python/issues" -[tool.poetry.plugins."sqlalchemy.dialects"] -"databricks" = "databricks.sqlalchemy:DatabricksDialect" - [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" @@ -50,3 +49,11 @@ exclude = ['ttypes\.py$', 'TCLIService\.py$'] [tool.black] exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|thrift_api)/' + +[tool.pytest.ini_options] +markers = {"reviewed" = "Test case has been reviewed by Databricks"} +minversion = "6.0" +log_cli = "false" +log_cli_level = "INFO" +testpaths = ["tests"] +env_files = ["test.env"] \ No newline at end of file diff --git a/src/databricks/__init__.py b/src/databricks/__init__.py index 2c691f3a..40d3f2e8 100644 --- a/src/databricks/__init__.py +++ b/src/databricks/__init__.py @@ -1,4 +1,7 @@ -# https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages -# This file should only contain the following line. Otherwise other sub-packages databricks.* namespace -# may not be importable. +# See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages +# +# This file must only contain the following line, or other packages in the databricks.* namespace +# may not be importable. The contents of this file must be byte-for-byte equivalent across all packages. +# If they are not, parallel package installation may lead to clobbered and invalid files. +# Also see https://github.com/databricks/databricks-sdk-py/issues/343. __path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/src/databricks/sql/__init__.py b/src/databricks/sql/__init__.py index fdfb3fb6..fd61ee6c 100644 --- a/src/databricks/sql/__init__.py +++ b/src/databricks/sql/__init__.py @@ -5,7 +5,47 @@ # PEP 249 module globals apilevel = "2.0" threadsafety = 1 # Threads may share the module, but not connections. -paramstyle = "pyformat" # Python extended format codes, e.g. ...WHERE name=%(name)s + +paramstyle = "named" + +import re + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Use this import purely for type annotations, a la https://mypy.readthedocs.io/en/latest/runtime_troubles.html#import-cycles + from .client import Connection + + +class RedactUrlQueryParamsFilter(logging.Filter): + pattern = re.compile(r"(\?|&)([\w-]+)=([^&]+)") + mask = r"\1\2=" + + def __init__(self): + super().__init__() + + def redact(self, string): + return re.sub(self.pattern, self.mask, str(string)) + + def filter(self, record): + record.msg = self.redact(str(record.msg)) + if isinstance(record.args, dict): + for k in record.args.keys(): + record.args[k] = ( + self.redact(record.args[k]) + if isinstance(record.arg[k], str) + else record.args[k] + ) + else: + record.args = tuple( + (self.redact(arg) if isinstance(arg, str) else arg) + for arg in record.args + ) + + return True + + +logging.getLogger("urllib3.connectionpool").addFilter(RedactUrlQueryParamsFilter()) class DBAPITypeObject(object): @@ -28,7 +68,7 @@ def __repr__(self): DATE = DBAPITypeObject("date") ROWID = DBAPITypeObject() -__version__ = "2.5.2" +__version__ = "4.0.0" USER_AGENT_NAME = "PyDatabricksSqlConnector" # These two functions are pyhive legacy @@ -44,7 +84,7 @@ def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) -def connect(server_hostname, http_path, access_token=None, **kwargs): +def connect(server_hostname, http_path, access_token=None, **kwargs) -> "Connection": from .client import Connection return Connection(server_hostname, http_path, access_token, **kwargs) diff --git a/src/databricks/sql/auth/auth.py b/src/databricks/sql/auth/auth.py old mode 100644 new mode 100755 index b56d8f7f..347934ee --- a/src/databricks/sql/auth/auth.py +++ b/src/databricks/sql/auth/auth.py @@ -1,19 +1,18 @@ from enum import Enum -from typing import List +from typing import Optional, List from databricks.sql.auth.authenticators import ( AuthProvider, AccessTokenAuthProvider, - BasicAuthProvider, ExternalAuthProvider, DatabricksOAuthProvider, ) -from databricks.sql.experimental.oauth_persistence import OAuthPersistence class AuthType(Enum): DATABRICKS_OAUTH = "databricks-oauth" - # other supported types (access_token, user/pass) can be inferred + AZURE_OAUTH = "azure-oauth" + # other supported types (access_token) can be inferred # we can add more types as needed later @@ -21,21 +20,17 @@ class ClientContext: def __init__( self, hostname: str, - username: str = None, - password: str = None, - access_token: str = None, - auth_type: str = None, - oauth_scopes: List[str] = None, - oauth_client_id: str = None, - oauth_redirect_port_range: List[int] = None, - use_cert_as_auth: str = None, - tls_client_cert_file: str = None, + access_token: Optional[str] = None, + auth_type: Optional[str] = None, + oauth_scopes: Optional[List[str]] = None, + oauth_client_id: Optional[str] = None, + oauth_redirect_port_range: Optional[List[int]] = None, + use_cert_as_auth: Optional[str] = None, + tls_client_cert_file: Optional[str] = None, oauth_persistence=None, credentials_provider=None, ): self.hostname = hostname - self.username = username - self.password = password self.access_token = access_token self.auth_type = auth_type self.oauth_scopes = oauth_scopes @@ -50,7 +45,7 @@ def __init__( def get_auth_provider(cfg: ClientContext): if cfg.credentials_provider: return ExternalAuthProvider(cfg.credentials_provider) - if cfg.auth_type == AuthType.DATABRICKS_OAUTH.value: + if cfg.auth_type in [AuthType.DATABRICKS_OAUTH.value, AuthType.AZURE_OAUTH.value]: assert cfg.oauth_redirect_port_range is not None assert cfg.oauth_client_id is not None assert cfg.oauth_scopes is not None @@ -61,21 +56,35 @@ def get_auth_provider(cfg: ClientContext): cfg.oauth_redirect_port_range, cfg.oauth_client_id, cfg.oauth_scopes, + cfg.auth_type, ) elif cfg.access_token is not None: return AccessTokenAuthProvider(cfg.access_token) - elif cfg.username is not None and cfg.password is not None: - return BasicAuthProvider(cfg.username, cfg.password) elif cfg.use_cert_as_auth and cfg.tls_client_cert_file: # no op authenticator. authentication is performed using ssl certificate outside of headers return AuthProvider() else: - raise RuntimeError("No valid authentication settings!") + if ( + cfg.oauth_redirect_port_range is not None + and cfg.oauth_client_id is not None + and cfg.oauth_scopes is not None + ): + return DatabricksOAuthProvider( + cfg.hostname, + cfg.oauth_persistence, + cfg.oauth_redirect_port_range, + cfg.oauth_client_id, + cfg.oauth_scopes, + ) + else: + raise RuntimeError("No valid authentication settings!") PYSQL_OAUTH_SCOPES = ["sql", "offline_access"] PYSQL_OAUTH_CLIENT_ID = "databricks-sql-python" +PYSQL_OAUTH_AZURE_CLIENT_ID = "96eecda7-19ea-49cc-abb5-240097d554f5" PYSQL_OAUTH_REDIRECT_PORT_RANGE = list(range(8020, 8025)) +PYSQL_OAUTH_AZURE_REDIRECT_PORT_RANGE = [8030] def normalize_host_name(hostname: str): @@ -84,20 +93,36 @@ def normalize_host_name(hostname: str): return f"{maybe_scheme}{hostname}{maybe_trailing_slash}" +def get_client_id_and_redirect_port(use_azure_auth: bool): + return ( + (PYSQL_OAUTH_CLIENT_ID, PYSQL_OAUTH_REDIRECT_PORT_RANGE) + if not use_azure_auth + else (PYSQL_OAUTH_AZURE_CLIENT_ID, PYSQL_OAUTH_AZURE_REDIRECT_PORT_RANGE) + ) + + def get_python_sql_connector_auth_provider(hostname: str, **kwargs): + auth_type = kwargs.get("auth_type") + (client_id, redirect_port_range) = get_client_id_and_redirect_port( + auth_type == AuthType.AZURE_OAUTH.value + ) + if kwargs.get("username") or kwargs.get("password"): + raise ValueError( + "Username/password authentication is no longer supported. " + "Please use OAuth or access token instead." + ) + cfg = ClientContext( hostname=normalize_host_name(hostname), - auth_type=kwargs.get("auth_type"), + auth_type=auth_type, access_token=kwargs.get("access_token"), - username=kwargs.get("_username"), - password=kwargs.get("_password"), use_cert_as_auth=kwargs.get("_use_cert_as_auth"), tls_client_cert_file=kwargs.get("_tls_client_cert_file"), oauth_scopes=PYSQL_OAUTH_SCOPES, - oauth_client_id=kwargs.get("oauth_client_id") or PYSQL_OAUTH_CLIENT_ID, + oauth_client_id=kwargs.get("oauth_client_id") or client_id, oauth_redirect_port_range=[kwargs["oauth_redirect_port"]] if kwargs.get("oauth_client_id") and kwargs.get("oauth_redirect_port") - else PYSQL_OAUTH_REDIRECT_PORT_RANGE, + else redirect_port_range, oauth_persistence=kwargs.get("experimental_oauth_persistence"), credentials_provider=kwargs.get("credentials_provider"), ) diff --git a/src/databricks/sql/auth/authenticators.py b/src/databricks/sql/auth/authenticators.py index eb368e1e..64eb91bb 100644 --- a/src/databricks/sql/auth/authenticators.py +++ b/src/databricks/sql/auth/authenticators.py @@ -4,6 +4,7 @@ from typing import Callable, Dict, List from databricks.sql.auth.oauth import OAuthManager +from databricks.sql.auth.endpoint import get_oauth_endpoints, infer_cloud_from_host # Private API: this is an evolving interface and it will change in the future. # Please must not depend on it in your applications. @@ -17,6 +18,7 @@ def add_headers(self, request_headers: Dict[str, str]): HeaderFactory = Callable[[], Dict[str, str]] + # In order to keep compatibility with SDK class CredentialsProvider(abc.ABC): """CredentialsProvider is the protocol (call-side interface) @@ -41,21 +43,6 @@ def add_headers(self, request_headers: Dict[str, str]): request_headers["Authorization"] = self.__authorization_header_value -# Private API: this is an evolving interface and it will change in the future. -# Please must not depend on it in your applications. -class BasicAuthProvider(AuthProvider): - def __init__(self, username: str, password: str): - auth_credentials = f"{username}:{password}".encode("UTF-8") - auth_credentials_base64 = base64.standard_b64encode(auth_credentials).decode( - "UTF-8" - ) - - self.__authorization_header_value = f"Basic {auth_credentials_base64}" - - def add_headers(self, request_headers: Dict[str, str]): - request_headers["Authorization"] = self.__authorization_header_value - - # Private API: this is an evolving interface and it will change in the future. # Please must not depend on it in your applications. class DatabricksOAuthProvider(AuthProvider): @@ -68,13 +55,25 @@ def __init__( redirect_port_range: List[int], client_id: str, scopes: List[str], + auth_type: str = "databricks-oauth", ): try: + idp_endpoint = get_oauth_endpoints(hostname, auth_type == "azure-oauth") + if not idp_endpoint: + raise NotImplementedError( + f"OAuth is not supported for host ${hostname}" + ) + + # Convert to the corresponding scopes in the corresponding IdP + cloud_scopes = idp_endpoint.get_scopes_mapping(scopes) + self.oauth_manager = OAuthManager( - port_range=redirect_port_range, client_id=client_id + port_range=redirect_port_range, + client_id=client_id, + idp_endpoint=idp_endpoint, ) self._hostname = hostname - self._scopes_as_str = DatabricksOAuthProvider.SCOPE_DELIM.join(scopes) + self._scopes_as_str = DatabricksOAuthProvider.SCOPE_DELIM.join(cloud_scopes) self._oauth_persistence = oauth_persistence self._client_id = client_id self._access_token = None diff --git a/src/databricks/sql/auth/endpoint.py b/src/databricks/sql/auth/endpoint.py new file mode 100644 index 00000000..5cb26ae3 --- /dev/null +++ b/src/databricks/sql/auth/endpoint.py @@ -0,0 +1,140 @@ +# +# It implements all the cloud specific OAuth configuration/metadata +# +# Azure: It uses Databricks internal IdP or Azure AD +# AWS: It uses Databricks internal IdP +# GCP: It uses Databricks internal IdP +# +from abc import ABC, abstractmethod +from enum import Enum +from typing import Optional, List +import os + +OIDC_REDIRECTOR_PATH = "oidc" + + +class OAuthScope: + OFFLINE_ACCESS = "offline_access" + SQL = "sql" + + +class CloudType(Enum): + AWS = "aws" + AZURE = "azure" + GCP = "gcp" + + +DATABRICKS_AWS_DOMAINS = [ + ".cloud.databricks.com", + ".cloud.databricks.us", + ".dev.databricks.com", +] + +DATABRICKS_AZURE_DOMAINS = [ + ".azuredatabricks.net", + ".databricks.azure.cn", + ".databricks.azure.us", +] +DATABRICKS_GCP_DOMAINS = [".gcp.databricks.com"] + +# Domain supported by Databricks InHouse OAuth +DATABRICKS_OAUTH_AZURE_DOMAINS = [".azuredatabricks.net"] + + +# Infer cloud type from Databricks SQL instance hostname +def infer_cloud_from_host(hostname: str) -> Optional[CloudType]: + # normalize + host = hostname.lower().replace("https://", "").split("/")[0] + + if any(e for e in DATABRICKS_AZURE_DOMAINS if host.endswith(e)): + return CloudType.AZURE + elif any(e for e in DATABRICKS_AWS_DOMAINS if host.endswith(e)): + return CloudType.AWS + elif any(e for e in DATABRICKS_GCP_DOMAINS if host.endswith(e)): + return CloudType.GCP + else: + return None + + +def is_supported_databricks_oauth_host(hostname: str) -> bool: + host = hostname.lower().replace("https://", "").split("/")[0] + domains = ( + DATABRICKS_AWS_DOMAINS + DATABRICKS_GCP_DOMAINS + DATABRICKS_OAUTH_AZURE_DOMAINS + ) + return any(e for e in domains if host.endswith(e)) + + +def get_databricks_oidc_url(hostname: str): + maybe_scheme = "https://" if not hostname.startswith("https://") else "" + maybe_trailing_slash = "/" if not hostname.endswith("/") else "" + return f"{maybe_scheme}{hostname}{maybe_trailing_slash}{OIDC_REDIRECTOR_PATH}" + + +class OAuthEndpointCollection(ABC): + @abstractmethod + def get_scopes_mapping(self, scopes: List[str]) -> List[str]: + raise NotImplementedError() + + # Endpoint for oauth2 authorization e.g https://idp.example.com/oauth2/v2.0/authorize + @abstractmethod + def get_authorization_url(self, hostname: str) -> str: + raise NotImplementedError() + + # Endpoint for well-known openid configuration e.g https://idp.example.com/oauth2/.well-known/openid-configuration + @abstractmethod + def get_openid_config_url(self, hostname: str) -> str: + raise NotImplementedError() + + +class AzureOAuthEndpointCollection(OAuthEndpointCollection): + DATATRICKS_AZURE_APP = "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d" + + def get_scopes_mapping(self, scopes: List[str]) -> List[str]: + # There is no corresponding scopes in Azure, instead, access control will be delegated to Databricks + tenant_id = os.getenv( + "DATABRICKS_AZURE_TENANT_ID", + AzureOAuthEndpointCollection.DATATRICKS_AZURE_APP, + ) + azure_scope = f"{tenant_id}/user_impersonation" + mapped_scopes = [azure_scope] + if OAuthScope.OFFLINE_ACCESS in scopes: + mapped_scopes.append(OAuthScope.OFFLINE_ACCESS) + return mapped_scopes + + def get_authorization_url(self, hostname: str): + # We need get account specific url, which can be redirected by databricks unified oidc endpoint + return f"{get_databricks_oidc_url(hostname)}/oauth2/v2.0/authorize" + + def get_openid_config_url(self, hostname: str): + return "https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration" + + +class InHouseOAuthEndpointCollection(OAuthEndpointCollection): + def get_scopes_mapping(self, scopes: List[str]) -> List[str]: + # No scope mapping in AWS + return scopes.copy() + + def get_authorization_url(self, hostname: str): + idp_url = get_databricks_oidc_url(hostname) + return f"{idp_url}/oauth2/v2.0/authorize" + + def get_openid_config_url(self, hostname: str): + idp_url = get_databricks_oidc_url(hostname) + return f"{idp_url}/.well-known/oauth-authorization-server" + + +def get_oauth_endpoints( + hostname: str, use_azure_auth: bool +) -> Optional[OAuthEndpointCollection]: + cloud = infer_cloud_from_host(hostname) + + if cloud in [CloudType.AWS, CloudType.GCP]: + return InHouseOAuthEndpointCollection() + elif cloud == CloudType.AZURE: + return ( + InHouseOAuthEndpointCollection() + if is_supported_databricks_oauth_host(hostname) and not use_azure_auth + else AzureOAuthEndpointCollection() + ) + else: + return None diff --git a/src/databricks/sql/auth/oauth.py b/src/databricks/sql/auth/oauth.py index 0f49aa88..806df08f 100644 --- a/src/databricks/sql/auth/oauth.py +++ b/src/databricks/sql/auth/oauth.py @@ -14,17 +14,38 @@ from requests.exceptions import RequestException from databricks.sql.auth.oauth_http_handler import OAuthHttpSingleRequestHandler +from databricks.sql.auth.endpoint import OAuthEndpointCollection logger = logging.getLogger(__name__) -class OAuthManager: - OIDC_REDIRECTOR_PATH = "oidc" +class IgnoreNetrcAuth(requests.auth.AuthBase): + """This auth method is a no-op. + + We use it to force requestslib to not use .netrc to write auth headers + when making .post() requests to the oauth token endpoints, since these + don't require authentication. + + In cases where .netrc is outdated or corrupt, these requests will fail. + + See issue #121 + """ - def __init__(self, port_range: List[int], client_id: str): + def __call__(self, r): + return r + + +class OAuthManager: + def __init__( + self, + port_range: List[int], + client_id: str, + idp_endpoint: OAuthEndpointCollection, + ): self.port_range = port_range self.client_id = client_id self.redirect_port = None + self.idp_endpoint = idp_endpoint @staticmethod def __token_urlsafe(nbytes=32): @@ -34,14 +55,14 @@ def __token_urlsafe(nbytes=32): def __get_redirect_url(redirect_port: int): return f"http://localhost:{redirect_port}" - @staticmethod - def __fetch_well_known_config(idp_url: str): - known_config_url = f"{idp_url}/.well-known/oauth-authorization-server" + def __fetch_well_known_config(self, hostname: str): + known_config_url = self.idp_endpoint.get_openid_config_url(hostname) + try: - response = requests.get(url=known_config_url) + response = requests.get(url=known_config_url, auth=IgnoreNetrcAuth()) except RequestException as e: logger.error( - f"Unable to fetch OAuth configuration from {idp_url}.\n" + f"Unable to fetch OAuth configuration from {known_config_url}.\n" "Verify it is a valid workspace URL and that OAuth is " "enabled on this account." ) @@ -50,7 +71,7 @@ def __fetch_well_known_config(idp_url: str): if response.status_code != 200: msg = ( f"Received status {response.status_code} OAuth configuration from " - f"{idp_url}.\n Verify it is a valid workspace URL and " + f"{known_config_url}.\n Verify it is a valid workspace URL and " "that OAuth is enabled on this account." ) logger.error(msg) @@ -59,18 +80,12 @@ def __fetch_well_known_config(idp_url: str): return response.json() except requests.exceptions.JSONDecodeError as e: logger.error( - f"Unable to decode OAuth configuration from {idp_url}.\n" + f"Unable to decode OAuth configuration from {known_config_url}.\n" "Verify it is a valid workspace URL and that OAuth is " "enabled on this account." ) raise e - @staticmethod - def __get_idp_url(host: str): - maybe_scheme = "https://" if not host.startswith("https://") else "" - maybe_trailing_slash = "/" if not host.endswith("/") else "" - return f"{maybe_scheme}{host}{maybe_trailing_slash}{OAuthManager.OIDC_REDIRECTOR_PATH}" - @staticmethod def __get_challenge(): verifier_string = OAuthManager.__token_urlsafe(32) @@ -110,7 +125,7 @@ def __get_authorization_code(self, client, auth_url, scope, state, challenge): logger.info(f"Port {port} is in use") last_error = e except Exception as e: - logger.error("unexpected error", e) + logger.error("unexpected error: %s", e) if self.redirect_port is None: logger.error( f"Tried all the ports {self.port_range} for oauth redirect, but can't find free port" @@ -150,12 +165,13 @@ def __send_token_request(token_request_url, data): "Accept": "application/json", "Content-Type": "application/x-www-form-urlencoded", } - response = requests.post(url=token_request_url, data=data, headers=headers) + response = requests.post( + url=token_request_url, data=data, headers=headers, auth=IgnoreNetrcAuth() + ) return response.json() def __send_refresh_token_request(self, hostname, refresh_token): - idp_url = OAuthManager.__get_idp_url(hostname) - oauth_config = OAuthManager.__fetch_well_known_config(idp_url) + oauth_config = self.__fetch_well_known_config(hostname) token_request_url = oauth_config["token_endpoint"] client = oauthlib.oauth2.WebApplicationClient(self.client_id) token_request_body = client.prepare_refresh_body( @@ -215,14 +231,15 @@ def check_and_refresh_access_token( return fresh_access_token, fresh_refresh_token, True def get_tokens(self, hostname: str, scope=None): - idp_url = self.__get_idp_url(hostname) - oauth_config = self.__fetch_well_known_config(idp_url) + oauth_config = self.__fetch_well_known_config(hostname) # We are going to override oauth_config["authorization_endpoint"] use the # /oidc redirector on the hostname, which may inject additional parameters. - auth_url = f"{hostname}oidc/v1/authorize" + auth_url = self.idp_endpoint.get_authorization_url(hostname) + state = OAuthManager.__token_urlsafe(16) (verifier, challenge) = OAuthManager.__get_challenge() client = oauthlib.oauth2.WebApplicationClient(self.client_id) + try: auth_response = self.__get_authorization_code( client, auth_url, scope, state, challenge diff --git a/src/databricks/sql/auth/retry.py b/src/databricks/sql/auth/retry.py new file mode 100755 index 00000000..0243d0aa --- /dev/null +++ b/src/databricks/sql/auth/retry.py @@ -0,0 +1,442 @@ +import logging +import random +import time +import typing +from enum import Enum +from typing import List, Optional, Tuple, Union + +# We only use this import for type hinting +try: + # If urllib3~=2.0 is installed + from urllib3 import BaseHTTPResponse +except ImportError: + # If urllib3~=1.0 is installed + from urllib3 import HTTPResponse as BaseHTTPResponse +from urllib3 import Retry +from urllib3.util.retry import RequestHistory + +from databricks.sql.exc import ( + CursorAlreadyClosedError, + MaxRetryDurationError, + NonRecoverableNetworkError, + OperationalError, + SessionAlreadyClosedError, + UnsafeToRetryError, +) + +logger = logging.getLogger(__name__) + + +class CommandType(Enum): + EXECUTE_STATEMENT = "ExecuteStatement" + CLOSE_SESSION = "CloseSession" + CLOSE_OPERATION = "CloseOperation" + GET_OPERATION_STATUS = "GetOperationStatus" + OTHER = "Other" + + @classmethod + def get(cls, value: str): + value_name_map = {i.value: i.name for i in cls} + valid_command = value_name_map.get(value, False) + if valid_command: + return getattr(cls, str(valid_command)) + else: + return cls.OTHER + + +class DatabricksRetryPolicy(Retry): + """ + Implements our v3 retry policy by extending urllib3's robust default retry behaviour. + + Retry logic varies based on the overall wall-clock request time and Thrift CommandType + being issued. ThriftBackend starts a timer and sets the current CommandType prior to + initiating a network request. See `self.should_retry()` for details about what we do + and do not retry. + + :param delay_min: + Float of seconds for the minimum delay between retries. This is an alias for urllib3's + `backoff_factor`. + + :param delay_max: + Float of seconds for the maximum delay between retries. + + :param stop_after_attempts_count: + Integer maximum number of attempts that will be retried. This is an alias for urllib3's + `total`. + + :param stop_after_attempts_duration: + Float of maximum number of seconds within which a request may be retried starting from + the beginning of the first request. + + :param delay_default: + Float of seconds the connector will wait between sucessive GetOperationStatus + requests. This parameter is not used to retry failed network requests. We include + it in this class to keep all retry behaviour encapsulated in this file. + + :param force_dangerous_codes: + List of integer HTTP status codes that the connector will retry, even for dangerous + commands like ExecuteStatement. This is passed to urllib3 by extending its status_forcelist + + :param urllib3_kwargs: + Dictionary of arguments that are passed to Retry.__init__. Any setting of Retry() that + Databricks does not override or extend may be modified here. + """ + + def __init__( + self, + delay_min: float, + delay_max: float, + stop_after_attempts_count: int, + stop_after_attempts_duration: float, + delay_default: float, + force_dangerous_codes: List[int], + urllib3_kwargs: dict = {}, + ): + # These values do not change from one command to the next + self.delay_max = delay_max + self.delay_min = delay_min + self.stop_after_attempts_count = stop_after_attempts_count + self.stop_after_attempts_duration = stop_after_attempts_duration + self._delay_default = delay_default + self.force_dangerous_codes = force_dangerous_codes + + # the urllib3 kwargs are a mix of configuration (some of which we override) + # and counters like `total` or `connect` which may change between successive retries + # we only care about urllib3 kwargs that we alias, override, or add to in some way + + # the length of _history increases as retries are performed + _history: Optional[Tuple[RequestHistory, ...]] = urllib3_kwargs.get("history") + + if not _history: + # no attempts were made so we can retry the current command as many times as specified + # by the user + _attempts_remaining = self.stop_after_attempts_count + else: + # at least one of our attempts has been consumed, and urllib3 will have set a total + # `total` is a counter that begins equal to self.stop_after_attempts_count and is + # decremented after each unsuccessful request. When `total` is zero, urllib3 raises a + # MaxRetryError + _total: int = urllib3_kwargs.pop("total") + _attempts_remaining = _total + + _urllib_kwargs_we_care_about = dict( + total=_attempts_remaining, + respect_retry_after_header=True, + backoff_factor=self.delay_min, + allowed_methods=["POST"], + status_forcelist=[429, 503, *self.force_dangerous_codes], + ) + + urllib3_kwargs.update(**_urllib_kwargs_we_care_about) + + super().__init__( + **urllib3_kwargs, + ) + + @classmethod + def __private_init__( + cls, retry_start_time: float, command_type: Optional[CommandType], **init_kwargs + ): + """ + Returns a new instance of DatabricksRetryPolicy with the _retry_start_time and _command_type + properties already set. This method should only be called by DatabricksRetryPolicy itself between + successive Retry attempts. + + :param retry_start_time: + Float unix timestamp. Used to monitor the overall request duration across successive + retries. Never set this value directly. Use self.start_retry_timer() instead. Users + never set this value. It is set by ThriftBackend immediately before issuing a network + request. + + :param command_type: + CommandType of the current request being retried. Used to modify retry behaviour based + on the type of Thrift command being issued. See self.should_retry() for details. Users + never set this value directly. It is set by ThriftBackend immediately before issuing + a network request. + + :param init_kwargs: + A dictionary of parameters that will be passed to __init__ in the new object + """ + + new_object = cls(**init_kwargs) + new_object._retry_start_time = retry_start_time + new_object.command_type = command_type + return new_object + + def new( + self, **urllib3_incremented_counters: typing.Any + ) -> "DatabricksRetryPolicy": + """This method is responsible for passing the entire Retry state to its next iteration. + + urllib3 calls Retry.new() between successive requests as part of its `.increment()` method + as shown below: + + ```python + new_retry = self.new( + total=total, + connect=connect, + read=read, + redirect=redirect, + status=status_count, + other=other, + history=history, + ) + ``` + + The arguments it passes to `.new()` (total, connect, read, etc.) are those modified by `.increment()`. + + Since self.__init__ has a different signature than Retry.__init__ , we implement our own `self.new()` + to pipe our Databricks-specific state while preserving the super-class's behaviour. + + """ + + # These arguments will match the function signature for self.__init__ + databricks_init_params = dict( + delay_min=self.delay_min, + delay_max=self.delay_max, + stop_after_attempts_count=self.stop_after_attempts_count, + stop_after_attempts_duration=self.stop_after_attempts_duration, + delay_default=self.delay_default, + force_dangerous_codes=self.force_dangerous_codes, + urllib3_kwargs={}, + ) + + # Gather urllib3's current retry state _before_ increment was called + # These arguments match the function signature for Retry.__init__ + # Note: if we update urllib3 we may need to add/remove arguments from this dict + urllib3_init_params = dict( + total=self.total, + connect=self.connect, + read=self.read, + redirect=self.redirect, + status=self.status, + other=self.other, + allowed_methods=self.allowed_methods, + status_forcelist=self.status_forcelist, + backoff_factor=self.backoff_factor, + raise_on_redirect=self.raise_on_redirect, + raise_on_status=self.raise_on_status, + history=self.history, + remove_headers_on_redirect=self.remove_headers_on_redirect, + respect_retry_after_header=self.respect_retry_after_header, + ) + + # Update urllib3's current state to reflect the incremented counters + urllib3_init_params.update(**urllib3_incremented_counters) + + # Include urllib3's current state in our __init__ params + databricks_init_params["urllib3_kwargs"].update(**urllib3_init_params) # type: ignore[attr-defined] + + return type(self).__private_init__( + retry_start_time=self._retry_start_time, + command_type=self.command_type, + **databricks_init_params, + ) + + @property + def command_type(self) -> Optional[CommandType]: + return self._command_type + + @command_type.setter + def command_type(self, value: CommandType) -> None: + self._command_type = value + + @property + def delay_default(self) -> float: + """Time in seconds the connector will wait between requests polling a GetOperationStatus Request + + This property is never read by urllib3 for the purpose of retries. It's stored in this class + to keep all retry logic in one place. + + This property is only set by __init__ and cannot be modified afterward. + """ + return self._delay_default + + def start_retry_timer(self) -> None: + """Timer is used to monitor the overall time across successive requests + + Should only be called by ThriftBackend before sending a Thrift command""" + self._retry_start_time = time.time() + + def check_timer_duration(self) -> float: + """Return time in seconds since the timer was started""" + + if self._retry_start_time is None: + raise OperationalError( + "Cannot check retry timer. Timer was not started for this request." + ) + else: + return time.time() - self._retry_start_time + + def check_proposed_wait(self, proposed_wait: Union[int, float]) -> None: + """Raise an exception if the proposed wait would exceed the configured max_attempts_duration""" + + proposed_overall_time = self.check_timer_duration() + proposed_wait + if proposed_overall_time > self.stop_after_attempts_duration: + raise MaxRetryDurationError( + f"Retry request would exceed Retry policy max retry duration of {self.stop_after_attempts_duration} seconds" + ) + + def sleep_for_retry(self, response: BaseHTTPResponse) -> bool: + """Sleeps for the duration specified in the response Retry-After header, if present + + A MaxRetryDurationError will be raised if doing so would exceed self.max_attempts_duration + + This method is only called by urllib3 internals. + """ + retry_after = self.get_retry_after(response) + if retry_after: + proposed_wait = retry_after + else: + proposed_wait = self.get_backoff_time() + + proposed_wait = min(proposed_wait, self.delay_max) + self.check_proposed_wait(proposed_wait) + time.sleep(proposed_wait) + return True + + def get_backoff_time(self) -> float: + """ + This method implements the exponential backoff algorithm to calculate the delay between retries. + + Never returns a value larger than self.delay_max + A MaxRetryDurationError will be raised if the calculated backoff would exceed self.max_attempts_duration + + :return: + """ + + current_attempt = self.stop_after_attempts_count - int(self.total or 0) + proposed_backoff = (2**current_attempt) * self.delay_min + if self.backoff_jitter != 0.0: + proposed_backoff += random.random() * self.backoff_jitter + + proposed_backoff = min(proposed_backoff, self.delay_max) + self.check_proposed_wait(proposed_backoff) + + return proposed_backoff + + def should_retry(self, method: str, status_code: int) -> Tuple[bool, str]: + """This method encapsulates the connector's approach to retries. + + We always retry a request unless one of these conditions is met: + + 1. The request received a 200 (Success) status code + Because the request succeeded . + 2. The request received a 501 (Not Implemented) status code + Because this request can never succeed. + 3. The request received a 404 (Not Found) code and the request CommandType + was GetOperationStatus, CloseSession or CloseOperation. This code indicates + that the command, session or cursor was already closed. Further retries will + always return the same code. + 4. The request CommandType was ExecuteStatement and the HTTP code does not + appear in the default status_forcelist or force_dangerous_codes list. By + default, this means ExecuteStatement is only retried for codes 429 and 503. + This limit prevents automatically retrying non-idempotent commands that could + be destructive. + 5. The request received a 401 response, because this can never succeed. + 6. The request received a 403 response, because this can never succeed. + + + Q: What about OSErrors and Redirects? + A: urllib3 automatically retries in both scenarios + + Returns True if the request should be retried. Returns False or raises an exception + if a retry would violate the configured policy. + """ + + # Request succeeded. Don't retry. + if status_code == 200: + return False, "200 codes are not retried" + + if status_code == 401: + raise NonRecoverableNetworkError( + "Received 401 - UNAUTHORIZED. Confirm your authentication credentials." + ) + + if status_code == 403: + raise NonRecoverableNetworkError( + "Received 403 - FORBIDDEN. Confirm your authentication credentials." + ) + + # Request failed and server said NotImplemented. This isn't recoverable. Don't retry. + if status_code == 501: + raise NonRecoverableNetworkError("Received code 501 from server.") + + # Request failed and this method is not retryable. We only retry POST requests. + if not self._is_method_retryable(method): + return False, "Only POST requests are retried" + + # Request failed with 404 and was a GetOperationStatus. This is not recoverable. Don't retry. + if status_code == 404 and self.command_type == CommandType.GET_OPERATION_STATUS: + return ( + False, + "GetOperationStatus received 404 code from Databricks. Operation was canceled.", + ) + + # Request failed with 404 because CloseSession returns 404 if you repeat the request. + if ( + status_code == 404 + and self.command_type == CommandType.CLOSE_SESSION + and len(self.history) > 0 + ): + raise SessionAlreadyClosedError( + "CloseSession received 404 code from Databricks. Session is already closed." + ) + + # Request failed with 404 because CloseOperation returns 404 if you repeat the request. + if ( + status_code == 404 + and self.command_type == CommandType.CLOSE_OPERATION + and len(self.history) > 0 + ): + raise CursorAlreadyClosedError( + "CloseOperation received 404 code from Databricks. Cursor is already closed." + ) + + # Request failed, was an ExecuteStatement and the command may have reached the server + if ( + self.command_type == CommandType.EXECUTE_STATEMENT + and status_code not in self.status_forcelist + and status_code not in self.force_dangerous_codes + ): + raise UnsafeToRetryError( + "ExecuteStatement command can only be retried for codes 429 and 503" + ) + + # Request failed with a dangerous code, was an ExecuteStatement, but user forced retries for this + # dangerous code. Note that these lines _are not required_ to make these requests retry. They would + # retry automatically. This code is included only so that we can log the exact reason for the retry. + # This gives users signal that their _retry_dangerous_codes setting actually did something. + if ( + self.command_type == CommandType.EXECUTE_STATEMENT + and status_code in self.force_dangerous_codes + ): + return ( + True, + f"Request failed with dangerous code {status_code} that is one of the configured _retry_dangerous_codes.", + ) + + # None of the above conditions applied. Eagerly retry. + logger.debug( + f"This request should be retried: {self.command_type and self.command_type.value}" + ) + return ( + True, + "Failed requests are retried by default per configured DatabricksRetryPolicy", + ) + + def is_retry( + self, method: str, status_code: int, has_retry_after: bool = False + ) -> bool: + """ + Called by urllib3 when determining whether or not to retry + + Logs a debug message if the request will be retried + """ + + should_retry, msg = self.should_retry(method, status_code) + + if should_retry: + logger.debug(msg) + + return should_retry diff --git a/src/databricks/sql/auth/thrift_http_client.py b/src/databricks/sql/auth/thrift_http_client.py index a924ea63..6273ab28 100644 --- a/src/databricks/sql/auth/thrift_http_client.py +++ b/src/databricks/sql/auth/thrift_http_client.py @@ -1,10 +1,20 @@ +import base64 import logging -from typing import Dict - +import urllib.parse +from typing import Dict, Union, Optional +import six import thrift -import urllib.parse, six, base64 +import ssl +import warnings +from http.client import HTTPResponse +from io import BytesIO + +from urllib3 import HTTPConnectionPool, HTTPSConnectionPool, ProxyManager +from urllib3.util import make_headers +from databricks.sql.auth.retry import CommandType, DatabricksRetryPolicy +from databricks.sql.types import SSLOptions logger = logging.getLogger(__name__) @@ -16,34 +26,193 @@ def __init__( uri_or_host, port=None, path=None, - cafile=None, - cert_file=None, - key_file=None, - ssl_context=None, + ssl_options: Optional[SSLOptions] = None, + max_connections: int = 1, + retry_policy: Union[DatabricksRetryPolicy, int] = 0, ): - super().__init__( - uri_or_host, port, path, cafile, cert_file, key_file, ssl_context - ) + self._ssl_options = ssl_options + + if port is not None: + warnings.warn( + "Please use the THttpClient('http{s}://host:port/path') constructor", + DeprecationWarning, + stacklevel=2, + ) + self.host = uri_or_host + self.port = port + assert path + self.path = path + self.scheme = "http" + else: + parsed = urllib.parse.urlsplit(uri_or_host) + self.scheme = parsed.scheme + assert self.scheme in ("http", "https") + if self.scheme == "https": + if self._ssl_options is not None: + # TODO: Not sure if those options are used anywhere - need to double-check + self.certfile = self._ssl_options.tls_client_cert_file + self.keyfile = self._ssl_options.tls_client_cert_key_file + self.context = self._ssl_options.create_ssl_context() + self.port = parsed.port + self.host = parsed.hostname + self.path = parsed.path + if parsed.query: + self.path += "?%s" % parsed.query + try: + proxy = urllib.request.getproxies()[self.scheme] + except KeyError: + proxy = None + else: + if urllib.request.proxy_bypass(self.host): + proxy = None + if proxy: + parsed = urllib.parse.urlparse(proxy) + + # realhost and realport are the host and port of the actual request + self.realhost = self.host + self.realport = self.port + + # this is passed to ProxyManager + self.proxy_uri: str = proxy + self.host = parsed.hostname + self.port = parsed.port + self.proxy_auth = self.basic_proxy_auth_headers(parsed) + else: + self.realhost = self.realport = self.proxy_auth = None + + self.max_connections = max_connections + + # If retry_policy == 0 then urllib3 will not retry automatically + # this falls back to the pre-v3 behaviour where thrift_backend.py handles retry logic + self.retry_policy = retry_policy + + self.__wbuf = BytesIO() + self.__resp: Union[None, HTTPResponse] = None + self.__timeout = None + self.__custom_headers = None + self.__auth_provider = auth_provider def setCustomHeaders(self, headers: Dict[str, str]): self._headers = headers super().setCustomHeaders(headers) + def startRetryTimer(self): + """Notify DatabricksRetryPolicy of the request start time + + This is used to enforce the retry_stop_after_attempts_duration + """ + self.retry_policy and self.retry_policy.start_retry_timer() + + def open(self): + + # self.__pool replaces the self.__http used by the original THttpClient + _pool_kwargs = {"maxsize": self.max_connections} + + if self.scheme == "http": + pool_class = HTTPConnectionPool + elif self.scheme == "https": + pool_class = HTTPSConnectionPool + _pool_kwargs.update( + { + "cert_reqs": ssl.CERT_REQUIRED + if self._ssl_options.tls_verify + else ssl.CERT_NONE, + "ca_certs": self._ssl_options.tls_trusted_ca_file, + "cert_file": self._ssl_options.tls_client_cert_file, + "key_file": self._ssl_options.tls_client_cert_key_file, + "key_password": self._ssl_options.tls_client_cert_key_password, + } + ) + + if self.using_proxy(): + proxy_manager = ProxyManager( + self.proxy_uri, + num_pools=1, + proxy_headers=self.proxy_auth, + ) + self.__pool = proxy_manager.connection_from_host( + host=self.realhost, + port=self.realport, + scheme=self.scheme, + pool_kwargs=_pool_kwargs, + ) + else: + self.__pool = pool_class(self.host, self.port, **_pool_kwargs) + + def close(self): + self.__resp and self.__resp.drain_conn() + self.__resp and self.__resp.release_conn() + self.__resp = None + + def read(self, sz): + return self.__resp.read(sz) + + def isOpen(self): + return self.__resp is not None + def flush(self): + + # Pull data out of buffer that will be sent in this request + data = self.__wbuf.getvalue() + self.__wbuf = BytesIO() + + # Header handling + headers = dict(self._headers) self.__auth_provider.add_headers(headers) self._headers = headers self.setCustomHeaders(self._headers) - super().flush() + + # Note: we don't set User-Agent explicitly in this class because PySQL + # should always provide one. Unlike the original THttpClient class, our version + # doesn't define a default User-Agent and so should raise an exception if one + # isn't provided. + assert self.__custom_headers and "User-Agent" in self.__custom_headers + + headers = { + "Content-Type": "application/x-thrift", + "Content-Length": str(len(data)), + } + + if self.using_proxy() and self.scheme == "http" and self.proxy_auth is not None: + headers.update(self.proxy_auth) + + if self.__custom_headers: + custom_headers = {key: val for key, val in self.__custom_headers.items()} + headers.update(**custom_headers) + + # HTTP request + self.__resp = self.__pool.request( + "POST", + url=self.path, + body=data, + headers=headers, + preload_content=False, + timeout=self.__timeout, + retries=self.retry_policy, + ) + + # Get reply to flush the request + self.code = self.__resp.status + self.message = self.__resp.reason + self.headers = self.__resp.headers @staticmethod - def basic_proxy_auth_header(proxy): + def basic_proxy_auth_headers(proxy): if proxy is None or not proxy.username: return None ap = "%s:%s" % ( urllib.parse.unquote(proxy.username), urllib.parse.unquote(proxy.password), ) - cr = base64.b64encode(ap.encode()).strip() - return "Basic " + six.ensure_str(cr) + return make_headers(proxy_basic_auth=ap) + + def set_retry_command_type(self, value: CommandType): + """Pass the provided CommandType to the retry policy""" + if isinstance(self.retry_policy, DatabricksRetryPolicy): + self.retry_policy.command_type = value + else: + logger.warning( + "DatabricksRetryPolicy is currently bypassed. The CommandType cannot be set." + ) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py old mode 100644 new mode 100755 index 722ed778..dca286ef --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -1,25 +1,71 @@ -from typing import Dict, Tuple, List, Optional, Any, Union +import time +from typing import Dict, Tuple, List, Optional, Any, Union, Sequence import pandas -import pyarrow + +try: + import pyarrow +except ImportError: + pyarrow = None import requests import json import os +import decimal +from uuid import UUID from databricks.sql import __version__ from databricks.sql import * -from databricks.sql.exc import OperationalError +from databricks.sql.exc import ( + OperationalError, + SessionAlreadyClosedError, + CursorAlreadyClosedError, +) +from databricks.sql.thrift_api.TCLIService import ttypes from databricks.sql.thrift_backend import ThriftBackend -from databricks.sql.utils import ExecuteResponse, ParamEscaper, inject_parameters -from databricks.sql.types import Row +from databricks.sql.utils import ( + ExecuteResponse, + ParamEscaper, + inject_parameters, + transform_paramstyle, + ColumnTable, + ColumnQueue, +) +from databricks.sql.parameters.native import ( + DbsqlParameterBase, + TDbsqlParameter, + TParameterDict, + TParameterSequence, + TParameterCollection, + ParameterStructure, + dbsql_parameter_from_primitive, + ParameterApproach, +) + + +from databricks.sql.types import Row, SSLOptions from databricks.sql.auth.auth import get_python_sql_connector_auth_provider from databricks.sql.experimental.oauth_persistence import OAuthPersistence +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TSparkParameter, + TOperationState, +) + + logger = logging.getLogger(__name__) -DEFAULT_RESULT_BUFFER_SIZE_BYTES = 10485760 +if pyarrow is None: + logger.warning( + "[WARN] pyarrow is not installed by default since databricks-sql-connector 4.0.0," + "any arrow specific api (e.g. fetchmany_arrow) and cloud fetch will be disabled." + "If you need these features, please run pip install pyarrow or pip install databricks-sql-connector[pyarrow] to install" + ) + +DEFAULT_RESULT_BUFFER_SIZE_BYTES = 104857600 DEFAULT_ARRAY_SIZE = 100000 +NO_NATIVE_PARAMS: List = [] + class Connection: def __init__( @@ -28,9 +74,10 @@ def __init__( http_path: str, access_token: Optional[str] = None, http_headers: Optional[List[Tuple[str, str]]] = None, - session_configuration: Dict[str, Any] = None, + session_configuration: Optional[Dict[str, Any]] = None, catalog: Optional[str] = None, schema: Optional[str] = None, + _use_arrow_native_complex_types: Optional[bool] = True, **kwargs, ) -> None: """ @@ -44,11 +91,13 @@ def __init__( Http Bearer access token, e.g. Databricks Personal Access Token. Unless if you use auth_type=`databricks-oauth` you need to pass `access_token. Examples: + ``` connection = sql.connect( server_hostname='dbc-12345.staging.cloud.databricks.com', http_path='sql/protocolv1/o/6789/12abc567', access_token='dabpi12345678' ) + ``` :param http_headers: An optional list of (k, v) pairs that will be set as Http headers on every request :param session_configuration: An optional dictionary of Spark session parameters. Defaults to None. Execute the SQL command `SET -v` to get a full list of available commands. @@ -56,12 +105,15 @@ def __init__( :param schema: An optional initial schema to use. Requires DBR version 9.0+ Other Parameters: - auth_type: `str`, optional - `databricks-oauth` : to use oauth with fine-grained permission scopes, set to `databricks-oauth`. - This is currently in private preview for Databricks accounts on AWS. - This supports User to Machine OAuth authentication for Databricks on AWS with - any IDP configured. This is only for interactive python applications and open a browser window. - Note this is beta (private preview) + use_inline_params: `boolean` | str, optional (default is False) + When True, parameterized calls to cursor.execute() will try to render parameter values inline with the + query text instead of using native bound parameters supported in DBR 14.1 and above. This connector will attempt to + sanitise parameterized inputs to prevent SQL injection. The inline parameter approach is maintained for + legacy purposes and will be deprecated in a future release. When this parameter is `True` you will see + a warning log message. To suppress this log message, set `use_inline_params="silent"`. + auth_type: `str`, optional (default is databricks-oauth if neither `access_token` nor `tls_client_cert_file` is set) + `databricks-oauth` : to use Databricks OAuth with fine-grained permission scopes, set to `databricks-oauth`. + `azure-oauth` : to use Microsoft Entra ID OAuth flow, set to `azure-oauth`. oauth_client_id: `str`, optional custom oauth client_id. If not specified, it will use the built-in client_id of databricks-sql-python. @@ -72,9 +124,9 @@ def __init__( experimental_oauth_persistence: configures preferred storage for persisting oauth tokens. This has to be a class implementing `OAuthPersistence`. - When `auth_type` is set to `databricks-oauth` without persisting the oauth token in a persistence storage - the oauth tokens will only be maintained in memory and if the python process restarts the end user - will have to login again. + When `auth_type` is set to `databricks-oauth` or `azure-oauth` without persisting the oauth token in a + persistence storage the oauth tokens will only be maintained in memory and if the python process + restarts the end user will have to login again. Note this is beta (private preview) For persisting the oauth token in a prod environment you should subclass and implement OAuthPersistence @@ -103,6 +155,7 @@ def read(self) -> Optional[OAuthToken]: own implementation of OAuthPersistence. Examples: + ``` # for development only from databricks.sql.experimental.oauth_persistence import DevOnlyFilePersistence @@ -112,30 +165,37 @@ def read(self) -> Optional[OAuthToken]: auth_type="databricks-oauth", experimental_oauth_persistence=DevOnlyFilePersistence("~/dev-oauth.json") ) - - + ``` + :param _use_arrow_native_complex_types: `bool`, optional + Controls whether a complex type field value is returned as a string or as a native Arrow type. Defaults to True. + When True: + MAP is returned as List[Tuple[str, Any]] + STRUCT is returned as Dict[str, Any] + ARRAY is returned as numpy.ndarray + When False, complex types are returned as a strings. These are generally deserializable as JSON. """ # Internal arguments in **kwargs: # _user_agent_entry # Tag to add to User-Agent header. For use by partners. - # _username, _password - # Username and password Basic authentication (no official support) # _use_cert_as_auth - # Use a TLS cert instead of a token or username / password (internal use only) + # Use a TLS cert instead of a token # _enable_ssl # Connect over HTTP instead of HTTPS # _port # Which port to connect to # _skip_routing_headers: # Don't set routing headers if set to True (for use when connecting directly to server) + # _tls_no_verify + # Set to True (Boolean) to completely disable SSL verification. # _tls_verify_hostname # Set to False (Boolean) to disable SSL hostname verification, but check certificate. # _tls_trusted_ca_file # Set to the path of the file containing trusted CA certificates for server certificate # verification. If not provide, uses system truststore. - # _tls_client_cert_file, _tls_client_cert_key_file + # _tls_client_cert_file, _tls_client_cert_key_file, _tls_client_cert_key_password # Set client SSL certificate. + # See https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_cert_chain # _retry_stop_after_attempts_count # The maximum number of attempts during a request retry sequence (defaults to 24) # _socket_timeout @@ -144,15 +204,14 @@ def read(self) -> Optional[OAuthToken]: # _disable_pandas # In case the deserialisation through pandas causes any issues, it can be disabled with # this flag. - # _use_arrow_native_complex_types - # DBR will return native Arrow types for structs, arrays and maps instead of Arrow strings - # (True by default) # _use_arrow_native_decimals # Databricks runtime will return native Arrow types for decimals instead of Arrow strings # (True by default) # _use_arrow_native_timestamps # Databricks runtime will return native Arrow types for timestamps instead of Arrow strings # (True by default) + # use_cloud_fetch + # Enable use of cloud fetch to extract large query results in parallel via cloud storage if access_token: access_token_kv = {"access_token": access_token} @@ -177,23 +236,72 @@ def read(self) -> Optional[OAuthToken]: base_headers = [("User-Agent", useragent_header)] + self._ssl_options = SSLOptions( + # Double negation is generally a bad thing, but we have to keep backward compatibility + tls_verify=not kwargs.get( + "_tls_no_verify", False + ), # by default - verify cert and host + tls_verify_hostname=kwargs.get("_tls_verify_hostname", True), + tls_trusted_ca_file=kwargs.get("_tls_trusted_ca_file"), + tls_client_cert_file=kwargs.get("_tls_client_cert_file"), + tls_client_cert_key_file=kwargs.get("_tls_client_cert_key_file"), + tls_client_cert_key_password=kwargs.get("_tls_client_cert_key_password"), + ) + self.thrift_backend = ThriftBackend( self.host, self.port, http_path, (http_headers or []) + base_headers, auth_provider, + ssl_options=self._ssl_options, + _use_arrow_native_complex_types=_use_arrow_native_complex_types, **kwargs, ) - self._session_handle = self.thrift_backend.open_session( + self._open_session_resp = self.thrift_backend.open_session( session_configuration, catalog, schema ) + self._session_handle = self._open_session_resp.sessionHandle + self.protocol_version = self.get_protocol_version(self._open_session_resp) + self.use_cloud_fetch = kwargs.get("use_cloud_fetch", True) self.open = True - logger.info("Successfully opened session " + str(self.get_session_id())) + logger.info("Successfully opened session " + str(self.get_session_id_hex())) self._cursors = [] # type: List[Cursor] - def __enter__(self): + self.use_inline_params = self._set_use_inline_params_with_warning( + kwargs.get("use_inline_params", False) + ) + + def _set_use_inline_params_with_warning(self, value: Union[bool, str]): + """Valid values are True, False, and "silent" + + False: Use native parameters + True: Use inline parameters and log a warning + "silent": Use inline parameters and don't log a warning + """ + + if value is False: + return False + + if value not in [True, "silent"]: + raise ValueError( + f"Invalid value for use_inline_params: {value}. " + + 'Valid values are True, False, and "silent"' + ) + + if value is True: + logger.warning( + "Parameterised queries executed with this client will use the inline parameter approach." + "This approach will be deprecated in a future release. Consider using native parameters." + "Learn more: https://github.com/databricks/databricks-sql-python/tree/main/docs/parameters.md" + 'To suppress this warning, set use_inline_params="silent"' + ) + + return value + + # The ideal return type for this method is perhaps Self, but that was not added until 3.11, and we support pre-3.11 pythons, currently. + def __enter__(self) -> "Connection": return self def __exit__(self, exc_type, exc_value, traceback): @@ -203,7 +311,7 @@ def __del__(self): if self.open: logger.debug( "Closing unclosed connection for session " - "{}".format(self.get_session_id()) + "{}".format(self.get_session_id_hex()) ) try: self._close(close_cursors=False) @@ -214,6 +322,33 @@ def __del__(self): def get_session_id(self): return self.thrift_backend.handle_to_id(self._session_handle) + @staticmethod + def get_protocol_version(openSessionResp): + """ + Since the sessionHandle will sometimes have a serverProtocolVersion, it takes + precedence over the serverProtocolVersion defined in the OpenSessionResponse. + """ + if ( + openSessionResp.sessionHandle + and hasattr(openSessionResp.sessionHandle, "serverProtocolVersion") + and openSessionResp.sessionHandle.serverProtocolVersion + ): + return openSessionResp.sessionHandle.serverProtocolVersion + return openSessionResp.serverProtocolVersion + + @staticmethod + def server_parameterized_queries_enabled(protocolVersion): + if ( + protocolVersion + and protocolVersion >= ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V8 + ): + return True + else: + return False + + def get_session_id_hex(self): + return self.thrift_backend.handle_to_hex_id(self._session_handle) + def cursor( self, arraysize: int = DEFAULT_ARRAY_SIZE, @@ -244,7 +379,28 @@ def _close(self, close_cursors=True) -> None: if close_cursors: for cursor in self._cursors: cursor.close() - self.thrift_backend.close_session(self._session_handle) + + logger.info(f"Closing session {self.get_session_id_hex()}") + if not self.open: + logger.debug("Session appears to have been closed already") + + try: + self.thrift_backend.close_session(self._session_handle) + except RequestError as e: + if isinstance(e.args[1], SessionAlreadyClosedError): + logger.info("Session was closed by a prior request") + except DatabaseError as e: + if "Invalid SessionHandle" in str(e): + logger.warning( + f"Attempted to close session that was already closed: {e}" + ) + else: + logger.warning( + f"Attempt to close session raised an exception at the server: {e}" + ) + except Exception as e: + logger.error(f"Attempt to close session raised a local exception: {e}") + self.open = False def commit(self): @@ -283,7 +439,10 @@ def __init__( self.escaper = ParamEscaper() self.lastrowid = None - def __enter__(self): + self.ASYNC_DEFAULT_POLLING_INTERVAL = 2 + + # The ideal return type for this method is perhaps Self, but that was not added until 3.11, and we support pre-3.11 pythons, currently. + def __enter__(self) -> "Cursor": return self def __exit__(self, exc_type, exc_value, traceback): @@ -296,6 +455,132 @@ def __iter__(self): else: raise Error("There is no active result set") + def _determine_parameter_approach( + self, params: Optional[TParameterCollection] + ) -> ParameterApproach: + """Encapsulates the logic for choosing whether to send parameters in native vs inline mode + + If params is None then ParameterApproach.NONE is returned. + If self.use_inline_params is True then inline mode is used. + If self.use_inline_params is False, then check if the server supports them and proceed. + Else raise an exception. + + Returns a ParameterApproach enumeration or raises an exception + + If inline approach is used when the server supports native approach, a warning is logged + """ + + if params is None: + return ParameterApproach.NONE + + if self.connection.use_inline_params: + return ParameterApproach.INLINE + + else: + return ParameterApproach.NATIVE + + def _all_dbsql_parameters_are_named(self, params: List[TDbsqlParameter]) -> bool: + """Return True if all members of the list have a non-null .name attribute""" + return all([i.name is not None for i in params]) + + def _normalize_tparametersequence( + self, params: TParameterSequence + ) -> List[TDbsqlParameter]: + """Retains the same order as the input list.""" + + output: List[TDbsqlParameter] = [] + for p in params: + if isinstance(p, DbsqlParameterBase): + output.append(p) + else: + output.append(dbsql_parameter_from_primitive(value=p)) + + return output + + def _normalize_tparameterdict( + self, params: TParameterDict + ) -> List[TDbsqlParameter]: + return [ + dbsql_parameter_from_primitive(value=value, name=name) + for name, value in params.items() + ] + + def _normalize_tparametercollection( + self, params: Optional[TParameterCollection] + ) -> List[TDbsqlParameter]: + if params is None: + return [] + if isinstance(params, dict): + return self._normalize_tparameterdict(params) + if isinstance(params, Sequence): + return self._normalize_tparametersequence(list(params)) + + def _determine_parameter_structure( + self, + parameters: List[TDbsqlParameter], + ) -> ParameterStructure: + all_named = self._all_dbsql_parameters_are_named(parameters) + if all_named: + return ParameterStructure.NAMED + else: + return ParameterStructure.POSITIONAL + + def _prepare_inline_parameters( + self, stmt: str, params: Optional[Union[Sequence, Dict[str, Any]]] + ) -> Tuple[str, List]: + """Return a statement and list of native parameters to be passed to thrift_backend for execution + + :stmt: + A string SQL query containing parameter markers of PEP-249 paramstyle `pyformat`. + For example `%(param)s`. + + :params: + An iterable of parameter values to be rendered inline. If passed as a Dict, the keys + must match the names of the markers included in :stmt:. If passed as a List, its length + must equal the count of parameter markers in :stmt:. + + Returns a tuple of: + stmt: the passed statement with the param markers replaced by literal rendered values + params: an empty list representing the native parameters to be passed with this query. + The list is always empty because native parameters are never used under the inline approach + """ + + escaped_values = self.escaper.escape_args(params) + rendered_statement = inject_parameters(stmt, escaped_values) + + return rendered_statement, NO_NATIVE_PARAMS + + def _prepare_native_parameters( + self, + stmt: str, + params: List[TDbsqlParameter], + param_structure: ParameterStructure, + ) -> Tuple[str, List[TSparkParameter]]: + """Return a statement and a list of native parameters to be passed to thrift_backend for execution + + :stmt: + A string SQL query containing parameter markers of PEP-249 paramstyle `named`. + For example `:param`. + + :params: + An iterable of parameter values to be sent natively. If passed as a Dict, the keys + must match the names of the markers included in :stmt:. If passed as a List, its length + must equal the count of parameter markers in :stmt:. In list form, any member of the list + can be wrapped in a DbsqlParameter class. + + Returns a tuple of: + stmt: the passed statement` with the param markers replaced by literal rendered values + params: a list of TSparkParameters that will be passed in native mode + """ + + stmt = stmt + output = [ + p.as_tspark_param(named=param_structure == ParameterStructure.NAMED) + for p in params + ] + + return stmt, output + def _close_and_clear_active_result_set(self): try: if self.active_result_set: @@ -355,12 +640,15 @@ def _handle_staging_operation( "Local file operations are restricted to paths within the configured staging_allowed_local_path" ) - # TODO: Experiment with DBR sending real headers. - # The specification says headers will be in JSON format but the current null value is actually an empty list [] + # May be real headers, or could be json string + headers = ( + json.loads(row.headers) if isinstance(row.headers, str) else row.headers + ) + handler_args = { "presigned_url": row.presignedUrl, "local_file": abs_localFile, - "headers": json.loads(row.headers or "{}"), + "headers": dict(headers) or {}, } logger.debug( @@ -383,7 +671,7 @@ def _handle_staging_operation( ) def _handle_staging_put( - self, presigned_url: str, local_file: str, headers: dict = None + self, presigned_url: str, local_file: str, headers: Optional[dict] = None ): """Make an HTTP PUT request @@ -398,7 +686,7 @@ def _handle_staging_put( # fmt: off # Design borrowed from: https://stackoverflow.com/a/2342589/5093960 - + OK = requests.codes.ok # 200 CREATED = requests.codes.created # 201 ACCEPTED = requests.codes.accepted # 202 @@ -418,7 +706,7 @@ def _handle_staging_put( ) def _handle_staging_get( - self, local_file: str, presigned_url: str, headers: dict = None + self, local_file: str, presigned_url: str, headers: Optional[dict] = None ): """Make an HTTP GET request, create a local file with the received data @@ -440,7 +728,9 @@ def _handle_staging_get( with open(local_file, "wb") as fp: fp.write(r.content) - def _handle_staging_remove(self, presigned_url: str, headers: dict = None): + def _handle_staging_remove( + self, presigned_url: str, headers: Optional[dict] = None + ): """Make an HTTP DELETE request to the presigned_url""" r = requests.delete(url=presigned_url, headers=headers) @@ -451,31 +741,73 @@ def _handle_staging_remove(self, presigned_url: str, headers: dict = None): ) def execute( - self, operation: str, parameters: Optional[Dict[str, str]] = None + self, + operation: str, + parameters: Optional[TParameterCollection] = None, ) -> "Cursor": """ Execute a query and wait for execution to complete. - Parameters should be given in extended param format style: %(...). - For example: - operation = "SELECT * FROM table WHERE field = %(some_value)s" - parameters = {"some_value": "foo"} - Will result in the query "SELECT * FROM table WHERE field = 'foo' being sent to the server + + The parameterisation behaviour of this method depends on which parameter approach is used: + - With INLINE mode, parameters are rendered inline with the query text + - With NATIVE mode (default), parameters are sent to the server separately for binding + + This behaviour is controlled by the `use_inline_params` argument passed when building a connection. + + The paramstyle for these approaches is different: + + If the connection was instantiated with use_inline_params=False (default), then parameters + should be given in PEP-249 `named` paramstyle like :param_name. Parameters passed by positionally + are indicated using a `?` in the query text. + + If the connection was instantiated with use_inline_params=True, then parameters + should be given in PEP-249 `pyformat` paramstyle like %(param_name)s. Parameters passed by positionally + are indicated using a `%s` marker in the query. Note: this approach is not recommended as it can break + your SQL query syntax and will be removed in a future release. + + ```python + inline_operation = "SELECT * FROM table WHERE field = %(some_value)s" + native_operation = "SELECT * FROM table WHERE field = :some_value" + parameters = {"some_value": "foo"} + ``` + + Both will result in the query equivalent to "SELECT * FROM table WHERE field = 'foo' + being sent to the server + :returns self """ - if parameters is not None: - operation = inject_parameters( - operation, self.escaper.escape_args(parameters) + + param_approach = self._determine_parameter_approach(parameters) + if param_approach == ParameterApproach.NONE: + prepared_params = NO_NATIVE_PARAMS + prepared_operation = operation + + elif param_approach == ParameterApproach.INLINE: + prepared_operation, prepared_params = self._prepare_inline_parameters( + operation, parameters + ) + elif param_approach == ParameterApproach.NATIVE: + normalized_parameters = self._normalize_tparametercollection(parameters) + param_structure = self._determine_parameter_structure(normalized_parameters) + transformed_operation = transform_paramstyle( + operation, normalized_parameters, param_structure + ) + prepared_operation, prepared_params = self._prepare_native_parameters( + transformed_operation, normalized_parameters, param_structure ) self._check_not_closed() self._close_and_clear_active_result_set() execute_response = self.thrift_backend.execute_command( - operation=operation, + operation=prepared_operation, session_handle=self.connection._session_handle, max_rows=self.arraysize, max_bytes=self.buffer_size_bytes, lz4_compression=self.connection.lz4_compression, cursor=self, + use_cloud_fetch=self.connection.use_cloud_fetch, + parameters=prepared_params, + async_op=False, ) self.active_result_set = ResultSet( self.connection, @@ -483,6 +815,7 @@ def execute( self.thrift_backend, self.buffer_size_bytes, self.arraysize, + self.connection.use_cloud_fetch, ) if execute_response.is_staging_operation: @@ -492,10 +825,112 @@ def execute( return self + def execute_async( + self, + operation: str, + parameters: Optional[TParameterCollection] = None, + ) -> "Cursor": + """ + + Execute a query and do not wait for it to complete and just move ahead + + :param operation: + :param parameters: + :return: + """ + param_approach = self._determine_parameter_approach(parameters) + if param_approach == ParameterApproach.NONE: + prepared_params = NO_NATIVE_PARAMS + prepared_operation = operation + + elif param_approach == ParameterApproach.INLINE: + prepared_operation, prepared_params = self._prepare_inline_parameters( + operation, parameters + ) + elif param_approach == ParameterApproach.NATIVE: + normalized_parameters = self._normalize_tparametercollection(parameters) + param_structure = self._determine_parameter_structure(normalized_parameters) + transformed_operation = transform_paramstyle( + operation, normalized_parameters, param_structure + ) + prepared_operation, prepared_params = self._prepare_native_parameters( + transformed_operation, normalized_parameters, param_structure + ) + + self._check_not_closed() + self._close_and_clear_active_result_set() + self.thrift_backend.execute_command( + operation=prepared_operation, + session_handle=self.connection._session_handle, + max_rows=self.arraysize, + max_bytes=self.buffer_size_bytes, + lz4_compression=self.connection.lz4_compression, + cursor=self, + use_cloud_fetch=self.connection.use_cloud_fetch, + parameters=prepared_params, + async_op=True, + ) + + return self + + def get_query_state(self) -> "TOperationState": + """ + Get the state of the async executing query or basically poll the status of the query + + :return: + """ + self._check_not_closed() + return self.thrift_backend.get_query_state(self.active_op_handle) + + def get_async_execution_result(self): + """ + + Checks for the status of the async executing query and fetches the result if the query is finished + Otherwise it will keep polling the status of the query till there is a Not pending state + :return: + """ + self._check_not_closed() + + def is_executing(operation_state) -> "bool": + return not operation_state or operation_state in [ + ttypes.TOperationState.RUNNING_STATE, + ttypes.TOperationState.PENDING_STATE, + ] + + while is_executing(self.get_query_state()): + # Poll after some default time + time.sleep(self.ASYNC_DEFAULT_POLLING_INTERVAL) + + operation_state = self.get_query_state() + if operation_state == ttypes.TOperationState.FINISHED_STATE: + execute_response = self.thrift_backend.get_execution_result( + self.active_op_handle, self + ) + self.active_result_set = ResultSet( + self.connection, + execute_response, + self.thrift_backend, + self.buffer_size_bytes, + self.arraysize, + ) + + if execute_response.is_staging_operation: + self._handle_staging_operation( + staging_allowed_local_path=self.thrift_backend.staging_allowed_local_path + ) + + return self + else: + raise Error( + f"get_execution_result failed with Operation status {operation_state}" + ) + def executemany(self, operation, seq_of_parameters): """ - Prepare a database operation (query or command) and then execute it against all parameter - sequences or mappings found in the sequence ``seq_of_parameters``. + Execute the operation once for every set of passed in parameters. + + This will issue N sequential request to the database where N is the length of the provided sequence. + No optimizations of the query (like batching) will be performed. Only the final result set is retained. @@ -561,7 +996,7 @@ def tables( catalog_name: Optional[str] = None, schema_name: Optional[str] = None, table_name: Optional[str] = None, - table_types: List[str] = None, + table_types: Optional[List[str]] = None, ) -> "Cursor": """ Get tables corresponding to the catalog_name, schema_name and table_name. @@ -675,14 +1110,14 @@ def fetchmany(self, size: int) -> List[Row]: else: raise Error("There is no active result set") - def fetchall_arrow(self) -> pyarrow.Table: + def fetchall_arrow(self) -> "pyarrow.Table": self._check_not_closed() if self.active_result_set: return self.active_result_set.fetchall_arrow() else: raise Error("There is no active result set") - def fetchmany_arrow(self, size) -> pyarrow.Table: + def fetchmany_arrow(self, size) -> "pyarrow.Table": self._check_not_closed() if self.active_result_set: return self.active_result_set.fetchmany_arrow(size) @@ -707,9 +1142,22 @@ def cancel(self) -> None: def close(self) -> None: """Close cursor""" self.open = False + self.active_op_handle = None if self.active_result_set: self._close_and_clear_active_result_set() + @property + def query_id(self) -> Optional[str]: + """ + This attribute is an identifier of last executed query. + + This attribute will be ``None`` if the cursor has not had an operation + invoked via the execute method yet, or if cursor was closed. + """ + if self.active_op_handle is not None: + return str(UUID(bytes=self.active_op_handle.operationId.guid)) + return None + @property def description(self) -> Optional[List[Tuple]]: """ @@ -762,6 +1210,7 @@ def __init__( thrift_backend: ThriftBackend, result_buffer_size_bytes: int = DEFAULT_RESULT_BUFFER_SIZE_BYTES, arraysize: int = 10000, + use_cloud_fetch: bool = True, ): """ A ResultSet manages the results of a single command. @@ -783,6 +1232,7 @@ def __init__( self.description = execute_response.description self._arrow_schema_bytes = execute_response.arrow_schema_bytes self._next_row_index = 0 + self._use_cloud_fetch = use_cloud_fetch if execute_response.arrow_queue: # In this case the server has taken the fast path and returned an initial batch of @@ -801,6 +1251,7 @@ def __iter__(self): break def _fill_results_buffer(self): + # At initialization or if the server does not have cloud fetch result links available results, has_more_rows = self.thrift_backend.fetch_results( op_handle=self.command_id, max_rows=self.arraysize, @@ -809,10 +1260,23 @@ def _fill_results_buffer(self): lz4_compressed=self.lz4_compressed, arrow_schema_bytes=self._arrow_schema_bytes, description=self.description, + use_cloud_fetch=self._use_cloud_fetch, ) self.results = results self.has_more_rows = has_more_rows + def _convert_columnar_table(self, table): + column_names = [c[0] for c in self.description] + ResultRow = Row(*column_names) + result = [] + for row_index in range(table.num_rows): + curr_row = [] + for col_index in range(table.num_columns): + curr_row.append(table.get_item(col_index, row_index)) + result.append(ResultRow(*curr_row)) + + return result + def _convert_arrow_table(self, table): column_names = [c[0] for c in self.description] ResultRow = Row(*column_names) @@ -848,14 +1312,14 @@ def _convert_arrow_table(self, table): timestamp_as_object=True, ) - res = df.to_numpy(na_value=None) + res = df.to_numpy(na_value=None, dtype="object") return [ResultRow(*v) for v in res] @property def rownumber(self): return self._next_row_index - def fetchmany_arrow(self, size: int) -> pyarrow.Table: + def fetchmany_arrow(self, size: int) -> "pyarrow.Table": """ Fetch the next set of rows of a query result, returning a PyArrow table. @@ -880,7 +1344,49 @@ def fetchmany_arrow(self, size: int) -> pyarrow.Table: return results - def fetchall_arrow(self) -> pyarrow.Table: + def merge_columnar(self, result1, result2): + """ + Function to merge / combining the columnar results into a single result + :param result1: + :param result2: + :return: + """ + + if result1.column_names != result2.column_names: + raise ValueError("The columns in the results don't match") + + merged_result = [ + result1.column_table[i] + result2.column_table[i] + for i in range(result1.num_columns) + ] + return ColumnTable(merged_result, result1.column_names) + + def fetchmany_columnar(self, size: int): + """ + Fetch the next set of rows of a query result, returning a Columnar Table. + An empty sequence is returned when no more rows are available. + """ + if size < 0: + raise ValueError("size argument for fetchmany is %s but must be >= 0", size) + + results = self.results.next_n_rows(size) + n_remaining_rows = size - results.num_rows + self._next_row_index += results.num_rows + + while ( + n_remaining_rows > 0 + and not self.has_been_closed_server_side + and self.has_more_rows + ): + self._fill_results_buffer() + partial_results = self.results.next_n_rows(n_remaining_rows) + results = self.merge_columnar(results, partial_results) + n_remaining_rows -= partial_results.num_rows + self._next_row_index += partial_results.num_rows + + return results + + def fetchall_arrow(self) -> "pyarrow.Table": """Fetch all (remaining) rows of a query result, returning them as a PyArrow table.""" results = self.results.remaining_rows() self._next_row_index += results.num_rows @@ -893,12 +1399,30 @@ def fetchall_arrow(self) -> pyarrow.Table: return results + def fetchall_columnar(self): + """Fetch all (remaining) rows of a query result, returning them as a Columnar table.""" + results = self.results.remaining_rows() + self._next_row_index += results.num_rows + + while not self.has_been_closed_server_side and self.has_more_rows: + self._fill_results_buffer() + partial_results = self.results.remaining_rows() + results = self.merge_columnar(results, partial_results) + self._next_row_index += partial_results.num_rows + + return results + def fetchone(self) -> Optional[Row]: """ Fetch the next row of a query result set, returning a single sequence, or None when no more data is available. """ - res = self._convert_arrow_table(self.fetchmany_arrow(1)) + + if isinstance(self.results, ColumnQueue): + res = self._convert_columnar_table(self.fetchmany_columnar(1)) + else: + res = self._convert_arrow_table(self.fetchmany_arrow(1)) + if len(res) > 0: return res[0] else: @@ -908,7 +1432,10 @@ def fetchall(self) -> List[Row]: """ Fetch all (remaining) rows of a query result, returning them as a list of rows. """ - return self._convert_arrow_table(self.fetchall_arrow()) + if isinstance(self.results, ColumnQueue): + return self._convert_columnar_table(self.fetchall_columnar()) + else: + return self._convert_arrow_table(self.fetchall_arrow()) def fetchmany(self, size: int) -> List[Row]: """ @@ -916,7 +1443,10 @@ def fetchmany(self, size: int) -> List[Row]: An empty sequence is returned when no more rows are available. """ - return self._convert_arrow_table(self.fetchmany_arrow(size)) + if isinstance(self.results, ColumnQueue): + return self._convert_columnar_table(self.fetchmany_columnar(size)) + else: + return self._convert_arrow_table(self.fetchmany_arrow(size)) def close(self) -> None: """ @@ -932,6 +1462,9 @@ def close(self) -> None: and self.connection.open ): self.thrift_backend.close_command(self.command_id) + except RequestError as e: + if isinstance(e.args[1], CursorAlreadyClosedError): + logger.info("Operation was canceled by a prior request") finally: self.has_been_closed_server_side = True self.op_state = self.thrift_backend.CLOSED_OP_STATE diff --git a/src/databricks/sql/cloudfetch/download_manager.py b/src/databricks/sql/cloudfetch/download_manager.py new file mode 100644 index 00000000..7e96cd32 --- /dev/null +++ b/src/databricks/sql/cloudfetch/download_manager.py @@ -0,0 +1,108 @@ +import logging + +from concurrent.futures import ThreadPoolExecutor, Future +from typing import List, Union + +from databricks.sql.cloudfetch.downloader import ( + ResultSetDownloadHandler, + DownloadableResultSettings, + DownloadedFile, +) +from databricks.sql.types import SSLOptions + +from databricks.sql.thrift_api.TCLIService.ttypes import TSparkArrowResultLink + +logger = logging.getLogger(__name__) + + +class ResultFileDownloadManager: + def __init__( + self, + links: List[TSparkArrowResultLink], + max_download_threads: int, + lz4_compressed: bool, + ssl_options: SSLOptions, + ): + self._pending_links: List[TSparkArrowResultLink] = [] + for link in links: + if link.rowCount <= 0: + continue + logger.debug( + "ResultFileDownloadManager: adding file link, start offset {}, row count: {}".format( + link.startRowOffset, link.rowCount + ) + ) + self._pending_links.append(link) + + self._download_tasks: List[Future[DownloadedFile]] = [] + self._max_download_threads: int = max_download_threads + self._thread_pool = ThreadPoolExecutor(max_workers=self._max_download_threads) + + self._downloadable_result_settings = DownloadableResultSettings(lz4_compressed) + self._ssl_options = ssl_options + + def get_next_downloaded_file( + self, next_row_offset: int + ) -> Union[DownloadedFile, None]: + """ + Get next file that starts at given offset. + + This function gets the next downloaded file in which its rows start at the specified next_row_offset + in relation to the full result. File downloads are scheduled if not already, and once the correct + download handler is located, the function waits for the download status and returns the resulting file. + If there are no more downloads, a download was not successful, or the correct file could not be located, + this function shuts down the thread pool and returns None. + + Args: + next_row_offset (int): The offset of the starting row of the next file we want data from. + """ + + # Make sure the download queue is always full + self._schedule_downloads() + + # No more files to download from this batch of links + if len(self._download_tasks) == 0: + self._shutdown_manager() + return None + + task = self._download_tasks.pop(0) + # Future's `result()` method will wait for the call to complete, and return + # the value returned by the call. If the call throws an exception - `result()` + # will throw the same exception + file = task.result() + if (next_row_offset < file.start_row_offset) or ( + next_row_offset > file.start_row_offset + file.row_count + ): + logger.debug( + "ResultFileDownloadManager: file does not contain row {}, start {}, row count {}".format( + next_row_offset, file.start_row_offset, file.row_count + ) + ) + + return file + + def _schedule_downloads(self): + """ + While download queue has a capacity, peek pending links and submit them to thread pool. + """ + logger.debug("ResultFileDownloadManager: schedule downloads") + while (len(self._download_tasks) < self._max_download_threads) and ( + len(self._pending_links) > 0 + ): + link = self._pending_links.pop(0) + logger.debug( + "- start: {}, row count: {}".format(link.startRowOffset, link.rowCount) + ) + handler = ResultSetDownloadHandler( + settings=self._downloadable_result_settings, + link=link, + ssl_options=self._ssl_options, + ) + task = self._thread_pool.submit(handler.run) + self._download_tasks.append(task) + + def _shutdown_manager(self): + # Clear download handlers and shutdown the thread pool + self._pending_links = [] + self._download_tasks = [] + self._thread_pool.shutdown(wait=False) diff --git a/src/databricks/sql/cloudfetch/downloader.py b/src/databricks/sql/cloudfetch/downloader.py new file mode 100644 index 00000000..228e07d6 --- /dev/null +++ b/src/databricks/sql/cloudfetch/downloader.py @@ -0,0 +1,176 @@ +import logging +from dataclasses import dataclass + +import requests +from requests.adapters import HTTPAdapter, Retry +import lz4.frame +import time + +from databricks.sql.thrift_api.TCLIService.ttypes import TSparkArrowResultLink +from databricks.sql.exc import Error +from databricks.sql.types import SSLOptions + +logger = logging.getLogger(__name__) + +# TODO: Ideally, we should use a common retry policy (DatabricksRetryPolicy) for all the requests across the library. +# But DatabricksRetryPolicy should be updated first - currently it can work only with Thrift requests +retryPolicy = Retry( + total=5, # max retry attempts + backoff_factor=1, # min delay, 1 second + # TODO: `backoff_max` is supported since `urllib3` v2.0.0, but we allow >= 1.26. + # The default value (120 seconds) used since v1.26 looks reasonable enough + # backoff_max=60, # max delay, 60 seconds + # retry all status codes below 100, 429 (Too Many Requests), and all codes above 500, + # excluding 501 Not implemented + status_forcelist=[*range(0, 101), 429, 500, *range(502, 1000)], +) + + +@dataclass +class DownloadedFile: + """ + Class for the result file and metadata. + + Attributes: + file_bytes (bytes): Downloaded file in bytes. + start_row_offset (int): The offset of the starting row in relation to the full result. + row_count (int): Number of rows the file represents in the result. + """ + + file_bytes: bytes + start_row_offset: int + row_count: int + + +@dataclass +class DownloadableResultSettings: + """ + Class for settings common to each download handler. + + Attributes: + is_lz4_compressed (bool): Whether file is expected to be lz4 compressed. + link_expiry_buffer_secs (int): Time in seconds to prevent download of a link before it expires. Default 0 secs. + download_timeout (int): Timeout for download requests. Default 60 secs. + max_consecutive_file_download_retries (int): Number of consecutive download retries before shutting down. + """ + + is_lz4_compressed: bool + link_expiry_buffer_secs: int = 0 + download_timeout: int = 60 + max_consecutive_file_download_retries: int = 0 + + +class ResultSetDownloadHandler: + def __init__( + self, + settings: DownloadableResultSettings, + link: TSparkArrowResultLink, + ssl_options: SSLOptions, + ): + self.settings = settings + self.link = link + self._ssl_options = ssl_options + + def run(self) -> DownloadedFile: + """ + Download the file described in the cloud fetch link. + + This function checks if the link has or is expiring, gets the file via a requests session, decompresses the + file, and signals to waiting threads that the download is finished and whether it was successful. + """ + + logger.debug( + "ResultSetDownloadHandler: starting file download, offset {}, row count {}".format( + self.link.startRowOffset, self.link.rowCount + ) + ) + + # Check if link is already expired or is expiring + ResultSetDownloadHandler._validate_link( + self.link, self.settings.link_expiry_buffer_secs + ) + + session = requests.Session() + session.mount("http://", HTTPAdapter(max_retries=retryPolicy)) + session.mount("https://", HTTPAdapter(max_retries=retryPolicy)) + + try: + # Get the file via HTTP request + response = session.get( + self.link.fileLink, + timeout=self.settings.download_timeout, + verify=self._ssl_options.tls_verify, + headers=self.link.httpHeaders + # TODO: Pass cert from `self._ssl_options` + ) + response.raise_for_status() + + # Save (and decompress if needed) the downloaded file + compressed_data = response.content + decompressed_data = ( + ResultSetDownloadHandler._decompress_data(compressed_data) + if self.settings.is_lz4_compressed + else compressed_data + ) + + # The size of the downloaded file should match the size specified from TSparkArrowResultLink + if len(decompressed_data) != self.link.bytesNum: + logger.debug( + "ResultSetDownloadHandler: downloaded file size {} does not match the expected value {}".format( + len(decompressed_data), self.link.bytesNum + ) + ) + + logger.debug( + "ResultSetDownloadHandler: successfully downloaded file, offset {}, row count {}".format( + self.link.startRowOffset, self.link.rowCount + ) + ) + + return DownloadedFile( + decompressed_data, + self.link.startRowOffset, + self.link.rowCount, + ) + finally: + if session: + session.close() + + @staticmethod + def _validate_link(link: TSparkArrowResultLink, expiry_buffer_secs: int): + """ + Check if a link has expired or will expire. + + Expiry buffer can be set to avoid downloading files that has not expired yet when the function is called, + but may expire before the file has fully downloaded. + """ + current_time = int(time.time()) + if ( + link.expiryTime <= current_time + or link.expiryTime - current_time <= expiry_buffer_secs + ): + raise Error("CloudFetch link has expired") + + @staticmethod + def _decompress_data(compressed_data: bytes) -> bytes: + """ + Decompress lz4 frame compressed data. + + Decompresses data that has been lz4 compressed, either via the whole frame or by series of chunks. + """ + uncompressed_data, bytes_read = lz4.frame.decompress( + compressed_data, return_bytes_read=True + ) + # The last cloud fetch file of the entire result is commonly punctuated by frequent end-of-frame markers. + # Full frame decompression above will short-circuit, so chunking is necessary + if bytes_read < len(compressed_data): + d_context = lz4.frame.create_decompression_context() + start = 0 + uncompressed_data = bytearray() + while start < len(compressed_data): + data, num_bytes, is_end = lz4.frame.decompress_chunk( + d_context, compressed_data[start:] + ) + uncompressed_data += data + start += num_bytes + return uncompressed_data diff --git a/src/databricks/sql/exc.py b/src/databricks/sql/exc.py index bb1e203e..3b27283a 100644 --- a/src/databricks/sql/exc.py +++ b/src/databricks/sql/exc.py @@ -93,3 +93,25 @@ class RequestError(OperationalError): """ pass + + +class MaxRetryDurationError(RequestError): + """Thrown if the next HTTP request retry would exceed the configured + stop_after_attempts_duration + """ + + +class NonRecoverableNetworkError(RequestError): + """Thrown if an HTTP code 501 is received""" + + +class UnsafeToRetryError(RequestError): + """Thrown if ExecuteStatement request receives a code other than 200, 429, or 503""" + + +class SessionAlreadyClosedError(RequestError): + """Thrown if CloseSession receives a code 404. ThriftBackend should gracefully proceed as this is expected.""" + + +class CursorAlreadyClosedError(RequestError): + """Thrown if CancelOperation receives a code 404. ThriftBackend should gracefully proceed as this is expected.""" diff --git a/src/databricks/sql/experimental/oauth_persistence.py b/src/databricks/sql/experimental/oauth_persistence.py index bd0066d9..13a96612 100644 --- a/src/databricks/sql/experimental/oauth_persistence.py +++ b/src/databricks/sql/experimental/oauth_persistence.py @@ -27,6 +27,17 @@ def read(self, hostname: str) -> Optional[OAuthToken]: pass +class OAuthPersistenceCache(OAuthPersistence): + def __init__(self): + self.tokens = {} + + def persist(self, hostname: str, oauth_token: OAuthToken): + self.tokens[hostname] = oauth_token + + def read(self, hostname: str) -> Optional[OAuthToken]: + return self.tokens.get(hostname) + + # Note this is only intended to be used for development class DevOnlyFilePersistence(OAuthPersistence): def __init__(self, file_path): diff --git a/src/databricks/sql/parameters/__init__.py b/src/databricks/sql/parameters/__init__.py new file mode 100644 index 00000000..3c39cf2b --- /dev/null +++ b/src/databricks/sql/parameters/__init__.py @@ -0,0 +1,15 @@ +from databricks.sql.parameters.native import ( + IntegerParameter, + StringParameter, + BigIntegerParameter, + BooleanParameter, + DateParameter, + DoubleParameter, + FloatParameter, + VoidParameter, + SmallIntParameter, + TimestampParameter, + TimestampNTZParameter, + TinyIntParameter, + DecimalParameter, +) diff --git a/src/databricks/sql/parameters/native.py b/src/databricks/sql/parameters/native.py new file mode 100644 index 00000000..8a436355 --- /dev/null +++ b/src/databricks/sql/parameters/native.py @@ -0,0 +1,606 @@ +import datetime +import decimal +from enum import Enum, auto +from typing import Optional, Sequence + +from databricks.sql.exc import NotSupportedError +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TSparkParameter, + TSparkParameterValue, +) + +import datetime +import decimal +from enum import Enum, auto +from typing import Dict, List, Union + + +class ParameterApproach(Enum): + INLINE = 1 + NATIVE = 2 + NONE = 3 + + +class ParameterStructure(Enum): + NAMED = 1 + POSITIONAL = 2 + NONE = 3 + + +class DatabricksSupportedType(Enum): + """Enumerate every supported Databricks SQL type shown here: + + https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html + """ + + BIGINT = auto() + BINARY = auto() + BOOLEAN = auto() + DATE = auto() + DECIMAL = auto() + DOUBLE = auto() + FLOAT = auto() + INT = auto() + INTERVAL = auto() + VOID = auto() + SMALLINT = auto() + STRING = auto() + TIMESTAMP = auto() + TIMESTAMP_NTZ = auto() + TINYINT = auto() + ARRAY = auto() + MAP = auto() + STRUCT = auto() + + +TAllowedParameterValue = Union[ + str, int, float, datetime.datetime, datetime.date, bool, decimal.Decimal, None +] + + +class DbsqlParameterBase: + """Parent class for IntegerParameter, DecimalParameter etc.. + + Each each instance that extends this base class should be capable of generating a TSparkParameter + It should know how to generate a cast expression based off its DatabricksSupportedType. + + By default the cast expression should render the string value of it's `value` and the literal + name of its Databricks Supported Type + + Interface should be: + + from databricks.sql.parameters import DecimalParameter + param = DecimalParameter(value, scale=None, precision=None) + cursor.execute("SELECT ?",[param]) + + Or + + from databricks.sql.parameters import IntegerParameter + param = IntegerParameter(42) + cursor.execute("SELECT ?", [param]) + """ + + CAST_EXPR: str + name: Optional[str] + + def as_tspark_param(self, named: bool) -> TSparkParameter: + """Returns a TSparkParameter object that can be passed to the DBR thrift server.""" + + tsp = TSparkParameter(value=self._tspark_param_value(), type=self._cast_expr()) + + if named: + tsp.name = self.name + tsp.ordinal = False + elif not named: + tsp.ordinal = True + return tsp + + def _tspark_param_value(self): + return TSparkParameterValue(stringValue=str(self.value)) + + def _cast_expr(self): + return self.CAST_EXPR + + def __str__(self): + return f"{self.__class__}(name={self.name}, value={self.value})" + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + +class IntegerParameter(DbsqlParameterBase): + """Wrap a Python `int` that will be bound to a Databricks SQL INT column.""" + + def __init__(self, value: int, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to an INT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.INT.name + + +class StringParameter(DbsqlParameterBase): + """Wrap a Python `str` that will be bound to a Databricks SQL STRING column.""" + + def __init__(self, value: str, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a STRING. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.STRING.name + + +class BigIntegerParameter(DbsqlParameterBase): + """Wrap a Python `int` that will be bound to a Databricks SQL BIGINT column.""" + + def __init__(self, value: int, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a BIGINT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.BIGINT.name + + +class BooleanParameter(DbsqlParameterBase): + """Wrap a Python `bool` that will be bound to a Databricks SQL BOOLEAN column.""" + + def __init__(self, value: bool, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a BOOLEAN. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.BOOLEAN.name + + +class DateParameter(DbsqlParameterBase): + """Wrap a Python `date` that will be bound to a Databricks SQL DATE column.""" + + def __init__(self, value: datetime.date, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a DATE. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.DATE.name + + +class DoubleParameter(DbsqlParameterBase): + """Wrap a Python `float` that will be bound to a Databricks SQL DOUBLE column.""" + + def __init__(self, value: float, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a DOUBLE. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.DOUBLE.name + + +class FloatParameter(DbsqlParameterBase): + """Wrap a Python `float` that will be bound to a Databricks SQL FLOAT column.""" + + def __init__(self, value: float, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a FLOAT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.FLOAT.name + + +class VoidParameter(DbsqlParameterBase): + """Wrap a Python `None` that will be bound to a Databricks SQL VOID type.""" + + def __init__(self, value: None, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a VOID. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.VOID.name + + def _tspark_param_value(self): + """For Void types, the TSparkParameter.value should be a Python NoneType""" + return None + + +class SmallIntParameter(DbsqlParameterBase): + """Wrap a Python `int` that will be bound to a Databricks SQL SMALLINT type.""" + + def __init__(self, value: int, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a SMALLINT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.SMALLINT.name + + +class TimestampParameter(DbsqlParameterBase): + """Wrap a Python `datetime` that will be bound to a Databricks SQL TIMESTAMP type.""" + + def __init__(self, value: datetime.datetime, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a TIMESTAMP. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.TIMESTAMP.name + + +class TimestampNTZParameter(DbsqlParameterBase): + """Wrap a Python `datetime` that will be bound to a Databricks SQL TIMESTAMP_NTZ type.""" + + def __init__(self, value: datetime.datetime, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a TIMESTAMP_NTZ. + If it contains a timezone, that info will be lost. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.TIMESTAMP_NTZ.name + + +class TinyIntParameter(DbsqlParameterBase): + """Wrap a Python `int` that will be bound to a Databricks SQL TINYINT type.""" + + def __init__(self, value: int, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a TINYINT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.TINYINT.name + + +class DecimalParameter(DbsqlParameterBase): + """Wrap a Python `Decimal` that will be bound to a Databricks SQL DECIMAL type.""" + + CAST_EXPR = "DECIMAL({},{})" + + def __init__( + self, + value: decimal.Decimal, + name: Optional[str] = None, + scale: Optional[int] = None, + precision: Optional[int] = None, + ): + """ + If set, `scale` and `precision` must both be set. If neither is set, the value + will be casted to the smallest possible DECIMAL type that can contain it. + + :value: + The value to bind for this parameter. This will be casted to a DECIMAL. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + :scale: + The maximum precision (total number of digits) of the number between 1 and 38. + :precision: + The number of digits to the right of the decimal point. + """ + self.value: decimal.Decimal = value + self.name = name + self.scale = scale + self.precision = precision + + if not self.valid_scale_and_precision(): + raise ValueError( + "DecimalParameter requires both or none of scale and precision to be set" + ) + + def valid_scale_and_precision(self): + if (self.scale is None and self.precision is None) or ( + isinstance(self.scale, int) and isinstance(self.precision, int) + ): + return True + else: + return False + + def _cast_expr(self): + if self.scale and self.precision: + return self.CAST_EXPR.format(self.scale, self.precision) + else: + return self.calculate_decimal_cast_string(self.value) + + def calculate_decimal_cast_string(self, input: decimal.Decimal) -> str: + """Returns the smallest SQL cast argument that can contain the passed decimal + + Example: + Input: Decimal("1234.5678") + Output: DECIMAL(8,4) + """ + + string_decimal = str(input) + + if string_decimal.startswith("0."): + # This decimal is less than 1 + overall = after = len(string_decimal) - 2 + elif "." not in string_decimal: + # This decimal has no fractional component + overall = len(string_decimal) + after = 0 + else: + # This decimal has both whole and fractional parts + parts = string_decimal.split(".") + parts_lengths = [len(i) for i in parts] + before, after = parts_lengths[:2] + overall = before + after + + return self.CAST_EXPR.format(overall, after) + + +def dbsql_parameter_from_int(value: int, name: Optional[str] = None): + """Returns IntegerParameter unless the passed int() requires a BIGINT. + + Note: TinyIntegerParameter is never inferred here because it is a rarely used type and clauses like LIMIT and OFFSET + cannot accept TINYINT bound parameter values. + """ + if -128 <= value <= 127: + # If DBR is ever updated to permit TINYINT values passed to LIMIT and OFFSET + # then we can change this line to return TinyIntParameter + return IntegerParameter(value=value, name=name) + elif -2147483648 <= value <= 2147483647: + return IntegerParameter(value=value, name=name) + else: + return BigIntegerParameter(value=value, name=name) + + +def dbsql_parameter_from_primitive( + value: TAllowedParameterValue, name: Optional[str] = None +) -> "TDbsqlParameter": + """Returns a DbsqlParameter subclass given an inferrable value + + This is a convenience function that can be used to create a DbsqlParameter subclass + without having to explicitly import a subclass of DbsqlParameter. + """ + + # This series of type checks are required for mypy not to raise + # havoc. We can't use TYPE_INFERRENCE_MAP because mypy doesn't trust + # its logic + + if type(value) is int: + return dbsql_parameter_from_int(value, name=name) + elif type(value) is str: + return StringParameter(value=value, name=name) + elif type(value) is float: + return FloatParameter(value=value, name=name) + elif type(value) is datetime.datetime: + return TimestampParameter(value=value, name=name) + elif type(value) is datetime.date: + return DateParameter(value=value, name=name) + elif type(value) is bool: + return BooleanParameter(value=value, name=name) + elif type(value) is decimal.Decimal: + return DecimalParameter(value=value, name=name) + elif value is None: + return VoidParameter(value=value, name=name) + + else: + raise NotSupportedError( + f"Could not infer parameter type from value: {value} - {type(value)} \n" + "Please specify the type explicitly." + ) + + +TDbsqlParameter = Union[ + IntegerParameter, + StringParameter, + BigIntegerParameter, + BooleanParameter, + DateParameter, + DoubleParameter, + FloatParameter, + VoidParameter, + SmallIntParameter, + TimestampParameter, + TimestampNTZParameter, + TinyIntParameter, + DecimalParameter, +] + + +TParameterSequence = Sequence[Union[TDbsqlParameter, TAllowedParameterValue]] +TParameterDict = Dict[str, TAllowedParameterValue] +TParameterCollection = Union[TParameterSequence, TParameterDict] + + +_all__ = [ + "IntegerParameter", + "StringParameter", + "BigIntegerParameter", + "BooleanParameter", + "DateParameter", + "DoubleParameter", + "FloatParameter", + "VoidParameter", + "SmallIntParameter", + "TimestampParameter", + "TimestampNTZParameter", + "TinyIntParameter", + "DecimalParameter", +] diff --git a/src/databricks/sql/parameters/py.typed b/src/databricks/sql/parameters/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/src/databricks/sql/py.typed b/src/databricks/sql/py.typed new file mode 100755 index 00000000..e69de29b diff --git a/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote b/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote index 5271f955..552b21e5 100755 --- a/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote +++ b/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.19.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -45,7 +45,6 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req)') print(' TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req)') print(' TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req)') - print(' TDBSqlGetLoadInformationResp GetLoadInformation(TDBSqlGetLoadInformationReq req)') print('') sys.exit(0) @@ -251,12 +250,6 @@ elif cmd == 'RenewDelegationToken': sys.exit(1) pp.pprint(client.RenewDelegationToken(eval(args[0]),)) -elif cmd == 'GetLoadInformation': - if len(args) != 1: - print('GetLoadInformation requires 1 args') - sys.exit(1) - pp.pprint(client.GetLoadInformation(eval(args[0]),)) - else: print('Unrecognized method %s' % cmd) sys.exit(1) diff --git a/src/databricks/sql/thrift_api/TCLIService/TCLIService.py b/src/databricks/sql/thrift_api/TCLIService/TCLIService.py index 6dde6702..071e78a9 100644 --- a/src/databricks/sql/thrift_api/TCLIService/TCLIService.py +++ b/src/databricks/sql/thrift_api/TCLIService/TCLIService.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.19.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -187,14 +187,6 @@ def RenewDelegationToken(self, req): """ pass - def GetLoadInformation(self, req): - """ - Parameters: - - req - - """ - pass - class Client(Iface): def __init__(self, iprot, oprot=None): @@ -875,38 +867,6 @@ def recv_RenewDelegationToken(self): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "RenewDelegationToken failed: unknown result") - def GetLoadInformation(self, req): - """ - Parameters: - - req - - """ - self.send_GetLoadInformation(req) - return self.recv_GetLoadInformation() - - def send_GetLoadInformation(self, req): - self._oprot.writeMessageBegin('GetLoadInformation', TMessageType.CALL, self._seqid) - args = GetLoadInformation_args() - args.req = req - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_GetLoadInformation(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = GetLoadInformation_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - raise TApplicationException(TApplicationException.MISSING_RESULT, "GetLoadInformation failed: unknown result") - class Processor(Iface, TProcessor): def __init__(self, handler): @@ -933,7 +893,6 @@ def __init__(self, handler): self._processMap["GetDelegationToken"] = Processor.process_GetDelegationToken self._processMap["CancelDelegationToken"] = Processor.process_CancelDelegationToken self._processMap["RenewDelegationToken"] = Processor.process_RenewDelegationToken - self._processMap["GetLoadInformation"] = Processor.process_GetLoadInformation self._on_message_begin = None def on_message_begin(self, func): @@ -1439,29 +1398,6 @@ def process_RenewDelegationToken(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() - def process_GetLoadInformation(self, seqid, iprot, oprot): - args = GetLoadInformation_args() - args.read(iprot) - iprot.readMessageEnd() - result = GetLoadInformation_result() - try: - result.success = self._handler.GetLoadInformation(args.req) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("GetLoadInformation", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - # HELPER FUNCTIONS AND STRUCTURES @@ -4088,130 +4024,5 @@ def __ne__(self, other): RenewDelegationToken_result.thrift_spec = ( (0, TType.STRUCT, 'success', [TRenewDelegationTokenResp, None], None, ), # 0 ) - - -class GetLoadInformation_args(object): - """ - Attributes: - - req - - """ - - - def __init__(self, req=None,): - self.req = req - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.req = TDBSqlGetLoadInformationReq() - self.req.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('GetLoadInformation_args') - if self.req is not None: - oprot.writeFieldBegin('req', TType.STRUCT, 1) - self.req.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(GetLoadInformation_args) -GetLoadInformation_args.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'req', [TDBSqlGetLoadInformationReq, None], None, ), # 1 -) - - -class GetLoadInformation_result(object): - """ - Attributes: - - success - - """ - - - def __init__(self, success=None,): - self.success = success - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = TDBSqlGetLoadInformationResp() - self.success.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('GetLoadInformation_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(GetLoadInformation_result) -GetLoadInformation_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [TDBSqlGetLoadInformationResp, None], None, ), # 0 -) fix_spec(all_structs) del all_structs diff --git a/src/databricks/sql/thrift_api/TCLIService/constants.py b/src/databricks/sql/thrift_api/TCLIService/constants.py index 66dfc322..2cdf2f41 100644 --- a/src/databricks/sql/thrift_api/TCLIService/constants.py +++ b/src/databricks/sql/thrift_api/TCLIService/constants.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.19.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # diff --git a/src/databricks/sql/thrift_api/TCLIService/ttypes.py b/src/databricks/sql/thrift_api/TCLIService/ttypes.py index 07ecfe48..16abbc2e 100644 --- a/src/databricks/sql/thrift_api/TCLIService/ttypes.py +++ b/src/databricks/sql/thrift_api/TCLIService/ttypes.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.19.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -36,6 +36,7 @@ class TProtocolVersion(object): SPARK_CLI_SERVICE_PROTOCOL_V5 = 42245 SPARK_CLI_SERVICE_PROTOCOL_V6 = 42246 SPARK_CLI_SERVICE_PROTOCOL_V7 = 42247 + SPARK_CLI_SERVICE_PROTOCOL_V8 = 42248 _VALUES_TO_NAMES = { -7: "__HIVE_JDBC_WORKAROUND", @@ -57,6 +58,7 @@ class TProtocolVersion(object): 42245: "SPARK_CLI_SERVICE_PROTOCOL_V5", 42246: "SPARK_CLI_SERVICE_PROTOCOL_V6", 42247: "SPARK_CLI_SERVICE_PROTOCOL_V7", + 42248: "SPARK_CLI_SERVICE_PROTOCOL_V8", } _NAMES_TO_VALUES = { @@ -79,6 +81,7 @@ class TProtocolVersion(object): "SPARK_CLI_SERVICE_PROTOCOL_V5": 42245, "SPARK_CLI_SERVICE_PROTOCOL_V6": 42246, "SPARK_CLI_SERVICE_PROTOCOL_V7": 42247, + "SPARK_CLI_SERVICE_PROTOCOL_V8": 42248, } @@ -178,6 +181,39 @@ class TSparkRowSetType(object): } +class TDBSqlCompressionCodec(object): + NONE = 0 + LZ4_FRAME = 1 + LZ4_BLOCK = 2 + + _VALUES_TO_NAMES = { + 0: "NONE", + 1: "LZ4_FRAME", + 2: "LZ4_BLOCK", + } + + _NAMES_TO_VALUES = { + "NONE": 0, + "LZ4_FRAME": 1, + "LZ4_BLOCK": 2, + } + + +class TDBSqlArrowLayout(object): + ARROW_BATCH = 0 + ARROW_STREAMING = 1 + + _VALUES_TO_NAMES = { + 0: "ARROW_BATCH", + 1: "ARROW_STREAMING", + } + + _NAMES_TO_VALUES = { + "ARROW_BATCH": 0, + "ARROW_STREAMING": 1, + } + + class TOperationIdempotencyType(object): UNKNOWN = 0 NON_IDEMPOTENT = 1 @@ -475,6 +511,21 @@ class TResultPersistenceMode(object): } +class TDBSqlCloseOperationReason(object): + NONE = 0 + COMMAND_INACTIVITY_TIMEOUT = 1 + + _VALUES_TO_NAMES = { + 0: "NONE", + 1: "COMMAND_INACTIVITY_TIMEOUT", + } + + _NAMES_TO_VALUES = { + "NONE": 0, + "COMMAND_INACTIVITY_TIMEOUT": 1, + } + + class TCacheLookupResult(object): CACHE_INELIGIBLE = 0 LOCAL_CACHE_HIT = 1 @@ -529,6 +580,18 @@ class TCloudFetchDisabledReason(object): } +class TDBSqlManifestFileFormat(object): + THRIFT_GET_RESULT_SET_METADATA_RESP = 0 + + _VALUES_TO_NAMES = { + 0: "THRIFT_GET_RESULT_SET_METADATA_RESP", + } + + _NAMES_TO_VALUES = { + "THRIFT_GET_RESULT_SET_METADATA_RESP": 0, + } + + class TFetchOrientation(object): FETCH_NEXT = 0 FETCH_PRIOR = 1 @@ -556,6 +619,27 @@ class TFetchOrientation(object): } +class TDBSqlFetchDisposition(object): + DISPOSITION_UNSPECIFIED = 0 + DISPOSITION_INLINE = 1 + DISPOSITION_EXTERNAL_LINKS = 2 + DISPOSITION_INTERNAL_DBFS = 3 + + _VALUES_TO_NAMES = { + 0: "DISPOSITION_UNSPECIFIED", + 1: "DISPOSITION_INLINE", + 2: "DISPOSITION_EXTERNAL_LINKS", + 3: "DISPOSITION_INTERNAL_DBFS", + } + + _NAMES_TO_VALUES = { + "DISPOSITION_UNSPECIFIED": 0, + "DISPOSITION_INLINE": 1, + "DISPOSITION_EXTERNAL_LINKS": 2, + "DISPOSITION_INTERNAL_DBFS": 3, + } + + class TJobExecutionStatus(object): IN_PROGRESS = 0 COMPLETE = 1 @@ -712,7 +796,7 @@ def __ne__(self, other): return not (self == other) -class TPrimitiveTypeEntry(object): +class TTAllowedParameterValueEntry(object): """ Attributes: - type @@ -754,7 +838,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('TPrimitiveTypeEntry') + oprot.writeStructBegin('TTAllowedParameterValueEntry') if self.type is not None: oprot.writeFieldBegin('type', TType.I32, 1) oprot.writeI32(self.type) @@ -1143,7 +1227,7 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRUCT: - self.primitiveEntry = TPrimitiveTypeEntry() + self.primitiveEntry = TTAllowedParameterValueEntry() self.primitiveEntry.read(iprot) else: iprot.skip(ftype) @@ -2841,6 +2925,270 @@ def __ne__(self, other): return not (self == other) +class TDBSqlJsonArrayFormat(object): + """ + Attributes: + - compressionCodec + + """ + + + def __init__(self, compressionCodec=None,): + self.compressionCodec = compressionCodec + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.compressionCodec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlJsonArrayFormat') + if self.compressionCodec is not None: + oprot.writeFieldBegin('compressionCodec', TType.I32, 1) + oprot.writeI32(self.compressionCodec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TDBSqlCsvFormat(object): + """ + Attributes: + - compressionCodec + + """ + + + def __init__(self, compressionCodec=None,): + self.compressionCodec = compressionCodec + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.compressionCodec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlCsvFormat') + if self.compressionCodec is not None: + oprot.writeFieldBegin('compressionCodec', TType.I32, 1) + oprot.writeI32(self.compressionCodec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TDBSqlArrowFormat(object): + """ + Attributes: + - arrowLayout + - compressionCodec + + """ + + + def __init__(self, arrowLayout=None, compressionCodec=None,): + self.arrowLayout = arrowLayout + self.compressionCodec = compressionCodec + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.arrowLayout = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.compressionCodec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlArrowFormat') + if self.arrowLayout is not None: + oprot.writeFieldBegin('arrowLayout', TType.I32, 1) + oprot.writeI32(self.arrowLayout) + oprot.writeFieldEnd() + if self.compressionCodec is not None: + oprot.writeFieldBegin('compressionCodec', TType.I32, 2) + oprot.writeI32(self.compressionCodec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TDBSqlResultFormat(object): + """ + Attributes: + - arrowFormat + - csvFormat + - jsonArrayFormat + + """ + + + def __init__(self, arrowFormat=None, csvFormat=None, jsonArrayFormat=None,): + self.arrowFormat = arrowFormat + self.csvFormat = csvFormat + self.jsonArrayFormat = jsonArrayFormat + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.arrowFormat = TDBSqlArrowFormat() + self.arrowFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.csvFormat = TDBSqlCsvFormat() + self.csvFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.jsonArrayFormat = TDBSqlJsonArrayFormat() + self.jsonArrayFormat.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlResultFormat') + if self.arrowFormat is not None: + oprot.writeFieldBegin('arrowFormat', TType.STRUCT, 1) + self.arrowFormat.write(oprot) + oprot.writeFieldEnd() + if self.csvFormat is not None: + oprot.writeFieldBegin('csvFormat', TType.STRUCT, 2) + self.csvFormat.write(oprot) + oprot.writeFieldEnd() + if self.jsonArrayFormat is not None: + oprot.writeFieldBegin('jsonArrayFormat', TType.STRUCT, 3) + self.jsonArrayFormat.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class TSparkArrowBatch(object): """ Attributes: @@ -2921,16 +3269,18 @@ class TSparkArrowResultLink(object): - startRowOffset - rowCount - bytesNum + - httpHeaders """ - def __init__(self, fileLink=None, expiryTime=None, startRowOffset=None, rowCount=None, bytesNum=None,): + def __init__(self, fileLink=None, expiryTime=None, startRowOffset=None, rowCount=None, bytesNum=None, httpHeaders=None,): self.fileLink = fileLink self.expiryTime = expiryTime self.startRowOffset = startRowOffset self.rowCount = rowCount self.bytesNum = bytesNum + self.httpHeaders = httpHeaders def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -2966,6 +3316,17 @@ def read(self, iprot): self.bytesNum = iprot.readI64() else: iprot.skip(ftype) + elif fid == 6: + if ftype == TType.MAP: + self.httpHeaders = {} + (_ktype105, _vtype106, _size104) = iprot.readMapBegin() + for _i108 in range(_size104): + _key109 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val110 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.httpHeaders[_key109] = _val110 + iprot.readMapEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -2996,6 +3357,14 @@ def write(self, oprot): oprot.writeFieldBegin('bytesNum', TType.I64, 5) oprot.writeI64(self.bytesNum) oprot.writeFieldEnd() + if self.httpHeaders is not None: + oprot.writeFieldBegin('httpHeaders', TType.MAP, 6) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.httpHeaders)) + for kiter111, viter112 in self.httpHeaders.items(): + oprot.writeString(kiter111.encode('utf-8') if sys.version_info[0] == 2 else kiter111) + oprot.writeString(viter112.encode('utf-8') if sys.version_info[0] == 2 else viter112) + oprot.writeMapEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -3032,16 +3401,22 @@ class TDBSqlCloudResultFile(object): - rowCount - uncompressedBytes - compressedBytes + - fileLink + - linkExpiryTime + - httpHeaders """ - def __init__(self, filePath=None, startRowOffset=None, rowCount=None, uncompressedBytes=None, compressedBytes=None,): + def __init__(self, filePath=None, startRowOffset=None, rowCount=None, uncompressedBytes=None, compressedBytes=None, fileLink=None, linkExpiryTime=None, httpHeaders=None,): self.filePath = filePath self.startRowOffset = startRowOffset self.rowCount = rowCount self.uncompressedBytes = uncompressedBytes self.compressedBytes = compressedBytes + self.fileLink = fileLink + self.linkExpiryTime = linkExpiryTime + self.httpHeaders = httpHeaders def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -3077,6 +3452,27 @@ def read(self, iprot): self.compressedBytes = iprot.readI64() else: iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.fileLink = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.linkExpiryTime = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.MAP: + self.httpHeaders = {} + (_ktype114, _vtype115, _size113) = iprot.readMapBegin() + for _i117 in range(_size113): + _key118 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val119 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.httpHeaders[_key118] = _val119 + iprot.readMapEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -3107,20 +3503,26 @@ def write(self, oprot): oprot.writeFieldBegin('compressedBytes', TType.I64, 5) oprot.writeI64(self.compressedBytes) oprot.writeFieldEnd() + if self.fileLink is not None: + oprot.writeFieldBegin('fileLink', TType.STRING, 6) + oprot.writeString(self.fileLink.encode('utf-8') if sys.version_info[0] == 2 else self.fileLink) + oprot.writeFieldEnd() + if self.linkExpiryTime is not None: + oprot.writeFieldBegin('linkExpiryTime', TType.I64, 7) + oprot.writeI64(self.linkExpiryTime) + oprot.writeFieldEnd() + if self.httpHeaders is not None: + oprot.writeFieldBegin('httpHeaders', TType.MAP, 8) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.httpHeaders)) + for kiter120, viter121 in self.httpHeaders.items(): + oprot.writeString(kiter120.encode('utf-8') if sys.version_info[0] == 2 else kiter120) + oprot.writeString(viter121.encode('utf-8') if sys.version_info[0] == 2 else viter121) + oprot.writeMapEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): - if self.filePath is None: - raise TProtocolException(message='Required field filePath is unset!') - if self.startRowOffset is None: - raise TProtocolException(message='Required field startRowOffset is unset!') - if self.rowCount is None: - raise TProtocolException(message='Required field rowCount is unset!') - if self.uncompressedBytes is None: - raise TProtocolException(message='Required field uncompressedBytes is unset!') - if self.compressedBytes is None: - raise TProtocolException(message='Required field compressedBytes is unset!') return def __repr__(self): @@ -3145,11 +3547,12 @@ class TRowSet(object): - columnCount - arrowBatches - resultLinks + - cloudFetchResults """ - def __init__(self, startRowOffset=None, rows=None, columns=None, binaryColumns=None, columnCount=None, arrowBatches=None, resultLinks=None,): + def __init__(self, startRowOffset=None, rows=None, columns=None, binaryColumns=None, columnCount=None, arrowBatches=None, resultLinks=None, cloudFetchResults=None,): self.startRowOffset = startRowOffset self.rows = rows self.columns = columns @@ -3157,6 +3560,7 @@ def __init__(self, startRowOffset=None, rows=None, columns=None, binaryColumns=N self.columnCount = columnCount self.arrowBatches = arrowBatches self.resultLinks = resultLinks + self.cloudFetchResults = cloudFetchResults def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -3175,22 +3579,22 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.rows = [] - (_etype107, _size104) = iprot.readListBegin() - for _i108 in range(_size104): - _elem109 = TRow() - _elem109.read(iprot) - self.rows.append(_elem109) + (_etype125, _size122) = iprot.readListBegin() + for _i126 in range(_size122): + _elem127 = TRow() + _elem127.read(iprot) + self.rows.append(_elem127) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.columns = [] - (_etype113, _size110) = iprot.readListBegin() - for _i114 in range(_size110): - _elem115 = TColumn() - _elem115.read(iprot) - self.columns.append(_elem115) + (_etype131, _size128) = iprot.readListBegin() + for _i132 in range(_size128): + _elem133 = TColumn() + _elem133.read(iprot) + self.columns.append(_elem133) iprot.readListEnd() else: iprot.skip(ftype) @@ -3207,22 +3611,33 @@ def read(self, iprot): elif fid == 1281: if ftype == TType.LIST: self.arrowBatches = [] - (_etype119, _size116) = iprot.readListBegin() - for _i120 in range(_size116): - _elem121 = TSparkArrowBatch() - _elem121.read(iprot) - self.arrowBatches.append(_elem121) + (_etype137, _size134) = iprot.readListBegin() + for _i138 in range(_size134): + _elem139 = TSparkArrowBatch() + _elem139.read(iprot) + self.arrowBatches.append(_elem139) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 1282: if ftype == TType.LIST: self.resultLinks = [] - (_etype125, _size122) = iprot.readListBegin() - for _i126 in range(_size122): - _elem127 = TSparkArrowResultLink() - _elem127.read(iprot) - self.resultLinks.append(_elem127) + (_etype143, _size140) = iprot.readListBegin() + for _i144 in range(_size140): + _elem145 = TSparkArrowResultLink() + _elem145.read(iprot) + self.resultLinks.append(_elem145) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3329: + if ftype == TType.LIST: + self.cloudFetchResults = [] + (_etype149, _size146) = iprot.readListBegin() + for _i150 in range(_size146): + _elem151 = TDBSqlCloudResultFile() + _elem151.read(iprot) + self.cloudFetchResults.append(_elem151) iprot.readListEnd() else: iprot.skip(ftype) @@ -3243,15 +3658,15 @@ def write(self, oprot): if self.rows is not None: oprot.writeFieldBegin('rows', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.rows)) - for iter128 in self.rows: - iter128.write(oprot) + for iter152 in self.rows: + iter152.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.columns is not None: oprot.writeFieldBegin('columns', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.columns)) - for iter129 in self.columns: - iter129.write(oprot) + for iter153 in self.columns: + iter153.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.binaryColumns is not None: @@ -3265,15 +3680,22 @@ def write(self, oprot): if self.arrowBatches is not None: oprot.writeFieldBegin('arrowBatches', TType.LIST, 1281) oprot.writeListBegin(TType.STRUCT, len(self.arrowBatches)) - for iter130 in self.arrowBatches: - iter130.write(oprot) + for iter154 in self.arrowBatches: + iter154.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.resultLinks is not None: oprot.writeFieldBegin('resultLinks', TType.LIST, 1282) oprot.writeListBegin(TType.STRUCT, len(self.resultLinks)) - for iter131 in self.resultLinks: - iter131.write(oprot) + for iter155 in self.resultLinks: + iter155.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.cloudFetchResults is not None: + oprot.writeFieldBegin('cloudFetchResults', TType.LIST, 3329) + oprot.writeListBegin(TType.STRUCT, len(self.cloudFetchResults)) + for iter156 in self.cloudFetchResults: + iter156.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -3337,11 +3759,11 @@ def read(self, iprot): elif fid == 3: if ftype == TType.MAP: self.properties = {} - (_ktype133, _vtype134, _size132) = iprot.readMapBegin() - for _i136 in range(_size132): - _key137 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val138 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.properties[_key137] = _val138 + (_ktype158, _vtype159, _size157) = iprot.readMapBegin() + for _i161 in range(_size157): + _key162 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val163 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.properties[_key162] = _val163 iprot.readMapEnd() else: iprot.skip(ftype) @@ -3371,9 +3793,9 @@ def write(self, oprot): if self.properties is not None: oprot.writeFieldBegin('properties', TType.MAP, 3) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.properties)) - for kiter139, viter140 in self.properties.items(): - oprot.writeString(kiter139.encode('utf-8') if sys.version_info[0] == 2 else kiter139) - oprot.writeString(viter140.encode('utf-8') if sys.version_info[0] == 2 else viter140) + for kiter164, viter165 in self.properties.items(): + oprot.writeString(kiter164.encode('utf-8') if sys.version_info[0] == 2 else kiter164) + oprot.writeString(viter165.encode('utf-8') if sys.version_info[0] == 2 else viter165) oprot.writeMapEnd() oprot.writeFieldEnd() if self.viewSchema is not None: @@ -3725,22 +4147,22 @@ def read(self, iprot): if fid == 1: if ftype == TType.MAP: self.confs = {} - (_ktype142, _vtype143, _size141) = iprot.readMapBegin() - for _i145 in range(_size141): - _key146 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val147 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.confs[_key146] = _val147 + (_ktype167, _vtype168, _size166) = iprot.readMapBegin() + for _i170 in range(_size166): + _key171 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val172 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.confs[_key171] = _val172 iprot.readMapEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.tempViews = [] - (_etype151, _size148) = iprot.readListBegin() - for _i152 in range(_size148): - _elem153 = TDBSqlTempView() - _elem153.read(iprot) - self.tempViews.append(_elem153) + (_etype176, _size173) = iprot.readListBegin() + for _i177 in range(_size173): + _elem178 = TDBSqlTempView() + _elem178.read(iprot) + self.tempViews.append(_elem178) iprot.readListEnd() else: iprot.skip(ftype) @@ -3763,23 +4185,23 @@ def read(self, iprot): elif fid == 6: if ftype == TType.LIST: self.expressionsInfos = [] - (_etype157, _size154) = iprot.readListBegin() - for _i158 in range(_size154): - _elem159 = TExpressionInfo() - _elem159.read(iprot) - self.expressionsInfos.append(_elem159) + (_etype182, _size179) = iprot.readListBegin() + for _i183 in range(_size179): + _elem184 = TExpressionInfo() + _elem184.read(iprot) + self.expressionsInfos.append(_elem184) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 7: if ftype == TType.MAP: self.internalConfs = {} - (_ktype161, _vtype162, _size160) = iprot.readMapBegin() - for _i164 in range(_size160): - _key165 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val166 = TDBSqlConfValue() - _val166.read(iprot) - self.internalConfs[_key165] = _val166 + (_ktype186, _vtype187, _size185) = iprot.readMapBegin() + for _i189 in range(_size185): + _key190 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val191 = TDBSqlConfValue() + _val191.read(iprot) + self.internalConfs[_key190] = _val191 iprot.readMapEnd() else: iprot.skip(ftype) @@ -3796,16 +4218,16 @@ def write(self, oprot): if self.confs is not None: oprot.writeFieldBegin('confs', TType.MAP, 1) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.confs)) - for kiter167, viter168 in self.confs.items(): - oprot.writeString(kiter167.encode('utf-8') if sys.version_info[0] == 2 else kiter167) - oprot.writeString(viter168.encode('utf-8') if sys.version_info[0] == 2 else viter168) + for kiter192, viter193 in self.confs.items(): + oprot.writeString(kiter192.encode('utf-8') if sys.version_info[0] == 2 else kiter192) + oprot.writeString(viter193.encode('utf-8') if sys.version_info[0] == 2 else viter193) oprot.writeMapEnd() oprot.writeFieldEnd() if self.tempViews is not None: oprot.writeFieldBegin('tempViews', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.tempViews)) - for iter169 in self.tempViews: - iter169.write(oprot) + for iter194 in self.tempViews: + iter194.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.currentDatabase is not None: @@ -3823,16 +4245,16 @@ def write(self, oprot): if self.expressionsInfos is not None: oprot.writeFieldBegin('expressionsInfos', TType.LIST, 6) oprot.writeListBegin(TType.STRUCT, len(self.expressionsInfos)) - for iter170 in self.expressionsInfos: - iter170.write(oprot) + for iter195 in self.expressionsInfos: + iter195.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.internalConfs is not None: oprot.writeFieldBegin('internalConfs', TType.MAP, 7) oprot.writeMapBegin(TType.STRING, TType.STRUCT, len(self.internalConfs)) - for kiter171, viter172 in self.internalConfs.items(): - oprot.writeString(kiter171.encode('utf-8') if sys.version_info[0] == 2 else kiter171) - viter172.write(oprot) + for kiter196, viter197 in self.internalConfs.items(): + oprot.writeString(kiter196.encode('utf-8') if sys.version_info[0] == 2 else kiter196) + viter197.write(oprot) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -3862,18 +4284,20 @@ class TStatus(object): - errorCode - errorMessage - displayMessage + - errorDetailsJson - responseValidation """ - def __init__(self, statusCode=None, infoMessages=None, sqlState=None, errorCode=None, errorMessage=None, displayMessage=None, responseValidation=None,): + def __init__(self, statusCode=None, infoMessages=None, sqlState=None, errorCode=None, errorMessage=None, displayMessage=None, errorDetailsJson=None, responseValidation=None,): self.statusCode = statusCode self.infoMessages = infoMessages self.sqlState = sqlState self.errorCode = errorCode self.errorMessage = errorMessage self.displayMessage = displayMessage + self.errorDetailsJson = errorDetailsJson self.responseValidation = responseValidation def read(self, iprot): @@ -3893,10 +4317,10 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.infoMessages = [] - (_etype176, _size173) = iprot.readListBegin() - for _i177 in range(_size173): - _elem178 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.infoMessages.append(_elem178) + (_etype201, _size198) = iprot.readListBegin() + for _i202 in range(_size198): + _elem203 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.infoMessages.append(_elem203) iprot.readListEnd() else: iprot.skip(ftype) @@ -3920,6 +4344,11 @@ def read(self, iprot): self.displayMessage = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) + elif fid == 1281: + if ftype == TType.STRING: + self.errorDetailsJson = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) elif fid == 3329: if ftype == TType.STRING: self.responseValidation = iprot.readBinary() @@ -3942,8 +4371,8 @@ def write(self, oprot): if self.infoMessages is not None: oprot.writeFieldBegin('infoMessages', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.infoMessages)) - for iter179 in self.infoMessages: - oprot.writeString(iter179.encode('utf-8') if sys.version_info[0] == 2 else iter179) + for iter204 in self.infoMessages: + oprot.writeString(iter204.encode('utf-8') if sys.version_info[0] == 2 else iter204) oprot.writeListEnd() oprot.writeFieldEnd() if self.sqlState is not None: @@ -3962,6 +4391,10 @@ def write(self, oprot): oprot.writeFieldBegin('displayMessage', TType.STRING, 6) oprot.writeString(self.displayMessage.encode('utf-8') if sys.version_info[0] == 2 else self.displayMessage) oprot.writeFieldEnd() + if self.errorDetailsJson is not None: + oprot.writeFieldBegin('errorDetailsJson', TType.STRING, 1281) + oprot.writeString(self.errorDetailsJson.encode('utf-8') if sys.version_info[0] == 2 else self.errorDetailsJson) + oprot.writeFieldEnd() if self.responseValidation is not None: oprot.writeFieldBegin('responseValidation', TType.STRING, 3329) oprot.writeBinary(self.responseValidation) @@ -4361,21 +4794,21 @@ def read(self, iprot): elif fid == 4: if ftype == TType.MAP: self.configuration = {} - (_ktype181, _vtype182, _size180) = iprot.readMapBegin() - for _i184 in range(_size180): - _key185 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val186 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.configuration[_key185] = _val186 + (_ktype206, _vtype207, _size205) = iprot.readMapBegin() + for _i209 in range(_size205): + _key210 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val211 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.configuration[_key210] = _val211 iprot.readMapEnd() else: iprot.skip(ftype) elif fid == 1281: if ftype == TType.LIST: self.getInfos = [] - (_etype190, _size187) = iprot.readListBegin() - for _i191 in range(_size187): - _elem192 = iprot.readI32() - self.getInfos.append(_elem192) + (_etype215, _size212) = iprot.readListBegin() + for _i216 in range(_size212): + _elem217 = iprot.readI32() + self.getInfos.append(_elem217) iprot.readListEnd() else: iprot.skip(ftype) @@ -4387,11 +4820,11 @@ def read(self, iprot): elif fid == 1283: if ftype == TType.MAP: self.connectionProperties = {} - (_ktype194, _vtype195, _size193) = iprot.readMapBegin() - for _i197 in range(_size193): - _key198 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val199 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.connectionProperties[_key198] = _val199 + (_ktype219, _vtype220, _size218) = iprot.readMapBegin() + for _i222 in range(_size218): + _key223 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val224 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.connectionProperties[_key223] = _val224 iprot.readMapEnd() else: iprot.skip(ftype) @@ -4437,16 +4870,16 @@ def write(self, oprot): if self.configuration is not None: oprot.writeFieldBegin('configuration', TType.MAP, 4) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.configuration)) - for kiter200, viter201 in self.configuration.items(): - oprot.writeString(kiter200.encode('utf-8') if sys.version_info[0] == 2 else kiter200) - oprot.writeString(viter201.encode('utf-8') if sys.version_info[0] == 2 else viter201) + for kiter225, viter226 in self.configuration.items(): + oprot.writeString(kiter225.encode('utf-8') if sys.version_info[0] == 2 else kiter225) + oprot.writeString(viter226.encode('utf-8') if sys.version_info[0] == 2 else viter226) oprot.writeMapEnd() oprot.writeFieldEnd() if self.getInfos is not None: oprot.writeFieldBegin('getInfos', TType.LIST, 1281) oprot.writeListBegin(TType.I32, len(self.getInfos)) - for iter202 in self.getInfos: - oprot.writeI32(iter202) + for iter227 in self.getInfos: + oprot.writeI32(iter227) oprot.writeListEnd() oprot.writeFieldEnd() if self.client_protocol_i64 is not None: @@ -4456,9 +4889,9 @@ def write(self, oprot): if self.connectionProperties is not None: oprot.writeFieldBegin('connectionProperties', TType.MAP, 1283) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.connectionProperties)) - for kiter203, viter204 in self.connectionProperties.items(): - oprot.writeString(kiter203.encode('utf-8') if sys.version_info[0] == 2 else kiter203) - oprot.writeString(viter204.encode('utf-8') if sys.version_info[0] == 2 else viter204) + for kiter228, viter229 in self.connectionProperties.items(): + oprot.writeString(kiter228.encode('utf-8') if sys.version_info[0] == 2 else kiter228) + oprot.writeString(viter229.encode('utf-8') if sys.version_info[0] == 2 else viter229) oprot.writeMapEnd() oprot.writeFieldEnd() if self.initialNamespace is not None: @@ -4543,11 +4976,11 @@ def read(self, iprot): elif fid == 4: if ftype == TType.MAP: self.configuration = {} - (_ktype206, _vtype207, _size205) = iprot.readMapBegin() - for _i209 in range(_size205): - _key210 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val211 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.configuration[_key210] = _val211 + (_ktype231, _vtype232, _size230) = iprot.readMapBegin() + for _i234 in range(_size230): + _key235 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val236 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.configuration[_key235] = _val236 iprot.readMapEnd() else: iprot.skip(ftype) @@ -4565,11 +4998,11 @@ def read(self, iprot): elif fid == 1281: if ftype == TType.LIST: self.getInfos = [] - (_etype215, _size212) = iprot.readListBegin() - for _i216 in range(_size212): - _elem217 = TGetInfoValue() - _elem217.read(iprot) - self.getInfos.append(_elem217) + (_etype240, _size237) = iprot.readListBegin() + for _i241 in range(_size237): + _elem242 = TGetInfoValue() + _elem242.read(iprot) + self.getInfos.append(_elem242) iprot.readListEnd() else: iprot.skip(ftype) @@ -4598,16 +5031,16 @@ def write(self, oprot): if self.configuration is not None: oprot.writeFieldBegin('configuration', TType.MAP, 4) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.configuration)) - for kiter218, viter219 in self.configuration.items(): - oprot.writeString(kiter218.encode('utf-8') if sys.version_info[0] == 2 else kiter218) - oprot.writeString(viter219.encode('utf-8') if sys.version_info[0] == 2 else viter219) + for kiter243, viter244 in self.configuration.items(): + oprot.writeString(kiter243.encode('utf-8') if sys.version_info[0] == 2 else kiter243) + oprot.writeString(viter244.encode('utf-8') if sys.version_info[0] == 2 else viter244) oprot.writeMapEnd() oprot.writeFieldEnd() if self.getInfos is not None: oprot.writeFieldBegin('getInfos', TType.LIST, 1281) oprot.writeListBegin(TType.STRUCT, len(self.getInfos)) - for iter220 in self.getInfos: - iter220.write(oprot) + for iter245 in self.getInfos: + iter245.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.initialNamespace is not None: @@ -5202,15 +5635,17 @@ class TSparkArrowTypes(object): - decimalAsArrow - complexTypesAsArrow - intervalTypesAsArrow + - nullTypeAsArrow """ - def __init__(self, timestampAsArrow=None, decimalAsArrow=None, complexTypesAsArrow=None, intervalTypesAsArrow=None,): + def __init__(self, timestampAsArrow=None, decimalAsArrow=None, complexTypesAsArrow=None, intervalTypesAsArrow=None, nullTypeAsArrow=None,): self.timestampAsArrow = timestampAsArrow self.decimalAsArrow = decimalAsArrow self.complexTypesAsArrow = complexTypesAsArrow self.intervalTypesAsArrow = intervalTypesAsArrow + self.nullTypeAsArrow = nullTypeAsArrow def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -5241,6 +5676,11 @@ def read(self, iprot): self.intervalTypesAsArrow = iprot.readBool() else: iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.nullTypeAsArrow = iprot.readBool() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5267,6 +5707,10 @@ def write(self, oprot): oprot.writeFieldBegin('intervalTypesAsArrow', TType.BOOL, 4) oprot.writeBool(self.intervalTypesAsArrow) oprot.writeFieldEnd() + if self.nullTypeAsArrow is not None: + oprot.writeFieldBegin('nullTypeAsArrow', TType.BOOL, 5) + oprot.writeBool(self.nullTypeAsArrow) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -5300,6 +5744,9 @@ class TExecuteStatementReq(object): - maxBytesPerFile - useArrowNativeTypes - resultRowLimit + - parameters + - maxBytesPerBatch + - statementConf - operationId - sessionConf - rejectHighCostQueries @@ -5308,11 +5755,24 @@ class TExecuteStatementReq(object): - requestValidation - resultPersistenceMode - trimArrowBatchesToLimit + - fetchDisposition + - enforceResultPersistenceMode + - statementList + - persistResultManifest + - resultRetentionSeconds + - resultByteLimit + - resultDataFormat + - originatingClientIdentity + - preferSingleFileResult + - preferDriverOnlyUpload + - enforceEmbeddedSchemaCorrectness + - idempotencyToken + - throwErrorOnByteLimitTruncation """ - def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsync=False, getDirectResults=None, queryTimeout=0, canReadArrowResult=None, canDownloadResult=None, canDecompressLZ4Result=None, maxBytesPerFile=None, useArrowNativeTypes=None, resultRowLimit=None, operationId=None, sessionConf=None, rejectHighCostQueries=None, estimatedCost=None, executionVersion=None, requestValidation=None, resultPersistenceMode=None, trimArrowBatchesToLimit=None,): + def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsync=False, getDirectResults=None, queryTimeout=0, canReadArrowResult=None, canDownloadResult=None, canDecompressLZ4Result=None, maxBytesPerFile=None, useArrowNativeTypes=None, resultRowLimit=None, parameters=None, maxBytesPerBatch=None, statementConf=None, operationId=None, sessionConf=None, rejectHighCostQueries=None, estimatedCost=None, executionVersion=None, requestValidation=None, resultPersistenceMode=None, trimArrowBatchesToLimit=None, fetchDisposition=None, enforceResultPersistenceMode=None, statementList=None, persistResultManifest=None, resultRetentionSeconds=None, resultByteLimit=None, resultDataFormat=None, originatingClientIdentity=None, preferSingleFileResult=None, preferDriverOnlyUpload=None, enforceEmbeddedSchemaCorrectness=False, idempotencyToken=None, throwErrorOnByteLimitTruncation=None,): self.sessionHandle = sessionHandle self.statement = statement self.confOverlay = confOverlay @@ -5325,6 +5785,9 @@ def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsyn self.maxBytesPerFile = maxBytesPerFile self.useArrowNativeTypes = useArrowNativeTypes self.resultRowLimit = resultRowLimit + self.parameters = parameters + self.maxBytesPerBatch = maxBytesPerBatch + self.statementConf = statementConf self.operationId = operationId self.sessionConf = sessionConf self.rejectHighCostQueries = rejectHighCostQueries @@ -5333,6 +5796,19 @@ def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsyn self.requestValidation = requestValidation self.resultPersistenceMode = resultPersistenceMode self.trimArrowBatchesToLimit = trimArrowBatchesToLimit + self.fetchDisposition = fetchDisposition + self.enforceResultPersistenceMode = enforceResultPersistenceMode + self.statementList = statementList + self.persistResultManifest = persistResultManifest + self.resultRetentionSeconds = resultRetentionSeconds + self.resultByteLimit = resultByteLimit + self.resultDataFormat = resultDataFormat + self.originatingClientIdentity = originatingClientIdentity + self.preferSingleFileResult = preferSingleFileResult + self.preferDriverOnlyUpload = preferDriverOnlyUpload + self.enforceEmbeddedSchemaCorrectness = enforceEmbeddedSchemaCorrectness + self.idempotencyToken = idempotencyToken + self.throwErrorOnByteLimitTruncation = throwErrorOnByteLimitTruncation def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -5357,11 +5833,11 @@ def read(self, iprot): elif fid == 3: if ftype == TType.MAP: self.confOverlay = {} - (_ktype222, _vtype223, _size221) = iprot.readMapBegin() - for _i225 in range(_size221): - _key226 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val227 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.confOverlay[_key226] = _val227 + (_ktype247, _vtype248, _size246) = iprot.readMapBegin() + for _i250 in range(_size246): + _key251 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val252 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.confOverlay[_key251] = _val252 iprot.readMapEnd() else: iprot.skip(ftype) @@ -5412,6 +5888,28 @@ def read(self, iprot): self.resultRowLimit = iprot.readI64() else: iprot.skip(ftype) + elif fid == 1288: + if ftype == TType.LIST: + self.parameters = [] + (_etype256, _size253) = iprot.readListBegin() + for _i257 in range(_size253): + _elem258 = TSparkParameter() + _elem258.read(iprot) + self.parameters.append(_elem258) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1289: + if ftype == TType.I64: + self.maxBytesPerBatch = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 1296: + if ftype == TType.STRUCT: + self.statementConf = TStatementConf() + self.statementConf.read(iprot) + else: + iprot.skip(ftype) elif fid == 3329: if ftype == TType.STRUCT: self.operationId = THandleIdentifier() @@ -5454,6 +5952,78 @@ def read(self, iprot): self.trimArrowBatchesToLimit = iprot.readBool() else: iprot.skip(ftype) + elif fid == 3337: + if ftype == TType.I32: + self.fetchDisposition = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3344: + if ftype == TType.BOOL: + self.enforceResultPersistenceMode = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3345: + if ftype == TType.LIST: + self.statementList = [] + (_etype262, _size259) = iprot.readListBegin() + for _i263 in range(_size259): + _elem264 = TDBSqlStatement() + _elem264.read(iprot) + self.statementList.append(_elem264) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3346: + if ftype == TType.BOOL: + self.persistResultManifest = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3347: + if ftype == TType.I64: + self.resultRetentionSeconds = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3348: + if ftype == TType.I64: + self.resultByteLimit = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3349: + if ftype == TType.STRUCT: + self.resultDataFormat = TDBSqlResultFormat() + self.resultDataFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3350: + if ftype == TType.STRING: + self.originatingClientIdentity = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3351: + if ftype == TType.BOOL: + self.preferSingleFileResult = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3352: + if ftype == TType.BOOL: + self.preferDriverOnlyUpload = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3353: + if ftype == TType.BOOL: + self.enforceEmbeddedSchemaCorrectness = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3360: + if ftype == TType.STRING: + self.idempotencyToken = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3361: + if ftype == TType.BOOL: + self.throwErrorOnByteLimitTruncation = iprot.readBool() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5475,9 +6045,9 @@ def write(self, oprot): if self.confOverlay is not None: oprot.writeFieldBegin('confOverlay', TType.MAP, 3) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.confOverlay)) - for kiter228, viter229 in self.confOverlay.items(): - oprot.writeString(kiter228.encode('utf-8') if sys.version_info[0] == 2 else kiter228) - oprot.writeString(viter229.encode('utf-8') if sys.version_info[0] == 2 else viter229) + for kiter265, viter266 in self.confOverlay.items(): + oprot.writeString(kiter265.encode('utf-8') if sys.version_info[0] == 2 else kiter265) + oprot.writeString(viter266.encode('utf-8') if sys.version_info[0] == 2 else viter266) oprot.writeMapEnd() oprot.writeFieldEnd() if self.runAsync is not None: @@ -5516,6 +6086,21 @@ def write(self, oprot): oprot.writeFieldBegin('resultRowLimit', TType.I64, 1287) oprot.writeI64(self.resultRowLimit) oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin('parameters', TType.LIST, 1288) + oprot.writeListBegin(TType.STRUCT, len(self.parameters)) + for iter267 in self.parameters: + iter267.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.maxBytesPerBatch is not None: + oprot.writeFieldBegin('maxBytesPerBatch', TType.I64, 1289) + oprot.writeI64(self.maxBytesPerBatch) + oprot.writeFieldEnd() + if self.statementConf is not None: + oprot.writeFieldBegin('statementConf', TType.STRUCT, 1296) + self.statementConf.write(oprot) + oprot.writeFieldEnd() if self.operationId is not None: oprot.writeFieldBegin('operationId', TType.STRUCT, 3329) self.operationId.write(oprot) @@ -5548,6 +6133,61 @@ def write(self, oprot): oprot.writeFieldBegin('trimArrowBatchesToLimit', TType.BOOL, 3336) oprot.writeBool(self.trimArrowBatchesToLimit) oprot.writeFieldEnd() + if self.fetchDisposition is not None: + oprot.writeFieldBegin('fetchDisposition', TType.I32, 3337) + oprot.writeI32(self.fetchDisposition) + oprot.writeFieldEnd() + if self.enforceResultPersistenceMode is not None: + oprot.writeFieldBegin('enforceResultPersistenceMode', TType.BOOL, 3344) + oprot.writeBool(self.enforceResultPersistenceMode) + oprot.writeFieldEnd() + if self.statementList is not None: + oprot.writeFieldBegin('statementList', TType.LIST, 3345) + oprot.writeListBegin(TType.STRUCT, len(self.statementList)) + for iter268 in self.statementList: + iter268.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.persistResultManifest is not None: + oprot.writeFieldBegin('persistResultManifest', TType.BOOL, 3346) + oprot.writeBool(self.persistResultManifest) + oprot.writeFieldEnd() + if self.resultRetentionSeconds is not None: + oprot.writeFieldBegin('resultRetentionSeconds', TType.I64, 3347) + oprot.writeI64(self.resultRetentionSeconds) + oprot.writeFieldEnd() + if self.resultByteLimit is not None: + oprot.writeFieldBegin('resultByteLimit', TType.I64, 3348) + oprot.writeI64(self.resultByteLimit) + oprot.writeFieldEnd() + if self.resultDataFormat is not None: + oprot.writeFieldBegin('resultDataFormat', TType.STRUCT, 3349) + self.resultDataFormat.write(oprot) + oprot.writeFieldEnd() + if self.originatingClientIdentity is not None: + oprot.writeFieldBegin('originatingClientIdentity', TType.STRING, 3350) + oprot.writeString(self.originatingClientIdentity.encode('utf-8') if sys.version_info[0] == 2 else self.originatingClientIdentity) + oprot.writeFieldEnd() + if self.preferSingleFileResult is not None: + oprot.writeFieldBegin('preferSingleFileResult', TType.BOOL, 3351) + oprot.writeBool(self.preferSingleFileResult) + oprot.writeFieldEnd() + if self.preferDriverOnlyUpload is not None: + oprot.writeFieldBegin('preferDriverOnlyUpload', TType.BOOL, 3352) + oprot.writeBool(self.preferDriverOnlyUpload) + oprot.writeFieldEnd() + if self.enforceEmbeddedSchemaCorrectness is not None: + oprot.writeFieldBegin('enforceEmbeddedSchemaCorrectness', TType.BOOL, 3353) + oprot.writeBool(self.enforceEmbeddedSchemaCorrectness) + oprot.writeFieldEnd() + if self.idempotencyToken is not None: + oprot.writeFieldBegin('idempotencyToken', TType.STRING, 3360) + oprot.writeString(self.idempotencyToken.encode('utf-8') if sys.version_info[0] == 2 else self.idempotencyToken) + oprot.writeFieldEnd() + if self.throwErrorOnByteLimitTruncation is not None: + oprot.writeFieldBegin('throwErrorOnByteLimitTruncation', TType.BOOL, 3361) + oprot.writeBool(self.throwErrorOnByteLimitTruncation) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -5570,6 +6210,324 @@ def __ne__(self, other): return not (self == other) +class TDBSqlStatement(object): + """ + Attributes: + - statement + + """ + + + def __init__(self, statement=None,): + self.statement = statement + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.statement = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlStatement') + if self.statement is not None: + oprot.writeFieldBegin('statement', TType.STRING, 1) + oprot.writeString(self.statement.encode('utf-8') if sys.version_info[0] == 2 else self.statement) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TSparkParameterValue(object): + """ + Attributes: + - stringValue + - doubleValue + - booleanValue + + """ + + + def __init__(self, stringValue=None, doubleValue=None, booleanValue=None,): + self.stringValue = stringValue + self.doubleValue = doubleValue + self.booleanValue = booleanValue + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.stringValue = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.DOUBLE: + self.doubleValue = iprot.readDouble() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.booleanValue = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TSparkParameterValue') + if self.stringValue is not None: + oprot.writeFieldBegin('stringValue', TType.STRING, 1) + oprot.writeString(self.stringValue.encode('utf-8') if sys.version_info[0] == 2 else self.stringValue) + oprot.writeFieldEnd() + if self.doubleValue is not None: + oprot.writeFieldBegin('doubleValue', TType.DOUBLE, 2) + oprot.writeDouble(self.doubleValue) + oprot.writeFieldEnd() + if self.booleanValue is not None: + oprot.writeFieldBegin('booleanValue', TType.BOOL, 3) + oprot.writeBool(self.booleanValue) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TSparkParameter(object): + """ + Attributes: + - ordinal + - name + - type + - value + + """ + + + def __init__(self, ordinal=None, name=None, type=None, value=None,): + self.ordinal = ordinal + self.name = name + self.type = type + self.value = value + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.ordinal = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.type = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.value = TSparkParameterValue() + self.value.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TSparkParameter') + if self.ordinal is not None: + oprot.writeFieldBegin('ordinal', TType.I32, 1) + oprot.writeI32(self.ordinal) + oprot.writeFieldEnd() + if self.name is not None: + oprot.writeFieldBegin('name', TType.STRING, 2) + oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin('type', TType.STRING, 3) + oprot.writeString(self.type.encode('utf-8') if sys.version_info[0] == 2 else self.type) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin('value', TType.STRUCT, 4) + self.value.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TStatementConf(object): + """ + Attributes: + - sessionless + - initialNamespace + - client_protocol + - client_protocol_i64 + + """ + + + def __init__(self, sessionless=None, initialNamespace=None, client_protocol=None, client_protocol_i64=None,): + self.sessionless = sessionless + self.initialNamespace = initialNamespace + self.client_protocol = client_protocol + self.client_protocol_i64 = client_protocol_i64 + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.sessionless = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.initialNamespace = TNamespace() + self.initialNamespace.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.client_protocol = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.client_protocol_i64 = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TStatementConf') + if self.sessionless is not None: + oprot.writeFieldBegin('sessionless', TType.BOOL, 1) + oprot.writeBool(self.sessionless) + oprot.writeFieldEnd() + if self.initialNamespace is not None: + oprot.writeFieldBegin('initialNamespace', TType.STRUCT, 2) + self.initialNamespace.write(oprot) + oprot.writeFieldEnd() + if self.client_protocol is not None: + oprot.writeFieldBegin('client_protocol', TType.I32, 3) + oprot.writeI32(self.client_protocol) + oprot.writeFieldEnd() + if self.client_protocol_i64 is not None: + oprot.writeFieldBegin('client_protocol_i64', TType.I64, 4) + oprot.writeI64(self.client_protocol_i64) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class TExecuteStatementResp(object): """ Attributes: @@ -5582,11 +6540,14 @@ class TExecuteStatementResp(object): - sessionConf - currentClusterLoad - idempotencyType + - remoteResultCacheEnabled + - isServerless + - operationHandles """ - def __init__(self, status=None, operationHandle=None, directResults=None, executionRejected=None, maxClusterCapacity=None, queryCost=None, sessionConf=None, currentClusterLoad=None, idempotencyType=None,): + def __init__(self, status=None, operationHandle=None, directResults=None, executionRejected=None, maxClusterCapacity=None, queryCost=None, sessionConf=None, currentClusterLoad=None, idempotencyType=None, remoteResultCacheEnabled=None, isServerless=None, operationHandles=None,): self.status = status self.operationHandle = operationHandle self.directResults = directResults @@ -5596,6 +6557,9 @@ def __init__(self, status=None, operationHandle=None, directResults=None, execut self.sessionConf = sessionConf self.currentClusterLoad = currentClusterLoad self.idempotencyType = idempotencyType + self.remoteResultCacheEnabled = remoteResultCacheEnabled + self.isServerless = isServerless + self.operationHandles = operationHandles def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -5655,6 +6619,27 @@ def read(self, iprot): self.idempotencyType = iprot.readI32() else: iprot.skip(ftype) + elif fid == 3335: + if ftype == TType.BOOL: + self.remoteResultCacheEnabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3336: + if ftype == TType.BOOL: + self.isServerless = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3337: + if ftype == TType.LIST: + self.operationHandles = [] + (_etype272, _size269) = iprot.readListBegin() + for _i273 in range(_size269): + _elem274 = TOperationHandle() + _elem274.read(iprot) + self.operationHandles.append(_elem274) + iprot.readListEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5701,6 +6686,21 @@ def write(self, oprot): oprot.writeFieldBegin('idempotencyType', TType.I32, 3334) oprot.writeI32(self.idempotencyType) oprot.writeFieldEnd() + if self.remoteResultCacheEnabled is not None: + oprot.writeFieldBegin('remoteResultCacheEnabled', TType.BOOL, 3335) + oprot.writeBool(self.remoteResultCacheEnabled) + oprot.writeFieldEnd() + if self.isServerless is not None: + oprot.writeFieldBegin('isServerless', TType.BOOL, 3336) + oprot.writeBool(self.isServerless) + oprot.writeFieldEnd() + if self.operationHandles is not None: + oprot.writeFieldBegin('operationHandles', TType.LIST, 3337) + oprot.writeListBegin(TType.STRUCT, len(self.operationHandles)) + for iter275 in self.operationHandles: + iter275.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -6376,10 +7376,10 @@ def read(self, iprot): elif fid == 5: if ftype == TType.LIST: self.tableTypes = [] - (_etype233, _size230) = iprot.readListBegin() - for _i234 in range(_size230): - _elem235 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.tableTypes.append(_elem235) + (_etype279, _size276) = iprot.readListBegin() + for _i280 in range(_size276): + _elem281 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.tableTypes.append(_elem281) iprot.readListEnd() else: iprot.skip(ftype) @@ -6435,8 +7435,8 @@ def write(self, oprot): if self.tableTypes is not None: oprot.writeFieldBegin('tableTypes', TType.LIST, 5) oprot.writeListBegin(TType.STRING, len(self.tableTypes)) - for iter236 in self.tableTypes: - oprot.writeString(iter236.encode('utf-8') if sys.version_info[0] == 2 else iter236) + for iter282 in self.tableTypes: + oprot.writeString(iter282.encode('utf-8') if sys.version_info[0] == 2 else iter282) oprot.writeListEnd() oprot.writeFieldEnd() if self.getDirectResults is not None: @@ -7783,6 +8783,7 @@ class TGetOperationStatusResp(object): - numModifiedRows - displayMessage - diagnosticInfo + - errorDetailsJson - responseValidation - idempotencyType - statementTimeout @@ -7791,7 +8792,7 @@ class TGetOperationStatusResp(object): """ - def __init__(self, status=None, operationState=None, sqlState=None, errorCode=None, errorMessage=None, taskStatus=None, operationStarted=None, operationCompleted=None, hasResultSet=None, progressUpdateResponse=None, numModifiedRows=None, displayMessage=None, diagnosticInfo=None, responseValidation=None, idempotencyType=None, statementTimeout=None, statementTimeoutLevel=None,): + def __init__(self, status=None, operationState=None, sqlState=None, errorCode=None, errorMessage=None, taskStatus=None, operationStarted=None, operationCompleted=None, hasResultSet=None, progressUpdateResponse=None, numModifiedRows=None, displayMessage=None, diagnosticInfo=None, errorDetailsJson=None, responseValidation=None, idempotencyType=None, statementTimeout=None, statementTimeoutLevel=None,): self.status = status self.operationState = operationState self.sqlState = sqlState @@ -7805,6 +8806,7 @@ def __init__(self, status=None, operationState=None, sqlState=None, errorCode=No self.numModifiedRows = numModifiedRows self.displayMessage = displayMessage self.diagnosticInfo = diagnosticInfo + self.errorDetailsJson = errorDetailsJson self.responseValidation = responseValidation self.idempotencyType = idempotencyType self.statementTimeout = statementTimeout @@ -7886,6 +8888,11 @@ def read(self, iprot): self.diagnosticInfo = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) + elif fid == 1283: + if ftype == TType.STRING: + self.errorDetailsJson = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) elif fid == 3329: if ftype == TType.STRING: self.responseValidation = iprot.readBinary() @@ -7968,6 +8975,10 @@ def write(self, oprot): oprot.writeFieldBegin('diagnosticInfo', TType.STRING, 1282) oprot.writeString(self.diagnosticInfo.encode('utf-8') if sys.version_info[0] == 2 else self.diagnosticInfo) oprot.writeFieldEnd() + if self.errorDetailsJson is not None: + oprot.writeFieldBegin('errorDetailsJson', TType.STRING, 1283) + oprot.writeString(self.errorDetailsJson.encode('utf-8') if sys.version_info[0] == 2 else self.errorDetailsJson) + oprot.writeFieldEnd() if self.responseValidation is not None: oprot.writeFieldBegin('responseValidation', TType.STRING, 3329) oprot.writeBinary(self.responseValidation) @@ -8150,12 +9161,14 @@ class TCloseOperationReq(object): """ Attributes: - operationHandle + - closeReason """ - def __init__(self, operationHandle=None,): + def __init__(self, operationHandle=None, closeReason= 0,): self.operationHandle = operationHandle + self.closeReason = closeReason def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -8172,6 +9185,11 @@ def read(self, iprot): self.operationHandle.read(iprot) else: iprot.skip(ftype) + elif fid == 3329: + if ftype == TType.I32: + self.closeReason = iprot.readI32() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -8186,6 +9204,10 @@ def write(self, oprot): oprot.writeFieldBegin('operationHandle', TType.STRUCT, 1) self.operationHandle.write(oprot) oprot.writeFieldEnd() + if self.closeReason is not None: + oprot.writeFieldBegin('closeReason', TType.I32, 3329) + oprot.writeI32(self.closeReason) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -8353,11 +9375,19 @@ class TGetResultSetMetadataResp(object): - resultFiles - manifestFile - manifestFileFormat + - cacheLookupLatency + - remoteCacheMissReason + - fetchDisposition + - remoteResultCacheEnabled + - isServerless + - resultDataFormat + - truncatedByThriftLimit + - resultByteLimit """ - def __init__(self, status=None, schema=None, resultFormat=None, lz4Compressed=None, arrowSchema=None, cacheLookupResult=None, uncompressedBytes=None, compressedBytes=None, isStagingOperation=None, reasonForNoCloudFetch=None, resultFiles=None, manifestFile=None, manifestFileFormat=None,): + def __init__(self, status=None, schema=None, resultFormat=None, lz4Compressed=None, arrowSchema=None, cacheLookupResult=None, uncompressedBytes=None, compressedBytes=None, isStagingOperation=None, reasonForNoCloudFetch=None, resultFiles=None, manifestFile=None, manifestFileFormat=None, cacheLookupLatency=None, remoteCacheMissReason=None, fetchDisposition=None, remoteResultCacheEnabled=None, isServerless=None, resultDataFormat=None, truncatedByThriftLimit=None, resultByteLimit=None,): self.status = status self.schema = schema self.resultFormat = resultFormat @@ -8371,6 +9401,14 @@ def __init__(self, status=None, schema=None, resultFormat=None, lz4Compressed=No self.resultFiles = resultFiles self.manifestFile = manifestFile self.manifestFileFormat = manifestFileFormat + self.cacheLookupLatency = cacheLookupLatency + self.remoteCacheMissReason = remoteCacheMissReason + self.fetchDisposition = fetchDisposition + self.remoteResultCacheEnabled = remoteResultCacheEnabled + self.isServerless = isServerless + self.resultDataFormat = resultDataFormat + self.truncatedByThriftLimit = truncatedByThriftLimit + self.resultByteLimit = resultByteLimit def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -8436,11 +9474,11 @@ def read(self, iprot): elif fid == 3330: if ftype == TType.LIST: self.resultFiles = [] - (_etype240, _size237) = iprot.readListBegin() - for _i241 in range(_size237): - _elem242 = TDBSqlCloudResultFile() - _elem242.read(iprot) - self.resultFiles.append(_elem242) + (_etype286, _size283) = iprot.readListBegin() + for _i287 in range(_size283): + _elem288 = TDBSqlCloudResultFile() + _elem288.read(iprot) + self.resultFiles.append(_elem288) iprot.readListEnd() else: iprot.skip(ftype) @@ -8450,8 +9488,49 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 3332: + if ftype == TType.I32: + self.manifestFileFormat = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3333: + if ftype == TType.I64: + self.cacheLookupLatency = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3334: if ftype == TType.STRING: - self.manifestFileFormat = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.remoteCacheMissReason = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3335: + if ftype == TType.I32: + self.fetchDisposition = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3336: + if ftype == TType.BOOL: + self.remoteResultCacheEnabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3337: + if ftype == TType.BOOL: + self.isServerless = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3344: + if ftype == TType.STRUCT: + self.resultDataFormat = TDBSqlResultFormat() + self.resultDataFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3345: + if ftype == TType.BOOL: + self.truncatedByThriftLimit = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3346: + if ftype == TType.I64: + self.resultByteLimit = iprot.readI64() else: iprot.skip(ftype) else: @@ -8507,8 +9586,8 @@ def write(self, oprot): if self.resultFiles is not None: oprot.writeFieldBegin('resultFiles', TType.LIST, 3330) oprot.writeListBegin(TType.STRUCT, len(self.resultFiles)) - for iter243 in self.resultFiles: - iter243.write(oprot) + for iter289 in self.resultFiles: + iter289.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.manifestFile is not None: @@ -8516,8 +9595,40 @@ def write(self, oprot): oprot.writeString(self.manifestFile.encode('utf-8') if sys.version_info[0] == 2 else self.manifestFile) oprot.writeFieldEnd() if self.manifestFileFormat is not None: - oprot.writeFieldBegin('manifestFileFormat', TType.STRING, 3332) - oprot.writeString(self.manifestFileFormat.encode('utf-8') if sys.version_info[0] == 2 else self.manifestFileFormat) + oprot.writeFieldBegin('manifestFileFormat', TType.I32, 3332) + oprot.writeI32(self.manifestFileFormat) + oprot.writeFieldEnd() + if self.cacheLookupLatency is not None: + oprot.writeFieldBegin('cacheLookupLatency', TType.I64, 3333) + oprot.writeI64(self.cacheLookupLatency) + oprot.writeFieldEnd() + if self.remoteCacheMissReason is not None: + oprot.writeFieldBegin('remoteCacheMissReason', TType.STRING, 3334) + oprot.writeString(self.remoteCacheMissReason.encode('utf-8') if sys.version_info[0] == 2 else self.remoteCacheMissReason) + oprot.writeFieldEnd() + if self.fetchDisposition is not None: + oprot.writeFieldBegin('fetchDisposition', TType.I32, 3335) + oprot.writeI32(self.fetchDisposition) + oprot.writeFieldEnd() + if self.remoteResultCacheEnabled is not None: + oprot.writeFieldBegin('remoteResultCacheEnabled', TType.BOOL, 3336) + oprot.writeBool(self.remoteResultCacheEnabled) + oprot.writeFieldEnd() + if self.isServerless is not None: + oprot.writeFieldBegin('isServerless', TType.BOOL, 3337) + oprot.writeBool(self.isServerless) + oprot.writeFieldEnd() + if self.resultDataFormat is not None: + oprot.writeFieldBegin('resultDataFormat', TType.STRUCT, 3344) + self.resultDataFormat.write(oprot) + oprot.writeFieldEnd() + if self.truncatedByThriftLimit is not None: + oprot.writeFieldBegin('truncatedByThriftLimit', TType.BOOL, 3345) + oprot.writeBool(self.truncatedByThriftLimit) + oprot.writeFieldEnd() + if self.resultByteLimit is not None: + oprot.writeFieldBegin('resultByteLimit', TType.I64, 3346) + oprot.writeI64(self.resultByteLimit) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -9267,25 +10378,25 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.headerNames = [] - (_etype247, _size244) = iprot.readListBegin() - for _i248 in range(_size244): - _elem249 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.headerNames.append(_elem249) + (_etype293, _size290) = iprot.readListBegin() + for _i294 in range(_size290): + _elem295 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.headerNames.append(_elem295) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.rows = [] - (_etype253, _size250) = iprot.readListBegin() - for _i254 in range(_size250): - _elem255 = [] - (_etype259, _size256) = iprot.readListBegin() - for _i260 in range(_size256): - _elem261 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _elem255.append(_elem261) + (_etype299, _size296) = iprot.readListBegin() + for _i300 in range(_size296): + _elem301 = [] + (_etype305, _size302) = iprot.readListBegin() + for _i306 in range(_size302): + _elem307 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _elem301.append(_elem307) iprot.readListEnd() - self.rows.append(_elem255) + self.rows.append(_elem301) iprot.readListEnd() else: iprot.skip(ftype) @@ -9322,17 +10433,17 @@ def write(self, oprot): if self.headerNames is not None: oprot.writeFieldBegin('headerNames', TType.LIST, 1) oprot.writeListBegin(TType.STRING, len(self.headerNames)) - for iter262 in self.headerNames: - oprot.writeString(iter262.encode('utf-8') if sys.version_info[0] == 2 else iter262) + for iter308 in self.headerNames: + oprot.writeString(iter308.encode('utf-8') if sys.version_info[0] == 2 else iter308) oprot.writeListEnd() oprot.writeFieldEnd() if self.rows is not None: oprot.writeFieldBegin('rows', TType.LIST, 2) oprot.writeListBegin(TType.LIST, len(self.rows)) - for iter263 in self.rows: - oprot.writeListBegin(TType.STRING, len(iter263)) - for iter264 in iter263: - oprot.writeString(iter264.encode('utf-8') if sys.version_info[0] == 2 else iter264) + for iter309 in self.rows: + oprot.writeListBegin(TType.STRING, len(iter309)) + for iter310 in iter309: + oprot.writeString(iter310.encode('utf-8') if sys.version_info[0] == 2 else iter310) oprot.writeListEnd() oprot.writeListEnd() oprot.writeFieldEnd() @@ -9380,532 +10491,6 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - -class TDBSqlClusterMetrics(object): - """ - Attributes: - - clusterCapacity - - numRunningTasks - - numPendingTasks - - rejectionThreshold - - tasksCompletedPerMinute - - """ - - - def __init__(self, clusterCapacity=None, numRunningTasks=None, numPendingTasks=None, rejectionThreshold=None, tasksCompletedPerMinute=None,): - self.clusterCapacity = clusterCapacity - self.numRunningTasks = numRunningTasks - self.numPendingTasks = numPendingTasks - self.rejectionThreshold = rejectionThreshold - self.tasksCompletedPerMinute = tasksCompletedPerMinute - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.clusterCapacity = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.numRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.numPendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.DOUBLE: - self.rejectionThreshold = iprot.readDouble() - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.DOUBLE: - self.tasksCompletedPerMinute = iprot.readDouble() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlClusterMetrics') - if self.clusterCapacity is not None: - oprot.writeFieldBegin('clusterCapacity', TType.I32, 1) - oprot.writeI32(self.clusterCapacity) - oprot.writeFieldEnd() - if self.numRunningTasks is not None: - oprot.writeFieldBegin('numRunningTasks', TType.I32, 2) - oprot.writeI32(self.numRunningTasks) - oprot.writeFieldEnd() - if self.numPendingTasks is not None: - oprot.writeFieldBegin('numPendingTasks', TType.I32, 3) - oprot.writeI32(self.numPendingTasks) - oprot.writeFieldEnd() - if self.rejectionThreshold is not None: - oprot.writeFieldBegin('rejectionThreshold', TType.DOUBLE, 4) - oprot.writeDouble(self.rejectionThreshold) - oprot.writeFieldEnd() - if self.tasksCompletedPerMinute is not None: - oprot.writeFieldBegin('tasksCompletedPerMinute', TType.DOUBLE, 5) - oprot.writeDouble(self.tasksCompletedPerMinute) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlQueryLaneMetrics(object): - """ - Attributes: - - fastLaneReservation - - numFastLaneRunningTasks - - numFastLanePendingTasks - - slowLaneReservation - - numSlowLaneRunningTasks - - numSlowLanePendingTasks - - """ - - - def __init__(self, fastLaneReservation=None, numFastLaneRunningTasks=None, numFastLanePendingTasks=None, slowLaneReservation=None, numSlowLaneRunningTasks=None, numSlowLanePendingTasks=None,): - self.fastLaneReservation = fastLaneReservation - self.numFastLaneRunningTasks = numFastLaneRunningTasks - self.numFastLanePendingTasks = numFastLanePendingTasks - self.slowLaneReservation = slowLaneReservation - self.numSlowLaneRunningTasks = numSlowLaneRunningTasks - self.numSlowLanePendingTasks = numSlowLanePendingTasks - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.fastLaneReservation = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.numFastLaneRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.numFastLanePendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.I32: - self.slowLaneReservation = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.I32: - self.numSlowLaneRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 6: - if ftype == TType.I32: - self.numSlowLanePendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlQueryLaneMetrics') - if self.fastLaneReservation is not None: - oprot.writeFieldBegin('fastLaneReservation', TType.I32, 1) - oprot.writeI32(self.fastLaneReservation) - oprot.writeFieldEnd() - if self.numFastLaneRunningTasks is not None: - oprot.writeFieldBegin('numFastLaneRunningTasks', TType.I32, 2) - oprot.writeI32(self.numFastLaneRunningTasks) - oprot.writeFieldEnd() - if self.numFastLanePendingTasks is not None: - oprot.writeFieldBegin('numFastLanePendingTasks', TType.I32, 3) - oprot.writeI32(self.numFastLanePendingTasks) - oprot.writeFieldEnd() - if self.slowLaneReservation is not None: - oprot.writeFieldBegin('slowLaneReservation', TType.I32, 4) - oprot.writeI32(self.slowLaneReservation) - oprot.writeFieldEnd() - if self.numSlowLaneRunningTasks is not None: - oprot.writeFieldBegin('numSlowLaneRunningTasks', TType.I32, 5) - oprot.writeI32(self.numSlowLaneRunningTasks) - oprot.writeFieldEnd() - if self.numSlowLanePendingTasks is not None: - oprot.writeFieldBegin('numSlowLanePendingTasks', TType.I32, 6) - oprot.writeI32(self.numSlowLanePendingTasks) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlQueryMetrics(object): - """ - Attributes: - - status - - operationHandle - - idempotencyType - - sessionHandle - - operationStarted - - queryCost - - numRunningTasks - - numPendingTasks - - numCompletedTasks - - """ - - - def __init__(self, status=None, operationHandle=None, idempotencyType=None, sessionHandle=None, operationStarted=None, queryCost=None, numRunningTasks=None, numPendingTasks=None, numCompletedTasks=None,): - self.status = status - self.operationHandle = operationHandle - self.idempotencyType = idempotencyType - self.sessionHandle = sessionHandle - self.operationStarted = operationStarted - self.queryCost = queryCost - self.numRunningTasks = numRunningTasks - self.numPendingTasks = numPendingTasks - self.numCompletedTasks = numCompletedTasks - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.status = TStatus() - self.status.read(iprot) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRUCT: - self.operationHandle = TOperationHandle() - self.operationHandle.read(iprot) - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.idempotencyType = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.sessionHandle = TSessionHandle() - self.sessionHandle.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.I64: - self.operationStarted = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 6: - if ftype == TType.DOUBLE: - self.queryCost = iprot.readDouble() - else: - iprot.skip(ftype) - elif fid == 7: - if ftype == TType.I32: - self.numRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 8: - if ftype == TType.I32: - self.numPendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 9: - if ftype == TType.I32: - self.numCompletedTasks = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlQueryMetrics') - if self.status is not None: - oprot.writeFieldBegin('status', TType.STRUCT, 1) - self.status.write(oprot) - oprot.writeFieldEnd() - if self.operationHandle is not None: - oprot.writeFieldBegin('operationHandle', TType.STRUCT, 2) - self.operationHandle.write(oprot) - oprot.writeFieldEnd() - if self.idempotencyType is not None: - oprot.writeFieldBegin('idempotencyType', TType.I32, 3) - oprot.writeI32(self.idempotencyType) - oprot.writeFieldEnd() - if self.sessionHandle is not None: - oprot.writeFieldBegin('sessionHandle', TType.STRUCT, 4) - self.sessionHandle.write(oprot) - oprot.writeFieldEnd() - if self.operationStarted is not None: - oprot.writeFieldBegin('operationStarted', TType.I64, 5) - oprot.writeI64(self.operationStarted) - oprot.writeFieldEnd() - if self.queryCost is not None: - oprot.writeFieldBegin('queryCost', TType.DOUBLE, 6) - oprot.writeDouble(self.queryCost) - oprot.writeFieldEnd() - if self.numRunningTasks is not None: - oprot.writeFieldBegin('numRunningTasks', TType.I32, 7) - oprot.writeI32(self.numRunningTasks) - oprot.writeFieldEnd() - if self.numPendingTasks is not None: - oprot.writeFieldBegin('numPendingTasks', TType.I32, 8) - oprot.writeI32(self.numPendingTasks) - oprot.writeFieldEnd() - if self.numCompletedTasks is not None: - oprot.writeFieldBegin('numCompletedTasks', TType.I32, 9) - oprot.writeI32(self.numCompletedTasks) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.status is None: - raise TProtocolException(message='Required field status is unset!') - if self.operationHandle is None: - raise TProtocolException(message='Required field operationHandle is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlGetLoadInformationReq(object): - """ - Attributes: - - includeQueryMetrics - - """ - - - def __init__(self, includeQueryMetrics=False,): - self.includeQueryMetrics = includeQueryMetrics - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.BOOL: - self.includeQueryMetrics = iprot.readBool() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlGetLoadInformationReq') - if self.includeQueryMetrics is not None: - oprot.writeFieldBegin('includeQueryMetrics', TType.BOOL, 1) - oprot.writeBool(self.includeQueryMetrics) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlGetLoadInformationResp(object): - """ - Attributes: - - status - - clusterMetrics - - queryLaneMetrics - - queryMetrics - - """ - - - def __init__(self, status=None, clusterMetrics=None, queryLaneMetrics=None, queryMetrics=None,): - self.status = status - self.clusterMetrics = clusterMetrics - self.queryLaneMetrics = queryLaneMetrics - self.queryMetrics = queryMetrics - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.status = TStatus() - self.status.read(iprot) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRUCT: - self.clusterMetrics = TDBSqlClusterMetrics() - self.clusterMetrics.read(iprot) - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.queryLaneMetrics = TDBSqlQueryLaneMetrics() - self.queryLaneMetrics.read(iprot) - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.LIST: - self.queryMetrics = [] - (_etype268, _size265) = iprot.readListBegin() - for _i269 in range(_size265): - _elem270 = TDBSqlQueryMetrics() - _elem270.read(iprot) - self.queryMetrics.append(_elem270) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlGetLoadInformationResp') - if self.status is not None: - oprot.writeFieldBegin('status', TType.STRUCT, 1) - self.status.write(oprot) - oprot.writeFieldEnd() - if self.clusterMetrics is not None: - oprot.writeFieldBegin('clusterMetrics', TType.STRUCT, 2) - self.clusterMetrics.write(oprot) - oprot.writeFieldEnd() - if self.queryLaneMetrics is not None: - oprot.writeFieldBegin('queryLaneMetrics', TType.STRUCT, 3) - self.queryLaneMetrics.write(oprot) - oprot.writeFieldEnd() - if self.queryMetrics is not None: - oprot.writeFieldBegin('queryMetrics', TType.LIST, 4) - oprot.writeListBegin(TType.STRUCT, len(self.queryMetrics)) - for iter271 in self.queryMetrics: - iter271.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.status is None: - raise TProtocolException(message='Required field status is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) all_structs.append(TTypeQualifierValue) TTypeQualifierValue.thrift_spec = ( None, # 0 @@ -9917,8 +10502,8 @@ def __ne__(self, other): None, # 0 (1, TType.MAP, 'qualifiers', (TType.STRING, 'UTF8', TType.STRUCT, [TTypeQualifierValue, None], False), None, ), # 1 ) -all_structs.append(TPrimitiveTypeEntry) -TPrimitiveTypeEntry.thrift_spec = ( +all_structs.append(TTAllowedParameterValueEntry) +TTAllowedParameterValueEntry.thrift_spec = ( None, # 0 (1, TType.I32, 'type', None, None, ), # 1 (2, TType.STRUCT, 'typeQualifiers', [TTypeQualifiers, None], None, ), # 2 @@ -9952,7 +10537,7 @@ def __ne__(self, other): all_structs.append(TTypeEntry) TTypeEntry.thrift_spec = ( None, # 0 - (1, TType.STRUCT, 'primitiveEntry', [TPrimitiveTypeEntry, None], None, ), # 1 + (1, TType.STRUCT, 'primitiveEntry', [TTAllowedParameterValueEntry, None], None, ), # 1 (2, TType.STRUCT, 'arrayEntry', [TArrayTypeEntry, None], None, ), # 2 (3, TType.STRUCT, 'mapEntry', [TMapTypeEntry, None], None, ), # 3 (4, TType.STRUCT, 'structEntry', [TStructTypeEntry, None], None, ), # 4 @@ -10088,6 +10673,29 @@ def __ne__(self, other): (7, TType.STRUCT, 'stringVal', [TStringColumn, None], None, ), # 7 (8, TType.STRUCT, 'binaryVal', [TBinaryColumn, None], None, ), # 8 ) +all_structs.append(TDBSqlJsonArrayFormat) +TDBSqlJsonArrayFormat.thrift_spec = ( + None, # 0 + (1, TType.I32, 'compressionCodec', None, None, ), # 1 +) +all_structs.append(TDBSqlCsvFormat) +TDBSqlCsvFormat.thrift_spec = ( + None, # 0 + (1, TType.I32, 'compressionCodec', None, None, ), # 1 +) +all_structs.append(TDBSqlArrowFormat) +TDBSqlArrowFormat.thrift_spec = ( + None, # 0 + (1, TType.I32, 'arrowLayout', None, None, ), # 1 + (2, TType.I32, 'compressionCodec', None, None, ), # 2 +) +all_structs.append(TDBSqlResultFormat) +TDBSqlResultFormat.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'arrowFormat', [TDBSqlArrowFormat, None], None, ), # 1 + (2, TType.STRUCT, 'csvFormat', [TDBSqlCsvFormat, None], None, ), # 2 + (3, TType.STRUCT, 'jsonArrayFormat', [TDBSqlJsonArrayFormat, None], None, ), # 3 +) all_structs.append(TSparkArrowBatch) TSparkArrowBatch.thrift_spec = ( None, # 0 @@ -10102,6 +10710,7 @@ def __ne__(self, other): (3, TType.I64, 'startRowOffset', None, None, ), # 3 (4, TType.I64, 'rowCount', None, None, ), # 4 (5, TType.I64, 'bytesNum', None, None, ), # 5 + (6, TType.MAP, 'httpHeaders', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 6 ) all_structs.append(TDBSqlCloudResultFile) TDBSqlCloudResultFile.thrift_spec = ( @@ -10111,6 +10720,9 @@ def __ne__(self, other): (3, TType.I64, 'rowCount', None, None, ), # 3 (4, TType.I64, 'uncompressedBytes', None, None, ), # 4 (5, TType.I64, 'compressedBytes', None, None, ), # 5 + (6, TType.STRING, 'fileLink', 'UTF8', None, ), # 6 + (7, TType.I64, 'linkExpiryTime', None, None, ), # 7 + (8, TType.MAP, 'httpHeaders', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 8 ) all_structs.append(TRowSet) TRowSet.thrift_spec = ( @@ -11397,510 +12009,2557 @@ def __ne__(self, other): None, # 1280 (1281, TType.LIST, 'arrowBatches', (TType.STRUCT, [TSparkArrowBatch, None], False), None, ), # 1281 (1282, TType.LIST, 'resultLinks', (TType.STRUCT, [TSparkArrowResultLink, None], False), None, ), # 1282 -) -all_structs.append(TDBSqlTempView) -TDBSqlTempView.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'name', 'UTF8', None, ), # 1 - (2, TType.STRING, 'sqlStatement', 'UTF8', None, ), # 2 - (3, TType.MAP, 'properties', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 - (4, TType.STRING, 'viewSchema', 'UTF8', None, ), # 4 -) -all_structs.append(TDBSqlSessionCapabilities) -TDBSqlSessionCapabilities.thrift_spec = ( - None, # 0 - (1, TType.BOOL, 'supportsMultipleCatalogs', None, None, ), # 1 -) -all_structs.append(TExpressionInfo) -TExpressionInfo.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'className', 'UTF8', None, ), # 1 - (2, TType.STRING, 'usage', 'UTF8', None, ), # 2 - (3, TType.STRING, 'name', 'UTF8', None, ), # 3 - (4, TType.STRING, 'extended', 'UTF8', None, ), # 4 - (5, TType.STRING, 'db', 'UTF8', None, ), # 5 - (6, TType.STRING, 'arguments', 'UTF8', None, ), # 6 - (7, TType.STRING, 'examples', 'UTF8', None, ), # 7 - (8, TType.STRING, 'note', 'UTF8', None, ), # 8 - (9, TType.STRING, 'group', 'UTF8', None, ), # 9 - (10, TType.STRING, 'since', 'UTF8', None, ), # 10 - (11, TType.STRING, 'deprecated', 'UTF8', None, ), # 11 - (12, TType.STRING, 'source', 'UTF8', None, ), # 12 -) -all_structs.append(TDBSqlConfValue) -TDBSqlConfValue.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'value', 'UTF8', None, ), # 1 -) -all_structs.append(TDBSqlSessionConf) -TDBSqlSessionConf.thrift_spec = ( - None, # 0 - (1, TType.MAP, 'confs', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 1 - (2, TType.LIST, 'tempViews', (TType.STRUCT, [TDBSqlTempView, None], False), None, ), # 2 - (3, TType.STRING, 'currentDatabase', 'UTF8', None, ), # 3 - (4, TType.STRING, 'currentCatalog', 'UTF8', None, ), # 4 - (5, TType.STRUCT, 'sessionCapabilities', [TDBSqlSessionCapabilities, None], None, ), # 5 - (6, TType.LIST, 'expressionsInfos', (TType.STRUCT, [TExpressionInfo, None], False), None, ), # 6 - (7, TType.MAP, 'internalConfs', (TType.STRING, 'UTF8', TType.STRUCT, [TDBSqlConfValue, None], False), None, ), # 7 -) -all_structs.append(TStatus) -TStatus.thrift_spec = ( - None, # 0 - (1, TType.I32, 'statusCode', None, None, ), # 1 - (2, TType.LIST, 'infoMessages', (TType.STRING, 'UTF8', False), None, ), # 2 - (3, TType.STRING, 'sqlState', 'UTF8', None, ), # 3 - (4, TType.I32, 'errorCode', None, None, ), # 4 - (5, TType.STRING, 'errorMessage', 'UTF8', None, ), # 5 - (6, TType.STRING, 'displayMessage', 'UTF8', None, ), # 6 - None, # 7 - None, # 8 - None, # 9 - None, # 10 - None, # 11 - None, # 12 - None, # 13 - None, # 14 - None, # 15 - None, # 16 - None, # 17 - None, # 18 - None, # 19 - None, # 20 - None, # 21 - None, # 22 - None, # 23 - None, # 24 - None, # 25 - None, # 26 - None, # 27 - None, # 28 - None, # 29 - None, # 30 - None, # 31 - None, # 32 - None, # 33 - None, # 34 - None, # 35 - None, # 36 - None, # 37 - None, # 38 - None, # 39 - None, # 40 - None, # 41 - None, # 42 - None, # 43 - None, # 44 - None, # 45 - None, # 46 - None, # 47 - None, # 48 - None, # 49 - None, # 50 - None, # 51 - None, # 52 - None, # 53 - None, # 54 - None, # 55 - None, # 56 - None, # 57 - None, # 58 - None, # 59 - None, # 60 - None, # 61 - None, # 62 - None, # 63 - None, # 64 - None, # 65 - None, # 66 - None, # 67 - None, # 68 - None, # 69 - None, # 70 - None, # 71 - None, # 72 - None, # 73 - None, # 74 - None, # 75 - None, # 76 - None, # 77 - None, # 78 - None, # 79 - None, # 80 - None, # 81 - None, # 82 - None, # 83 - None, # 84 - None, # 85 - None, # 86 - None, # 87 - None, # 88 - None, # 89 - None, # 90 - None, # 91 - None, # 92 - None, # 93 - None, # 94 - None, # 95 - None, # 96 - None, # 97 - None, # 98 - None, # 99 - None, # 100 - None, # 101 - None, # 102 - None, # 103 - None, # 104 - None, # 105 - None, # 106 - None, # 107 - None, # 108 - None, # 109 - None, # 110 - None, # 111 - None, # 112 - None, # 113 - None, # 114 - None, # 115 - None, # 116 - None, # 117 - None, # 118 - None, # 119 - None, # 120 - None, # 121 - None, # 122 - None, # 123 - None, # 124 - None, # 125 - None, # 126 - None, # 127 - None, # 128 - None, # 129 - None, # 130 - None, # 131 - None, # 132 - None, # 133 - None, # 134 - None, # 135 - None, # 136 - None, # 137 - None, # 138 - None, # 139 - None, # 140 - None, # 141 - None, # 142 - None, # 143 - None, # 144 - None, # 145 - None, # 146 - None, # 147 - None, # 148 - None, # 149 - None, # 150 - None, # 151 - None, # 152 - None, # 153 - None, # 154 - None, # 155 - None, # 156 - None, # 157 - None, # 158 - None, # 159 - None, # 160 - None, # 161 - None, # 162 - None, # 163 - None, # 164 - None, # 165 - None, # 166 - None, # 167 - None, # 168 - None, # 169 - None, # 170 - None, # 171 - None, # 172 - None, # 173 - None, # 174 - None, # 175 - None, # 176 - None, # 177 - None, # 178 - None, # 179 - None, # 180 - None, # 181 - None, # 182 - None, # 183 - None, # 184 - None, # 185 - None, # 186 - None, # 187 - None, # 188 - None, # 189 - None, # 190 - None, # 191 - None, # 192 - None, # 193 - None, # 194 - None, # 195 - None, # 196 - None, # 197 - None, # 198 - None, # 199 - None, # 200 - None, # 201 - None, # 202 - None, # 203 - None, # 204 - None, # 205 - None, # 206 - None, # 207 - None, # 208 - None, # 209 - None, # 210 - None, # 211 - None, # 212 - None, # 213 - None, # 214 - None, # 215 - None, # 216 - None, # 217 - None, # 218 - None, # 219 - None, # 220 - None, # 221 - None, # 222 - None, # 223 - None, # 224 - None, # 225 - None, # 226 - None, # 227 - None, # 228 - None, # 229 - None, # 230 - None, # 231 - None, # 232 - None, # 233 - None, # 234 - None, # 235 - None, # 236 - None, # 237 - None, # 238 - None, # 239 - None, # 240 - None, # 241 - None, # 242 - None, # 243 - None, # 244 - None, # 245 - None, # 246 - None, # 247 - None, # 248 - None, # 249 - None, # 250 - None, # 251 - None, # 252 - None, # 253 - None, # 254 - None, # 255 - None, # 256 - None, # 257 - None, # 258 - None, # 259 - None, # 260 - None, # 261 - None, # 262 - None, # 263 - None, # 264 - None, # 265 - None, # 266 - None, # 267 - None, # 268 - None, # 269 - None, # 270 - None, # 271 - None, # 272 - None, # 273 - None, # 274 - None, # 275 - None, # 276 - None, # 277 - None, # 278 - None, # 279 - None, # 280 - None, # 281 - None, # 282 - None, # 283 - None, # 284 - None, # 285 - None, # 286 - None, # 287 - None, # 288 - None, # 289 - None, # 290 - None, # 291 - None, # 292 - None, # 293 - None, # 294 - None, # 295 - None, # 296 - None, # 297 - None, # 298 - None, # 299 - None, # 300 - None, # 301 - None, # 302 - None, # 303 - None, # 304 - None, # 305 - None, # 306 - None, # 307 - None, # 308 - None, # 309 - None, # 310 - None, # 311 - None, # 312 - None, # 313 - None, # 314 - None, # 315 - None, # 316 - None, # 317 - None, # 318 - None, # 319 - None, # 320 - None, # 321 - None, # 322 - None, # 323 - None, # 324 - None, # 325 - None, # 326 - None, # 327 - None, # 328 - None, # 329 - None, # 330 - None, # 331 - None, # 332 - None, # 333 - None, # 334 - None, # 335 - None, # 336 - None, # 337 - None, # 338 - None, # 339 - None, # 340 - None, # 341 - None, # 342 - None, # 343 - None, # 344 - None, # 345 - None, # 346 - None, # 347 - None, # 348 - None, # 349 - None, # 350 - None, # 351 - None, # 352 - None, # 353 - None, # 354 - None, # 355 - None, # 356 - None, # 357 - None, # 358 - None, # 359 - None, # 360 - None, # 361 - None, # 362 - None, # 363 - None, # 364 - None, # 365 - None, # 366 - None, # 367 - None, # 368 - None, # 369 - None, # 370 - None, # 371 - None, # 372 - None, # 373 - None, # 374 - None, # 375 - None, # 376 - None, # 377 - None, # 378 - None, # 379 - None, # 380 - None, # 381 - None, # 382 - None, # 383 - None, # 384 - None, # 385 - None, # 386 - None, # 387 - None, # 388 - None, # 389 - None, # 390 - None, # 391 - None, # 392 - None, # 393 - None, # 394 - None, # 395 - None, # 396 - None, # 397 - None, # 398 - None, # 399 - None, # 400 - None, # 401 - None, # 402 - None, # 403 - None, # 404 - None, # 405 - None, # 406 - None, # 407 - None, # 408 - None, # 409 - None, # 410 - None, # 411 - None, # 412 - None, # 413 - None, # 414 - None, # 415 - None, # 416 - None, # 417 - None, # 418 - None, # 419 - None, # 420 - None, # 421 - None, # 422 - None, # 423 - None, # 424 - None, # 425 - None, # 426 - None, # 427 - None, # 428 - None, # 429 - None, # 430 - None, # 431 - None, # 432 - None, # 433 - None, # 434 - None, # 435 - None, # 436 - None, # 437 - None, # 438 - None, # 439 - None, # 440 - None, # 441 - None, # 442 - None, # 443 - None, # 444 - None, # 445 - None, # 446 - None, # 447 - None, # 448 - None, # 449 - None, # 450 - None, # 451 - None, # 452 - None, # 453 - None, # 454 - None, # 455 + None, # 1283 + None, # 1284 + None, # 1285 + None, # 1286 + None, # 1287 + None, # 1288 + None, # 1289 + None, # 1290 + None, # 1291 + None, # 1292 + None, # 1293 + None, # 1294 + None, # 1295 + None, # 1296 + None, # 1297 + None, # 1298 + None, # 1299 + None, # 1300 + None, # 1301 + None, # 1302 + None, # 1303 + None, # 1304 + None, # 1305 + None, # 1306 + None, # 1307 + None, # 1308 + None, # 1309 + None, # 1310 + None, # 1311 + None, # 1312 + None, # 1313 + None, # 1314 + None, # 1315 + None, # 1316 + None, # 1317 + None, # 1318 + None, # 1319 + None, # 1320 + None, # 1321 + None, # 1322 + None, # 1323 + None, # 1324 + None, # 1325 + None, # 1326 + None, # 1327 + None, # 1328 + None, # 1329 + None, # 1330 + None, # 1331 + None, # 1332 + None, # 1333 + None, # 1334 + None, # 1335 + None, # 1336 + None, # 1337 + None, # 1338 + None, # 1339 + None, # 1340 + None, # 1341 + None, # 1342 + None, # 1343 + None, # 1344 + None, # 1345 + None, # 1346 + None, # 1347 + None, # 1348 + None, # 1349 + None, # 1350 + None, # 1351 + None, # 1352 + None, # 1353 + None, # 1354 + None, # 1355 + None, # 1356 + None, # 1357 + None, # 1358 + None, # 1359 + None, # 1360 + None, # 1361 + None, # 1362 + None, # 1363 + None, # 1364 + None, # 1365 + None, # 1366 + None, # 1367 + None, # 1368 + None, # 1369 + None, # 1370 + None, # 1371 + None, # 1372 + None, # 1373 + None, # 1374 + None, # 1375 + None, # 1376 + None, # 1377 + None, # 1378 + None, # 1379 + None, # 1380 + None, # 1381 + None, # 1382 + None, # 1383 + None, # 1384 + None, # 1385 + None, # 1386 + None, # 1387 + None, # 1388 + None, # 1389 + None, # 1390 + None, # 1391 + None, # 1392 + None, # 1393 + None, # 1394 + None, # 1395 + None, # 1396 + None, # 1397 + None, # 1398 + None, # 1399 + None, # 1400 + None, # 1401 + None, # 1402 + None, # 1403 + None, # 1404 + None, # 1405 + None, # 1406 + None, # 1407 + None, # 1408 + None, # 1409 + None, # 1410 + None, # 1411 + None, # 1412 + None, # 1413 + None, # 1414 + None, # 1415 + None, # 1416 + None, # 1417 + None, # 1418 + None, # 1419 + None, # 1420 + None, # 1421 + None, # 1422 + None, # 1423 + None, # 1424 + None, # 1425 + None, # 1426 + None, # 1427 + None, # 1428 + None, # 1429 + None, # 1430 + None, # 1431 + None, # 1432 + None, # 1433 + None, # 1434 + None, # 1435 + None, # 1436 + None, # 1437 + None, # 1438 + None, # 1439 + None, # 1440 + None, # 1441 + None, # 1442 + None, # 1443 + None, # 1444 + None, # 1445 + None, # 1446 + None, # 1447 + None, # 1448 + None, # 1449 + None, # 1450 + None, # 1451 + None, # 1452 + None, # 1453 + None, # 1454 + None, # 1455 + None, # 1456 + None, # 1457 + None, # 1458 + None, # 1459 + None, # 1460 + None, # 1461 + None, # 1462 + None, # 1463 + None, # 1464 + None, # 1465 + None, # 1466 + None, # 1467 + None, # 1468 + None, # 1469 + None, # 1470 + None, # 1471 + None, # 1472 + None, # 1473 + None, # 1474 + None, # 1475 + None, # 1476 + None, # 1477 + None, # 1478 + None, # 1479 + None, # 1480 + None, # 1481 + None, # 1482 + None, # 1483 + None, # 1484 + None, # 1485 + None, # 1486 + None, # 1487 + None, # 1488 + None, # 1489 + None, # 1490 + None, # 1491 + None, # 1492 + None, # 1493 + None, # 1494 + None, # 1495 + None, # 1496 + None, # 1497 + None, # 1498 + None, # 1499 + None, # 1500 + None, # 1501 + None, # 1502 + None, # 1503 + None, # 1504 + None, # 1505 + None, # 1506 + None, # 1507 + None, # 1508 + None, # 1509 + None, # 1510 + None, # 1511 + None, # 1512 + None, # 1513 + None, # 1514 + None, # 1515 + None, # 1516 + None, # 1517 + None, # 1518 + None, # 1519 + None, # 1520 + None, # 1521 + None, # 1522 + None, # 1523 + None, # 1524 + None, # 1525 + None, # 1526 + None, # 1527 + None, # 1528 + None, # 1529 + None, # 1530 + None, # 1531 + None, # 1532 + None, # 1533 + None, # 1534 + None, # 1535 + None, # 1536 + None, # 1537 + None, # 1538 + None, # 1539 + None, # 1540 + None, # 1541 + None, # 1542 + None, # 1543 + None, # 1544 + None, # 1545 + None, # 1546 + None, # 1547 + None, # 1548 + None, # 1549 + None, # 1550 + None, # 1551 + None, # 1552 + None, # 1553 + None, # 1554 + None, # 1555 + None, # 1556 + None, # 1557 + None, # 1558 + None, # 1559 + None, # 1560 + None, # 1561 + None, # 1562 + None, # 1563 + None, # 1564 + None, # 1565 + None, # 1566 + None, # 1567 + None, # 1568 + None, # 1569 + None, # 1570 + None, # 1571 + None, # 1572 + None, # 1573 + None, # 1574 + None, # 1575 + None, # 1576 + None, # 1577 + None, # 1578 + None, # 1579 + None, # 1580 + None, # 1581 + None, # 1582 + None, # 1583 + None, # 1584 + None, # 1585 + None, # 1586 + None, # 1587 + None, # 1588 + None, # 1589 + None, # 1590 + None, # 1591 + None, # 1592 + None, # 1593 + None, # 1594 + None, # 1595 + None, # 1596 + None, # 1597 + None, # 1598 + None, # 1599 + None, # 1600 + None, # 1601 + None, # 1602 + None, # 1603 + None, # 1604 + None, # 1605 + None, # 1606 + None, # 1607 + None, # 1608 + None, # 1609 + None, # 1610 + None, # 1611 + None, # 1612 + None, # 1613 + None, # 1614 + None, # 1615 + None, # 1616 + None, # 1617 + None, # 1618 + None, # 1619 + None, # 1620 + None, # 1621 + None, # 1622 + None, # 1623 + None, # 1624 + None, # 1625 + None, # 1626 + None, # 1627 + None, # 1628 + None, # 1629 + None, # 1630 + None, # 1631 + None, # 1632 + None, # 1633 + None, # 1634 + None, # 1635 + None, # 1636 + None, # 1637 + None, # 1638 + None, # 1639 + None, # 1640 + None, # 1641 + None, # 1642 + None, # 1643 + None, # 1644 + None, # 1645 + None, # 1646 + None, # 1647 + None, # 1648 + None, # 1649 + None, # 1650 + None, # 1651 + None, # 1652 + None, # 1653 + None, # 1654 + None, # 1655 + None, # 1656 + None, # 1657 + None, # 1658 + None, # 1659 + None, # 1660 + None, # 1661 + None, # 1662 + None, # 1663 + None, # 1664 + None, # 1665 + None, # 1666 + None, # 1667 + None, # 1668 + None, # 1669 + None, # 1670 + None, # 1671 + None, # 1672 + None, # 1673 + None, # 1674 + None, # 1675 + None, # 1676 + None, # 1677 + None, # 1678 + None, # 1679 + None, # 1680 + None, # 1681 + None, # 1682 + None, # 1683 + None, # 1684 + None, # 1685 + None, # 1686 + None, # 1687 + None, # 1688 + None, # 1689 + None, # 1690 + None, # 1691 + None, # 1692 + None, # 1693 + None, # 1694 + None, # 1695 + None, # 1696 + None, # 1697 + None, # 1698 + None, # 1699 + None, # 1700 + None, # 1701 + None, # 1702 + None, # 1703 + None, # 1704 + None, # 1705 + None, # 1706 + None, # 1707 + None, # 1708 + None, # 1709 + None, # 1710 + None, # 1711 + None, # 1712 + None, # 1713 + None, # 1714 + None, # 1715 + None, # 1716 + None, # 1717 + None, # 1718 + None, # 1719 + None, # 1720 + None, # 1721 + None, # 1722 + None, # 1723 + None, # 1724 + None, # 1725 + None, # 1726 + None, # 1727 + None, # 1728 + None, # 1729 + None, # 1730 + None, # 1731 + None, # 1732 + None, # 1733 + None, # 1734 + None, # 1735 + None, # 1736 + None, # 1737 + None, # 1738 + None, # 1739 + None, # 1740 + None, # 1741 + None, # 1742 + None, # 1743 + None, # 1744 + None, # 1745 + None, # 1746 + None, # 1747 + None, # 1748 + None, # 1749 + None, # 1750 + None, # 1751 + None, # 1752 + None, # 1753 + None, # 1754 + None, # 1755 + None, # 1756 + None, # 1757 + None, # 1758 + None, # 1759 + None, # 1760 + None, # 1761 + None, # 1762 + None, # 1763 + None, # 1764 + None, # 1765 + None, # 1766 + None, # 1767 + None, # 1768 + None, # 1769 + None, # 1770 + None, # 1771 + None, # 1772 + None, # 1773 + None, # 1774 + None, # 1775 + None, # 1776 + None, # 1777 + None, # 1778 + None, # 1779 + None, # 1780 + None, # 1781 + None, # 1782 + None, # 1783 + None, # 1784 + None, # 1785 + None, # 1786 + None, # 1787 + None, # 1788 + None, # 1789 + None, # 1790 + None, # 1791 + None, # 1792 + None, # 1793 + None, # 1794 + None, # 1795 + None, # 1796 + None, # 1797 + None, # 1798 + None, # 1799 + None, # 1800 + None, # 1801 + None, # 1802 + None, # 1803 + None, # 1804 + None, # 1805 + None, # 1806 + None, # 1807 + None, # 1808 + None, # 1809 + None, # 1810 + None, # 1811 + None, # 1812 + None, # 1813 + None, # 1814 + None, # 1815 + None, # 1816 + None, # 1817 + None, # 1818 + None, # 1819 + None, # 1820 + None, # 1821 + None, # 1822 + None, # 1823 + None, # 1824 + None, # 1825 + None, # 1826 + None, # 1827 + None, # 1828 + None, # 1829 + None, # 1830 + None, # 1831 + None, # 1832 + None, # 1833 + None, # 1834 + None, # 1835 + None, # 1836 + None, # 1837 + None, # 1838 + None, # 1839 + None, # 1840 + None, # 1841 + None, # 1842 + None, # 1843 + None, # 1844 + None, # 1845 + None, # 1846 + None, # 1847 + None, # 1848 + None, # 1849 + None, # 1850 + None, # 1851 + None, # 1852 + None, # 1853 + None, # 1854 + None, # 1855 + None, # 1856 + None, # 1857 + None, # 1858 + None, # 1859 + None, # 1860 + None, # 1861 + None, # 1862 + None, # 1863 + None, # 1864 + None, # 1865 + None, # 1866 + None, # 1867 + None, # 1868 + None, # 1869 + None, # 1870 + None, # 1871 + None, # 1872 + None, # 1873 + None, # 1874 + None, # 1875 + None, # 1876 + None, # 1877 + None, # 1878 + None, # 1879 + None, # 1880 + None, # 1881 + None, # 1882 + None, # 1883 + None, # 1884 + None, # 1885 + None, # 1886 + None, # 1887 + None, # 1888 + None, # 1889 + None, # 1890 + None, # 1891 + None, # 1892 + None, # 1893 + None, # 1894 + None, # 1895 + None, # 1896 + None, # 1897 + None, # 1898 + None, # 1899 + None, # 1900 + None, # 1901 + None, # 1902 + None, # 1903 + None, # 1904 + None, # 1905 + None, # 1906 + None, # 1907 + None, # 1908 + None, # 1909 + None, # 1910 + None, # 1911 + None, # 1912 + None, # 1913 + None, # 1914 + None, # 1915 + None, # 1916 + None, # 1917 + None, # 1918 + None, # 1919 + None, # 1920 + None, # 1921 + None, # 1922 + None, # 1923 + None, # 1924 + None, # 1925 + None, # 1926 + None, # 1927 + None, # 1928 + None, # 1929 + None, # 1930 + None, # 1931 + None, # 1932 + None, # 1933 + None, # 1934 + None, # 1935 + None, # 1936 + None, # 1937 + None, # 1938 + None, # 1939 + None, # 1940 + None, # 1941 + None, # 1942 + None, # 1943 + None, # 1944 + None, # 1945 + None, # 1946 + None, # 1947 + None, # 1948 + None, # 1949 + None, # 1950 + None, # 1951 + None, # 1952 + None, # 1953 + None, # 1954 + None, # 1955 + None, # 1956 + None, # 1957 + None, # 1958 + None, # 1959 + None, # 1960 + None, # 1961 + None, # 1962 + None, # 1963 + None, # 1964 + None, # 1965 + None, # 1966 + None, # 1967 + None, # 1968 + None, # 1969 + None, # 1970 + None, # 1971 + None, # 1972 + None, # 1973 + None, # 1974 + None, # 1975 + None, # 1976 + None, # 1977 + None, # 1978 + None, # 1979 + None, # 1980 + None, # 1981 + None, # 1982 + None, # 1983 + None, # 1984 + None, # 1985 + None, # 1986 + None, # 1987 + None, # 1988 + None, # 1989 + None, # 1990 + None, # 1991 + None, # 1992 + None, # 1993 + None, # 1994 + None, # 1995 + None, # 1996 + None, # 1997 + None, # 1998 + None, # 1999 + None, # 2000 + None, # 2001 + None, # 2002 + None, # 2003 + None, # 2004 + None, # 2005 + None, # 2006 + None, # 2007 + None, # 2008 + None, # 2009 + None, # 2010 + None, # 2011 + None, # 2012 + None, # 2013 + None, # 2014 + None, # 2015 + None, # 2016 + None, # 2017 + None, # 2018 + None, # 2019 + None, # 2020 + None, # 2021 + None, # 2022 + None, # 2023 + None, # 2024 + None, # 2025 + None, # 2026 + None, # 2027 + None, # 2028 + None, # 2029 + None, # 2030 + None, # 2031 + None, # 2032 + None, # 2033 + None, # 2034 + None, # 2035 + None, # 2036 + None, # 2037 + None, # 2038 + None, # 2039 + None, # 2040 + None, # 2041 + None, # 2042 + None, # 2043 + None, # 2044 + None, # 2045 + None, # 2046 + None, # 2047 + None, # 2048 + None, # 2049 + None, # 2050 + None, # 2051 + None, # 2052 + None, # 2053 + None, # 2054 + None, # 2055 + None, # 2056 + None, # 2057 + None, # 2058 + None, # 2059 + None, # 2060 + None, # 2061 + None, # 2062 + None, # 2063 + None, # 2064 + None, # 2065 + None, # 2066 + None, # 2067 + None, # 2068 + None, # 2069 + None, # 2070 + None, # 2071 + None, # 2072 + None, # 2073 + None, # 2074 + None, # 2075 + None, # 2076 + None, # 2077 + None, # 2078 + None, # 2079 + None, # 2080 + None, # 2081 + None, # 2082 + None, # 2083 + None, # 2084 + None, # 2085 + None, # 2086 + None, # 2087 + None, # 2088 + None, # 2089 + None, # 2090 + None, # 2091 + None, # 2092 + None, # 2093 + None, # 2094 + None, # 2095 + None, # 2096 + None, # 2097 + None, # 2098 + None, # 2099 + None, # 2100 + None, # 2101 + None, # 2102 + None, # 2103 + None, # 2104 + None, # 2105 + None, # 2106 + None, # 2107 + None, # 2108 + None, # 2109 + None, # 2110 + None, # 2111 + None, # 2112 + None, # 2113 + None, # 2114 + None, # 2115 + None, # 2116 + None, # 2117 + None, # 2118 + None, # 2119 + None, # 2120 + None, # 2121 + None, # 2122 + None, # 2123 + None, # 2124 + None, # 2125 + None, # 2126 + None, # 2127 + None, # 2128 + None, # 2129 + None, # 2130 + None, # 2131 + None, # 2132 + None, # 2133 + None, # 2134 + None, # 2135 + None, # 2136 + None, # 2137 + None, # 2138 + None, # 2139 + None, # 2140 + None, # 2141 + None, # 2142 + None, # 2143 + None, # 2144 + None, # 2145 + None, # 2146 + None, # 2147 + None, # 2148 + None, # 2149 + None, # 2150 + None, # 2151 + None, # 2152 + None, # 2153 + None, # 2154 + None, # 2155 + None, # 2156 + None, # 2157 + None, # 2158 + None, # 2159 + None, # 2160 + None, # 2161 + None, # 2162 + None, # 2163 + None, # 2164 + None, # 2165 + None, # 2166 + None, # 2167 + None, # 2168 + None, # 2169 + None, # 2170 + None, # 2171 + None, # 2172 + None, # 2173 + None, # 2174 + None, # 2175 + None, # 2176 + None, # 2177 + None, # 2178 + None, # 2179 + None, # 2180 + None, # 2181 + None, # 2182 + None, # 2183 + None, # 2184 + None, # 2185 + None, # 2186 + None, # 2187 + None, # 2188 + None, # 2189 + None, # 2190 + None, # 2191 + None, # 2192 + None, # 2193 + None, # 2194 + None, # 2195 + None, # 2196 + None, # 2197 + None, # 2198 + None, # 2199 + None, # 2200 + None, # 2201 + None, # 2202 + None, # 2203 + None, # 2204 + None, # 2205 + None, # 2206 + None, # 2207 + None, # 2208 + None, # 2209 + None, # 2210 + None, # 2211 + None, # 2212 + None, # 2213 + None, # 2214 + None, # 2215 + None, # 2216 + None, # 2217 + None, # 2218 + None, # 2219 + None, # 2220 + None, # 2221 + None, # 2222 + None, # 2223 + None, # 2224 + None, # 2225 + None, # 2226 + None, # 2227 + None, # 2228 + None, # 2229 + None, # 2230 + None, # 2231 + None, # 2232 + None, # 2233 + None, # 2234 + None, # 2235 + None, # 2236 + None, # 2237 + None, # 2238 + None, # 2239 + None, # 2240 + None, # 2241 + None, # 2242 + None, # 2243 + None, # 2244 + None, # 2245 + None, # 2246 + None, # 2247 + None, # 2248 + None, # 2249 + None, # 2250 + None, # 2251 + None, # 2252 + None, # 2253 + None, # 2254 + None, # 2255 + None, # 2256 + None, # 2257 + None, # 2258 + None, # 2259 + None, # 2260 + None, # 2261 + None, # 2262 + None, # 2263 + None, # 2264 + None, # 2265 + None, # 2266 + None, # 2267 + None, # 2268 + None, # 2269 + None, # 2270 + None, # 2271 + None, # 2272 + None, # 2273 + None, # 2274 + None, # 2275 + None, # 2276 + None, # 2277 + None, # 2278 + None, # 2279 + None, # 2280 + None, # 2281 + None, # 2282 + None, # 2283 + None, # 2284 + None, # 2285 + None, # 2286 + None, # 2287 + None, # 2288 + None, # 2289 + None, # 2290 + None, # 2291 + None, # 2292 + None, # 2293 + None, # 2294 + None, # 2295 + None, # 2296 + None, # 2297 + None, # 2298 + None, # 2299 + None, # 2300 + None, # 2301 + None, # 2302 + None, # 2303 + None, # 2304 + None, # 2305 + None, # 2306 + None, # 2307 + None, # 2308 + None, # 2309 + None, # 2310 + None, # 2311 + None, # 2312 + None, # 2313 + None, # 2314 + None, # 2315 + None, # 2316 + None, # 2317 + None, # 2318 + None, # 2319 + None, # 2320 + None, # 2321 + None, # 2322 + None, # 2323 + None, # 2324 + None, # 2325 + None, # 2326 + None, # 2327 + None, # 2328 + None, # 2329 + None, # 2330 + None, # 2331 + None, # 2332 + None, # 2333 + None, # 2334 + None, # 2335 + None, # 2336 + None, # 2337 + None, # 2338 + None, # 2339 + None, # 2340 + None, # 2341 + None, # 2342 + None, # 2343 + None, # 2344 + None, # 2345 + None, # 2346 + None, # 2347 + None, # 2348 + None, # 2349 + None, # 2350 + None, # 2351 + None, # 2352 + None, # 2353 + None, # 2354 + None, # 2355 + None, # 2356 + None, # 2357 + None, # 2358 + None, # 2359 + None, # 2360 + None, # 2361 + None, # 2362 + None, # 2363 + None, # 2364 + None, # 2365 + None, # 2366 + None, # 2367 + None, # 2368 + None, # 2369 + None, # 2370 + None, # 2371 + None, # 2372 + None, # 2373 + None, # 2374 + None, # 2375 + None, # 2376 + None, # 2377 + None, # 2378 + None, # 2379 + None, # 2380 + None, # 2381 + None, # 2382 + None, # 2383 + None, # 2384 + None, # 2385 + None, # 2386 + None, # 2387 + None, # 2388 + None, # 2389 + None, # 2390 + None, # 2391 + None, # 2392 + None, # 2393 + None, # 2394 + None, # 2395 + None, # 2396 + None, # 2397 + None, # 2398 + None, # 2399 + None, # 2400 + None, # 2401 + None, # 2402 + None, # 2403 + None, # 2404 + None, # 2405 + None, # 2406 + None, # 2407 + None, # 2408 + None, # 2409 + None, # 2410 + None, # 2411 + None, # 2412 + None, # 2413 + None, # 2414 + None, # 2415 + None, # 2416 + None, # 2417 + None, # 2418 + None, # 2419 + None, # 2420 + None, # 2421 + None, # 2422 + None, # 2423 + None, # 2424 + None, # 2425 + None, # 2426 + None, # 2427 + None, # 2428 + None, # 2429 + None, # 2430 + None, # 2431 + None, # 2432 + None, # 2433 + None, # 2434 + None, # 2435 + None, # 2436 + None, # 2437 + None, # 2438 + None, # 2439 + None, # 2440 + None, # 2441 + None, # 2442 + None, # 2443 + None, # 2444 + None, # 2445 + None, # 2446 + None, # 2447 + None, # 2448 + None, # 2449 + None, # 2450 + None, # 2451 + None, # 2452 + None, # 2453 + None, # 2454 + None, # 2455 + None, # 2456 + None, # 2457 + None, # 2458 + None, # 2459 + None, # 2460 + None, # 2461 + None, # 2462 + None, # 2463 + None, # 2464 + None, # 2465 + None, # 2466 + None, # 2467 + None, # 2468 + None, # 2469 + None, # 2470 + None, # 2471 + None, # 2472 + None, # 2473 + None, # 2474 + None, # 2475 + None, # 2476 + None, # 2477 + None, # 2478 + None, # 2479 + None, # 2480 + None, # 2481 + None, # 2482 + None, # 2483 + None, # 2484 + None, # 2485 + None, # 2486 + None, # 2487 + None, # 2488 + None, # 2489 + None, # 2490 + None, # 2491 + None, # 2492 + None, # 2493 + None, # 2494 + None, # 2495 + None, # 2496 + None, # 2497 + None, # 2498 + None, # 2499 + None, # 2500 + None, # 2501 + None, # 2502 + None, # 2503 + None, # 2504 + None, # 2505 + None, # 2506 + None, # 2507 + None, # 2508 + None, # 2509 + None, # 2510 + None, # 2511 + None, # 2512 + None, # 2513 + None, # 2514 + None, # 2515 + None, # 2516 + None, # 2517 + None, # 2518 + None, # 2519 + None, # 2520 + None, # 2521 + None, # 2522 + None, # 2523 + None, # 2524 + None, # 2525 + None, # 2526 + None, # 2527 + None, # 2528 + None, # 2529 + None, # 2530 + None, # 2531 + None, # 2532 + None, # 2533 + None, # 2534 + None, # 2535 + None, # 2536 + None, # 2537 + None, # 2538 + None, # 2539 + None, # 2540 + None, # 2541 + None, # 2542 + None, # 2543 + None, # 2544 + None, # 2545 + None, # 2546 + None, # 2547 + None, # 2548 + None, # 2549 + None, # 2550 + None, # 2551 + None, # 2552 + None, # 2553 + None, # 2554 + None, # 2555 + None, # 2556 + None, # 2557 + None, # 2558 + None, # 2559 + None, # 2560 + None, # 2561 + None, # 2562 + None, # 2563 + None, # 2564 + None, # 2565 + None, # 2566 + None, # 2567 + None, # 2568 + None, # 2569 + None, # 2570 + None, # 2571 + None, # 2572 + None, # 2573 + None, # 2574 + None, # 2575 + None, # 2576 + None, # 2577 + None, # 2578 + None, # 2579 + None, # 2580 + None, # 2581 + None, # 2582 + None, # 2583 + None, # 2584 + None, # 2585 + None, # 2586 + None, # 2587 + None, # 2588 + None, # 2589 + None, # 2590 + None, # 2591 + None, # 2592 + None, # 2593 + None, # 2594 + None, # 2595 + None, # 2596 + None, # 2597 + None, # 2598 + None, # 2599 + None, # 2600 + None, # 2601 + None, # 2602 + None, # 2603 + None, # 2604 + None, # 2605 + None, # 2606 + None, # 2607 + None, # 2608 + None, # 2609 + None, # 2610 + None, # 2611 + None, # 2612 + None, # 2613 + None, # 2614 + None, # 2615 + None, # 2616 + None, # 2617 + None, # 2618 + None, # 2619 + None, # 2620 + None, # 2621 + None, # 2622 + None, # 2623 + None, # 2624 + None, # 2625 + None, # 2626 + None, # 2627 + None, # 2628 + None, # 2629 + None, # 2630 + None, # 2631 + None, # 2632 + None, # 2633 + None, # 2634 + None, # 2635 + None, # 2636 + None, # 2637 + None, # 2638 + None, # 2639 + None, # 2640 + None, # 2641 + None, # 2642 + None, # 2643 + None, # 2644 + None, # 2645 + None, # 2646 + None, # 2647 + None, # 2648 + None, # 2649 + None, # 2650 + None, # 2651 + None, # 2652 + None, # 2653 + None, # 2654 + None, # 2655 + None, # 2656 + None, # 2657 + None, # 2658 + None, # 2659 + None, # 2660 + None, # 2661 + None, # 2662 + None, # 2663 + None, # 2664 + None, # 2665 + None, # 2666 + None, # 2667 + None, # 2668 + None, # 2669 + None, # 2670 + None, # 2671 + None, # 2672 + None, # 2673 + None, # 2674 + None, # 2675 + None, # 2676 + None, # 2677 + None, # 2678 + None, # 2679 + None, # 2680 + None, # 2681 + None, # 2682 + None, # 2683 + None, # 2684 + None, # 2685 + None, # 2686 + None, # 2687 + None, # 2688 + None, # 2689 + None, # 2690 + None, # 2691 + None, # 2692 + None, # 2693 + None, # 2694 + None, # 2695 + None, # 2696 + None, # 2697 + None, # 2698 + None, # 2699 + None, # 2700 + None, # 2701 + None, # 2702 + None, # 2703 + None, # 2704 + None, # 2705 + None, # 2706 + None, # 2707 + None, # 2708 + None, # 2709 + None, # 2710 + None, # 2711 + None, # 2712 + None, # 2713 + None, # 2714 + None, # 2715 + None, # 2716 + None, # 2717 + None, # 2718 + None, # 2719 + None, # 2720 + None, # 2721 + None, # 2722 + None, # 2723 + None, # 2724 + None, # 2725 + None, # 2726 + None, # 2727 + None, # 2728 + None, # 2729 + None, # 2730 + None, # 2731 + None, # 2732 + None, # 2733 + None, # 2734 + None, # 2735 + None, # 2736 + None, # 2737 + None, # 2738 + None, # 2739 + None, # 2740 + None, # 2741 + None, # 2742 + None, # 2743 + None, # 2744 + None, # 2745 + None, # 2746 + None, # 2747 + None, # 2748 + None, # 2749 + None, # 2750 + None, # 2751 + None, # 2752 + None, # 2753 + None, # 2754 + None, # 2755 + None, # 2756 + None, # 2757 + None, # 2758 + None, # 2759 + None, # 2760 + None, # 2761 + None, # 2762 + None, # 2763 + None, # 2764 + None, # 2765 + None, # 2766 + None, # 2767 + None, # 2768 + None, # 2769 + None, # 2770 + None, # 2771 + None, # 2772 + None, # 2773 + None, # 2774 + None, # 2775 + None, # 2776 + None, # 2777 + None, # 2778 + None, # 2779 + None, # 2780 + None, # 2781 + None, # 2782 + None, # 2783 + None, # 2784 + None, # 2785 + None, # 2786 + None, # 2787 + None, # 2788 + None, # 2789 + None, # 2790 + None, # 2791 + None, # 2792 + None, # 2793 + None, # 2794 + None, # 2795 + None, # 2796 + None, # 2797 + None, # 2798 + None, # 2799 + None, # 2800 + None, # 2801 + None, # 2802 + None, # 2803 + None, # 2804 + None, # 2805 + None, # 2806 + None, # 2807 + None, # 2808 + None, # 2809 + None, # 2810 + None, # 2811 + None, # 2812 + None, # 2813 + None, # 2814 + None, # 2815 + None, # 2816 + None, # 2817 + None, # 2818 + None, # 2819 + None, # 2820 + None, # 2821 + None, # 2822 + None, # 2823 + None, # 2824 + None, # 2825 + None, # 2826 + None, # 2827 + None, # 2828 + None, # 2829 + None, # 2830 + None, # 2831 + None, # 2832 + None, # 2833 + None, # 2834 + None, # 2835 + None, # 2836 + None, # 2837 + None, # 2838 + None, # 2839 + None, # 2840 + None, # 2841 + None, # 2842 + None, # 2843 + None, # 2844 + None, # 2845 + None, # 2846 + None, # 2847 + None, # 2848 + None, # 2849 + None, # 2850 + None, # 2851 + None, # 2852 + None, # 2853 + None, # 2854 + None, # 2855 + None, # 2856 + None, # 2857 + None, # 2858 + None, # 2859 + None, # 2860 + None, # 2861 + None, # 2862 + None, # 2863 + None, # 2864 + None, # 2865 + None, # 2866 + None, # 2867 + None, # 2868 + None, # 2869 + None, # 2870 + None, # 2871 + None, # 2872 + None, # 2873 + None, # 2874 + None, # 2875 + None, # 2876 + None, # 2877 + None, # 2878 + None, # 2879 + None, # 2880 + None, # 2881 + None, # 2882 + None, # 2883 + None, # 2884 + None, # 2885 + None, # 2886 + None, # 2887 + None, # 2888 + None, # 2889 + None, # 2890 + None, # 2891 + None, # 2892 + None, # 2893 + None, # 2894 + None, # 2895 + None, # 2896 + None, # 2897 + None, # 2898 + None, # 2899 + None, # 2900 + None, # 2901 + None, # 2902 + None, # 2903 + None, # 2904 + None, # 2905 + None, # 2906 + None, # 2907 + None, # 2908 + None, # 2909 + None, # 2910 + None, # 2911 + None, # 2912 + None, # 2913 + None, # 2914 + None, # 2915 + None, # 2916 + None, # 2917 + None, # 2918 + None, # 2919 + None, # 2920 + None, # 2921 + None, # 2922 + None, # 2923 + None, # 2924 + None, # 2925 + None, # 2926 + None, # 2927 + None, # 2928 + None, # 2929 + None, # 2930 + None, # 2931 + None, # 2932 + None, # 2933 + None, # 2934 + None, # 2935 + None, # 2936 + None, # 2937 + None, # 2938 + None, # 2939 + None, # 2940 + None, # 2941 + None, # 2942 + None, # 2943 + None, # 2944 + None, # 2945 + None, # 2946 + None, # 2947 + None, # 2948 + None, # 2949 + None, # 2950 + None, # 2951 + None, # 2952 + None, # 2953 + None, # 2954 + None, # 2955 + None, # 2956 + None, # 2957 + None, # 2958 + None, # 2959 + None, # 2960 + None, # 2961 + None, # 2962 + None, # 2963 + None, # 2964 + None, # 2965 + None, # 2966 + None, # 2967 + None, # 2968 + None, # 2969 + None, # 2970 + None, # 2971 + None, # 2972 + None, # 2973 + None, # 2974 + None, # 2975 + None, # 2976 + None, # 2977 + None, # 2978 + None, # 2979 + None, # 2980 + None, # 2981 + None, # 2982 + None, # 2983 + None, # 2984 + None, # 2985 + None, # 2986 + None, # 2987 + None, # 2988 + None, # 2989 + None, # 2990 + None, # 2991 + None, # 2992 + None, # 2993 + None, # 2994 + None, # 2995 + None, # 2996 + None, # 2997 + None, # 2998 + None, # 2999 + None, # 3000 + None, # 3001 + None, # 3002 + None, # 3003 + None, # 3004 + None, # 3005 + None, # 3006 + None, # 3007 + None, # 3008 + None, # 3009 + None, # 3010 + None, # 3011 + None, # 3012 + None, # 3013 + None, # 3014 + None, # 3015 + None, # 3016 + None, # 3017 + None, # 3018 + None, # 3019 + None, # 3020 + None, # 3021 + None, # 3022 + None, # 3023 + None, # 3024 + None, # 3025 + None, # 3026 + None, # 3027 + None, # 3028 + None, # 3029 + None, # 3030 + None, # 3031 + None, # 3032 + None, # 3033 + None, # 3034 + None, # 3035 + None, # 3036 + None, # 3037 + None, # 3038 + None, # 3039 + None, # 3040 + None, # 3041 + None, # 3042 + None, # 3043 + None, # 3044 + None, # 3045 + None, # 3046 + None, # 3047 + None, # 3048 + None, # 3049 + None, # 3050 + None, # 3051 + None, # 3052 + None, # 3053 + None, # 3054 + None, # 3055 + None, # 3056 + None, # 3057 + None, # 3058 + None, # 3059 + None, # 3060 + None, # 3061 + None, # 3062 + None, # 3063 + None, # 3064 + None, # 3065 + None, # 3066 + None, # 3067 + None, # 3068 + None, # 3069 + None, # 3070 + None, # 3071 + None, # 3072 + None, # 3073 + None, # 3074 + None, # 3075 + None, # 3076 + None, # 3077 + None, # 3078 + None, # 3079 + None, # 3080 + None, # 3081 + None, # 3082 + None, # 3083 + None, # 3084 + None, # 3085 + None, # 3086 + None, # 3087 + None, # 3088 + None, # 3089 + None, # 3090 + None, # 3091 + None, # 3092 + None, # 3093 + None, # 3094 + None, # 3095 + None, # 3096 + None, # 3097 + None, # 3098 + None, # 3099 + None, # 3100 + None, # 3101 + None, # 3102 + None, # 3103 + None, # 3104 + None, # 3105 + None, # 3106 + None, # 3107 + None, # 3108 + None, # 3109 + None, # 3110 + None, # 3111 + None, # 3112 + None, # 3113 + None, # 3114 + None, # 3115 + None, # 3116 + None, # 3117 + None, # 3118 + None, # 3119 + None, # 3120 + None, # 3121 + None, # 3122 + None, # 3123 + None, # 3124 + None, # 3125 + None, # 3126 + None, # 3127 + None, # 3128 + None, # 3129 + None, # 3130 + None, # 3131 + None, # 3132 + None, # 3133 + None, # 3134 + None, # 3135 + None, # 3136 + None, # 3137 + None, # 3138 + None, # 3139 + None, # 3140 + None, # 3141 + None, # 3142 + None, # 3143 + None, # 3144 + None, # 3145 + None, # 3146 + None, # 3147 + None, # 3148 + None, # 3149 + None, # 3150 + None, # 3151 + None, # 3152 + None, # 3153 + None, # 3154 + None, # 3155 + None, # 3156 + None, # 3157 + None, # 3158 + None, # 3159 + None, # 3160 + None, # 3161 + None, # 3162 + None, # 3163 + None, # 3164 + None, # 3165 + None, # 3166 + None, # 3167 + None, # 3168 + None, # 3169 + None, # 3170 + None, # 3171 + None, # 3172 + None, # 3173 + None, # 3174 + None, # 3175 + None, # 3176 + None, # 3177 + None, # 3178 + None, # 3179 + None, # 3180 + None, # 3181 + None, # 3182 + None, # 3183 + None, # 3184 + None, # 3185 + None, # 3186 + None, # 3187 + None, # 3188 + None, # 3189 + None, # 3190 + None, # 3191 + None, # 3192 + None, # 3193 + None, # 3194 + None, # 3195 + None, # 3196 + None, # 3197 + None, # 3198 + None, # 3199 + None, # 3200 + None, # 3201 + None, # 3202 + None, # 3203 + None, # 3204 + None, # 3205 + None, # 3206 + None, # 3207 + None, # 3208 + None, # 3209 + None, # 3210 + None, # 3211 + None, # 3212 + None, # 3213 + None, # 3214 + None, # 3215 + None, # 3216 + None, # 3217 + None, # 3218 + None, # 3219 + None, # 3220 + None, # 3221 + None, # 3222 + None, # 3223 + None, # 3224 + None, # 3225 + None, # 3226 + None, # 3227 + None, # 3228 + None, # 3229 + None, # 3230 + None, # 3231 + None, # 3232 + None, # 3233 + None, # 3234 + None, # 3235 + None, # 3236 + None, # 3237 + None, # 3238 + None, # 3239 + None, # 3240 + None, # 3241 + None, # 3242 + None, # 3243 + None, # 3244 + None, # 3245 + None, # 3246 + None, # 3247 + None, # 3248 + None, # 3249 + None, # 3250 + None, # 3251 + None, # 3252 + None, # 3253 + None, # 3254 + None, # 3255 + None, # 3256 + None, # 3257 + None, # 3258 + None, # 3259 + None, # 3260 + None, # 3261 + None, # 3262 + None, # 3263 + None, # 3264 + None, # 3265 + None, # 3266 + None, # 3267 + None, # 3268 + None, # 3269 + None, # 3270 + None, # 3271 + None, # 3272 + None, # 3273 + None, # 3274 + None, # 3275 + None, # 3276 + None, # 3277 + None, # 3278 + None, # 3279 + None, # 3280 + None, # 3281 + None, # 3282 + None, # 3283 + None, # 3284 + None, # 3285 + None, # 3286 + None, # 3287 + None, # 3288 + None, # 3289 + None, # 3290 + None, # 3291 + None, # 3292 + None, # 3293 + None, # 3294 + None, # 3295 + None, # 3296 + None, # 3297 + None, # 3298 + None, # 3299 + None, # 3300 + None, # 3301 + None, # 3302 + None, # 3303 + None, # 3304 + None, # 3305 + None, # 3306 + None, # 3307 + None, # 3308 + None, # 3309 + None, # 3310 + None, # 3311 + None, # 3312 + None, # 3313 + None, # 3314 + None, # 3315 + None, # 3316 + None, # 3317 + None, # 3318 + None, # 3319 + None, # 3320 + None, # 3321 + None, # 3322 + None, # 3323 + None, # 3324 + None, # 3325 + None, # 3326 + None, # 3327 + None, # 3328 + (3329, TType.LIST, 'cloudFetchResults', (TType.STRUCT, [TDBSqlCloudResultFile, None], False), None, ), # 3329 +) +all_structs.append(TDBSqlTempView) +TDBSqlTempView.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'name', 'UTF8', None, ), # 1 + (2, TType.STRING, 'sqlStatement', 'UTF8', None, ), # 2 + (3, TType.MAP, 'properties', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 + (4, TType.STRING, 'viewSchema', 'UTF8', None, ), # 4 +) +all_structs.append(TDBSqlSessionCapabilities) +TDBSqlSessionCapabilities.thrift_spec = ( + None, # 0 + (1, TType.BOOL, 'supportsMultipleCatalogs', None, None, ), # 1 +) +all_structs.append(TExpressionInfo) +TExpressionInfo.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'className', 'UTF8', None, ), # 1 + (2, TType.STRING, 'usage', 'UTF8', None, ), # 2 + (3, TType.STRING, 'name', 'UTF8', None, ), # 3 + (4, TType.STRING, 'extended', 'UTF8', None, ), # 4 + (5, TType.STRING, 'db', 'UTF8', None, ), # 5 + (6, TType.STRING, 'arguments', 'UTF8', None, ), # 6 + (7, TType.STRING, 'examples', 'UTF8', None, ), # 7 + (8, TType.STRING, 'note', 'UTF8', None, ), # 8 + (9, TType.STRING, 'group', 'UTF8', None, ), # 9 + (10, TType.STRING, 'since', 'UTF8', None, ), # 10 + (11, TType.STRING, 'deprecated', 'UTF8', None, ), # 11 + (12, TType.STRING, 'source', 'UTF8', None, ), # 12 +) +all_structs.append(TDBSqlConfValue) +TDBSqlConfValue.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'value', 'UTF8', None, ), # 1 +) +all_structs.append(TDBSqlSessionConf) +TDBSqlSessionConf.thrift_spec = ( + None, # 0 + (1, TType.MAP, 'confs', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 1 + (2, TType.LIST, 'tempViews', (TType.STRUCT, [TDBSqlTempView, None], False), None, ), # 2 + (3, TType.STRING, 'currentDatabase', 'UTF8', None, ), # 3 + (4, TType.STRING, 'currentCatalog', 'UTF8', None, ), # 4 + (5, TType.STRUCT, 'sessionCapabilities', [TDBSqlSessionCapabilities, None], None, ), # 5 + (6, TType.LIST, 'expressionsInfos', (TType.STRUCT, [TExpressionInfo, None], False), None, ), # 6 + (7, TType.MAP, 'internalConfs', (TType.STRING, 'UTF8', TType.STRUCT, [TDBSqlConfValue, None], False), None, ), # 7 +) +all_structs.append(TStatus) +TStatus.thrift_spec = ( + None, # 0 + (1, TType.I32, 'statusCode', None, None, ), # 1 + (2, TType.LIST, 'infoMessages', (TType.STRING, 'UTF8', False), None, ), # 2 + (3, TType.STRING, 'sqlState', 'UTF8', None, ), # 3 + (4, TType.I32, 'errorCode', None, None, ), # 4 + (5, TType.STRING, 'errorMessage', 'UTF8', None, ), # 5 + (6, TType.STRING, 'displayMessage', 'UTF8', None, ), # 6 + None, # 7 + None, # 8 + None, # 9 + None, # 10 + None, # 11 + None, # 12 + None, # 13 + None, # 14 + None, # 15 + None, # 16 + None, # 17 + None, # 18 + None, # 19 + None, # 20 + None, # 21 + None, # 22 + None, # 23 + None, # 24 + None, # 25 + None, # 26 + None, # 27 + None, # 28 + None, # 29 + None, # 30 + None, # 31 + None, # 32 + None, # 33 + None, # 34 + None, # 35 + None, # 36 + None, # 37 + None, # 38 + None, # 39 + None, # 40 + None, # 41 + None, # 42 + None, # 43 + None, # 44 + None, # 45 + None, # 46 + None, # 47 + None, # 48 + None, # 49 + None, # 50 + None, # 51 + None, # 52 + None, # 53 + None, # 54 + None, # 55 + None, # 56 + None, # 57 + None, # 58 + None, # 59 + None, # 60 + None, # 61 + None, # 62 + None, # 63 + None, # 64 + None, # 65 + None, # 66 + None, # 67 + None, # 68 + None, # 69 + None, # 70 + None, # 71 + None, # 72 + None, # 73 + None, # 74 + None, # 75 + None, # 76 + None, # 77 + None, # 78 + None, # 79 + None, # 80 + None, # 81 + None, # 82 + None, # 83 + None, # 84 + None, # 85 + None, # 86 + None, # 87 + None, # 88 + None, # 89 + None, # 90 + None, # 91 + None, # 92 + None, # 93 + None, # 94 + None, # 95 + None, # 96 + None, # 97 + None, # 98 + None, # 99 + None, # 100 + None, # 101 + None, # 102 + None, # 103 + None, # 104 + None, # 105 + None, # 106 + None, # 107 + None, # 108 + None, # 109 + None, # 110 + None, # 111 + None, # 112 + None, # 113 + None, # 114 + None, # 115 + None, # 116 + None, # 117 + None, # 118 + None, # 119 + None, # 120 + None, # 121 + None, # 122 + None, # 123 + None, # 124 + None, # 125 + None, # 126 + None, # 127 + None, # 128 + None, # 129 + None, # 130 + None, # 131 + None, # 132 + None, # 133 + None, # 134 + None, # 135 + None, # 136 + None, # 137 + None, # 138 + None, # 139 + None, # 140 + None, # 141 + None, # 142 + None, # 143 + None, # 144 + None, # 145 + None, # 146 + None, # 147 + None, # 148 + None, # 149 + None, # 150 + None, # 151 + None, # 152 + None, # 153 + None, # 154 + None, # 155 + None, # 156 + None, # 157 + None, # 158 + None, # 159 + None, # 160 + None, # 161 + None, # 162 + None, # 163 + None, # 164 + None, # 165 + None, # 166 + None, # 167 + None, # 168 + None, # 169 + None, # 170 + None, # 171 + None, # 172 + None, # 173 + None, # 174 + None, # 175 + None, # 176 + None, # 177 + None, # 178 + None, # 179 + None, # 180 + None, # 181 + None, # 182 + None, # 183 + None, # 184 + None, # 185 + None, # 186 + None, # 187 + None, # 188 + None, # 189 + None, # 190 + None, # 191 + None, # 192 + None, # 193 + None, # 194 + None, # 195 + None, # 196 + None, # 197 + None, # 198 + None, # 199 + None, # 200 + None, # 201 + None, # 202 + None, # 203 + None, # 204 + None, # 205 + None, # 206 + None, # 207 + None, # 208 + None, # 209 + None, # 210 + None, # 211 + None, # 212 + None, # 213 + None, # 214 + None, # 215 + None, # 216 + None, # 217 + None, # 218 + None, # 219 + None, # 220 + None, # 221 + None, # 222 + None, # 223 + None, # 224 + None, # 225 + None, # 226 + None, # 227 + None, # 228 + None, # 229 + None, # 230 + None, # 231 + None, # 232 + None, # 233 + None, # 234 + None, # 235 + None, # 236 + None, # 237 + None, # 238 + None, # 239 + None, # 240 + None, # 241 + None, # 242 + None, # 243 + None, # 244 + None, # 245 + None, # 246 + None, # 247 + None, # 248 + None, # 249 + None, # 250 + None, # 251 + None, # 252 + None, # 253 + None, # 254 + None, # 255 + None, # 256 + None, # 257 + None, # 258 + None, # 259 + None, # 260 + None, # 261 + None, # 262 + None, # 263 + None, # 264 + None, # 265 + None, # 266 + None, # 267 + None, # 268 + None, # 269 + None, # 270 + None, # 271 + None, # 272 + None, # 273 + None, # 274 + None, # 275 + None, # 276 + None, # 277 + None, # 278 + None, # 279 + None, # 280 + None, # 281 + None, # 282 + None, # 283 + None, # 284 + None, # 285 + None, # 286 + None, # 287 + None, # 288 + None, # 289 + None, # 290 + None, # 291 + None, # 292 + None, # 293 + None, # 294 + None, # 295 + None, # 296 + None, # 297 + None, # 298 + None, # 299 + None, # 300 + None, # 301 + None, # 302 + None, # 303 + None, # 304 + None, # 305 + None, # 306 + None, # 307 + None, # 308 + None, # 309 + None, # 310 + None, # 311 + None, # 312 + None, # 313 + None, # 314 + None, # 315 + None, # 316 + None, # 317 + None, # 318 + None, # 319 + None, # 320 + None, # 321 + None, # 322 + None, # 323 + None, # 324 + None, # 325 + None, # 326 + None, # 327 + None, # 328 + None, # 329 + None, # 330 + None, # 331 + None, # 332 + None, # 333 + None, # 334 + None, # 335 + None, # 336 + None, # 337 + None, # 338 + None, # 339 + None, # 340 + None, # 341 + None, # 342 + None, # 343 + None, # 344 + None, # 345 + None, # 346 + None, # 347 + None, # 348 + None, # 349 + None, # 350 + None, # 351 + None, # 352 + None, # 353 + None, # 354 + None, # 355 + None, # 356 + None, # 357 + None, # 358 + None, # 359 + None, # 360 + None, # 361 + None, # 362 + None, # 363 + None, # 364 + None, # 365 + None, # 366 + None, # 367 + None, # 368 + None, # 369 + None, # 370 + None, # 371 + None, # 372 + None, # 373 + None, # 374 + None, # 375 + None, # 376 + None, # 377 + None, # 378 + None, # 379 + None, # 380 + None, # 381 + None, # 382 + None, # 383 + None, # 384 + None, # 385 + None, # 386 + None, # 387 + None, # 388 + None, # 389 + None, # 390 + None, # 391 + None, # 392 + None, # 393 + None, # 394 + None, # 395 + None, # 396 + None, # 397 + None, # 398 + None, # 399 + None, # 400 + None, # 401 + None, # 402 + None, # 403 + None, # 404 + None, # 405 + None, # 406 + None, # 407 + None, # 408 + None, # 409 + None, # 410 + None, # 411 + None, # 412 + None, # 413 + None, # 414 + None, # 415 + None, # 416 + None, # 417 + None, # 418 + None, # 419 + None, # 420 + None, # 421 + None, # 422 + None, # 423 + None, # 424 + None, # 425 + None, # 426 + None, # 427 + None, # 428 + None, # 429 + None, # 430 + None, # 431 + None, # 432 + None, # 433 + None, # 434 + None, # 435 + None, # 436 + None, # 437 + None, # 438 + None, # 439 + None, # 440 + None, # 441 + None, # 442 + None, # 443 + None, # 444 + None, # 445 + None, # 446 + None, # 447 + None, # 448 + None, # 449 + None, # 450 + None, # 451 + None, # 452 + None, # 453 + None, # 454 + None, # 455 None, # 456 None, # 457 None, # 458 @@ -12726,7 +15385,7 @@ def __ne__(self, other): None, # 1278 None, # 1279 None, # 1280 - None, # 1281 + (1281, TType.STRING, 'errorDetailsJson', 'UTF8', None, ), # 1281 None, # 1282 None, # 1283 None, # 1284 @@ -29458,6 +32117,7 @@ def __ne__(self, other): (2, TType.BOOL, 'decimalAsArrow', None, None, ), # 2 (3, TType.BOOL, 'complexTypesAsArrow', None, None, ), # 3 (4, TType.BOOL, 'intervalTypesAsArrow', None, None, ), # 4 + (5, TType.BOOL, 'nullTypeAsArrow', None, None, ), # 5 ) all_structs.append(TExecuteStatementReq) TExecuteStatementReq.thrift_spec = ( @@ -30749,15 +33409,15 @@ def __ne__(self, other): (1285, TType.I64, 'maxBytesPerFile', None, None, ), # 1285 (1286, TType.STRUCT, 'useArrowNativeTypes', [TSparkArrowTypes, None], None, ), # 1286 (1287, TType.I64, 'resultRowLimit', None, None, ), # 1287 - None, # 1288 - None, # 1289 + (1288, TType.LIST, 'parameters', (TType.STRUCT, [TSparkParameter, None], False), None, ), # 1288 + (1289, TType.I64, 'maxBytesPerBatch', None, None, ), # 1289 None, # 1290 None, # 1291 None, # 1292 None, # 1293 None, # 1294 None, # 1295 - None, # 1296 + (1296, TType.STRUCT, 'statementConf', [TStatementConf, None], None, ), # 1296 None, # 1297 None, # 1298 None, # 1299 @@ -32798,6 +35458,59 @@ def __ne__(self, other): (3334, TType.STRING, 'requestValidation', 'BINARY', None, ), # 3334 (3335, TType.I32, 'resultPersistenceMode', None, None, ), # 3335 (3336, TType.BOOL, 'trimArrowBatchesToLimit', None, None, ), # 3336 + (3337, TType.I32, 'fetchDisposition', None, None, ), # 3337 + None, # 3338 + None, # 3339 + None, # 3340 + None, # 3341 + None, # 3342 + None, # 3343 + (3344, TType.BOOL, 'enforceResultPersistenceMode', None, None, ), # 3344 + (3345, TType.LIST, 'statementList', (TType.STRUCT, [TDBSqlStatement, None], False), None, ), # 3345 + (3346, TType.BOOL, 'persistResultManifest', None, None, ), # 3346 + (3347, TType.I64, 'resultRetentionSeconds', None, None, ), # 3347 + (3348, TType.I64, 'resultByteLimit', None, None, ), # 3348 + (3349, TType.STRUCT, 'resultDataFormat', [TDBSqlResultFormat, None], None, ), # 3349 + (3350, TType.STRING, 'originatingClientIdentity', 'UTF8', None, ), # 3350 + (3351, TType.BOOL, 'preferSingleFileResult', None, None, ), # 3351 + (3352, TType.BOOL, 'preferDriverOnlyUpload', None, None, ), # 3352 + (3353, TType.BOOL, 'enforceEmbeddedSchemaCorrectness', None, False, ), # 3353 + None, # 3354 + None, # 3355 + None, # 3356 + None, # 3357 + None, # 3358 + None, # 3359 + (3360, TType.STRING, 'idempotencyToken', 'UTF8', None, ), # 3360 + (3361, TType.BOOL, 'throwErrorOnByteLimitTruncation', None, None, ), # 3361 +) +all_structs.append(TDBSqlStatement) +TDBSqlStatement.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'statement', 'UTF8', None, ), # 1 +) +all_structs.append(TSparkParameterValue) +TSparkParameterValue.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'stringValue', 'UTF8', None, ), # 1 + (2, TType.DOUBLE, 'doubleValue', None, None, ), # 2 + (3, TType.BOOL, 'booleanValue', None, None, ), # 3 +) +all_structs.append(TSparkParameter) +TSparkParameter.thrift_spec = ( + None, # 0 + (1, TType.I32, 'ordinal', None, None, ), # 1 + (2, TType.STRING, 'name', 'UTF8', None, ), # 2 + (3, TType.STRING, 'type', 'UTF8', None, ), # 3 + (4, TType.STRUCT, 'value', [TSparkParameterValue, None], None, ), # 4 +) +all_structs.append(TStatementConf) +TStatementConf.thrift_spec = ( + None, # 0 + (1, TType.BOOL, 'sessionless', None, None, ), # 1 + (2, TType.STRUCT, 'initialNamespace', [TNamespace, None], None, ), # 2 + (3, TType.I32, 'client_protocol', None, None, ), # 3 + (4, TType.I64, 'client_protocol_i64', None, None, ), # 4 ) all_structs.append(TExecuteStatementResp) TExecuteStatementResp.thrift_spec = ( @@ -36136,6 +38849,9 @@ def __ne__(self, other): (3332, TType.STRUCT, 'sessionConf', [TDBSqlSessionConf, None], None, ), # 3332 (3333, TType.DOUBLE, 'currentClusterLoad', None, None, ), # 3333 (3334, TType.I32, 'idempotencyType', None, None, ), # 3334 + (3335, TType.BOOL, 'remoteResultCacheEnabled', None, None, ), # 3335 + (3336, TType.BOOL, 'isServerless', None, None, ), # 3336 + (3337, TType.LIST, 'operationHandles', (TType.STRUCT, [TOperationHandle, None], False), None, ), # 3337 ) all_structs.append(TGetTypeInfoReq) TGetTypeInfoReq.thrift_spec = ( @@ -76423,20 +79139,1311 @@ def __ne__(self, other): (3329, TType.STRUCT, 'operationId', [THandleIdentifier, None], None, ), # 3329 (3330, TType.STRUCT, 'sessionConf', [TDBSqlSessionConf, None], None, ), # 3330 ) -all_structs.append(TGetCrossReferenceResp) -TGetCrossReferenceResp.thrift_spec = ( +all_structs.append(TGetCrossReferenceResp) +TGetCrossReferenceResp.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 + (2, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 2 + None, # 3 + None, # 4 + None, # 5 + None, # 6 + None, # 7 + None, # 8 + None, # 9 + None, # 10 + None, # 11 + None, # 12 + None, # 13 + None, # 14 + None, # 15 + None, # 16 + None, # 17 + None, # 18 + None, # 19 + None, # 20 + None, # 21 + None, # 22 + None, # 23 + None, # 24 + None, # 25 + None, # 26 + None, # 27 + None, # 28 + None, # 29 + None, # 30 + None, # 31 + None, # 32 + None, # 33 + None, # 34 + None, # 35 + None, # 36 + None, # 37 + None, # 38 + None, # 39 + None, # 40 + None, # 41 + None, # 42 + None, # 43 + None, # 44 + None, # 45 + None, # 46 + None, # 47 + None, # 48 + None, # 49 + None, # 50 + None, # 51 + None, # 52 + None, # 53 + None, # 54 + None, # 55 + None, # 56 + None, # 57 + None, # 58 + None, # 59 + None, # 60 + None, # 61 + None, # 62 + None, # 63 + None, # 64 + None, # 65 + None, # 66 + None, # 67 + None, # 68 + None, # 69 + None, # 70 + None, # 71 + None, # 72 + None, # 73 + None, # 74 + None, # 75 + None, # 76 + None, # 77 + None, # 78 + None, # 79 + None, # 80 + None, # 81 + None, # 82 + None, # 83 + None, # 84 + None, # 85 + None, # 86 + None, # 87 + None, # 88 + None, # 89 + None, # 90 + None, # 91 + None, # 92 + None, # 93 + None, # 94 + None, # 95 + None, # 96 + None, # 97 + None, # 98 + None, # 99 + None, # 100 + None, # 101 + None, # 102 + None, # 103 + None, # 104 + None, # 105 + None, # 106 + None, # 107 + None, # 108 + None, # 109 + None, # 110 + None, # 111 + None, # 112 + None, # 113 + None, # 114 + None, # 115 + None, # 116 + None, # 117 + None, # 118 + None, # 119 + None, # 120 + None, # 121 + None, # 122 + None, # 123 + None, # 124 + None, # 125 + None, # 126 + None, # 127 + None, # 128 + None, # 129 + None, # 130 + None, # 131 + None, # 132 + None, # 133 + None, # 134 + None, # 135 + None, # 136 + None, # 137 + None, # 138 + None, # 139 + None, # 140 + None, # 141 + None, # 142 + None, # 143 + None, # 144 + None, # 145 + None, # 146 + None, # 147 + None, # 148 + None, # 149 + None, # 150 + None, # 151 + None, # 152 + None, # 153 + None, # 154 + None, # 155 + None, # 156 + None, # 157 + None, # 158 + None, # 159 + None, # 160 + None, # 161 + None, # 162 + None, # 163 + None, # 164 + None, # 165 + None, # 166 + None, # 167 + None, # 168 + None, # 169 + None, # 170 + None, # 171 + None, # 172 + None, # 173 + None, # 174 + None, # 175 + None, # 176 + None, # 177 + None, # 178 + None, # 179 + None, # 180 + None, # 181 + None, # 182 + None, # 183 + None, # 184 + None, # 185 + None, # 186 + None, # 187 + None, # 188 + None, # 189 + None, # 190 + None, # 191 + None, # 192 + None, # 193 + None, # 194 + None, # 195 + None, # 196 + None, # 197 + None, # 198 + None, # 199 + None, # 200 + None, # 201 + None, # 202 + None, # 203 + None, # 204 + None, # 205 + None, # 206 + None, # 207 + None, # 208 + None, # 209 + None, # 210 + None, # 211 + None, # 212 + None, # 213 + None, # 214 + None, # 215 + None, # 216 + None, # 217 + None, # 218 + None, # 219 + None, # 220 + None, # 221 + None, # 222 + None, # 223 + None, # 224 + None, # 225 + None, # 226 + None, # 227 + None, # 228 + None, # 229 + None, # 230 + None, # 231 + None, # 232 + None, # 233 + None, # 234 + None, # 235 + None, # 236 + None, # 237 + None, # 238 + None, # 239 + None, # 240 + None, # 241 + None, # 242 + None, # 243 + None, # 244 + None, # 245 + None, # 246 + None, # 247 + None, # 248 + None, # 249 + None, # 250 + None, # 251 + None, # 252 + None, # 253 + None, # 254 + None, # 255 + None, # 256 + None, # 257 + None, # 258 + None, # 259 + None, # 260 + None, # 261 + None, # 262 + None, # 263 + None, # 264 + None, # 265 + None, # 266 + None, # 267 + None, # 268 + None, # 269 + None, # 270 + None, # 271 + None, # 272 + None, # 273 + None, # 274 + None, # 275 + None, # 276 + None, # 277 + None, # 278 + None, # 279 + None, # 280 + None, # 281 + None, # 282 + None, # 283 + None, # 284 + None, # 285 + None, # 286 + None, # 287 + None, # 288 + None, # 289 + None, # 290 + None, # 291 + None, # 292 + None, # 293 + None, # 294 + None, # 295 + None, # 296 + None, # 297 + None, # 298 + None, # 299 + None, # 300 + None, # 301 + None, # 302 + None, # 303 + None, # 304 + None, # 305 + None, # 306 + None, # 307 + None, # 308 + None, # 309 + None, # 310 + None, # 311 + None, # 312 + None, # 313 + None, # 314 + None, # 315 + None, # 316 + None, # 317 + None, # 318 + None, # 319 + None, # 320 + None, # 321 + None, # 322 + None, # 323 + None, # 324 + None, # 325 + None, # 326 + None, # 327 + None, # 328 + None, # 329 + None, # 330 + None, # 331 + None, # 332 + None, # 333 + None, # 334 + None, # 335 + None, # 336 + None, # 337 + None, # 338 + None, # 339 + None, # 340 + None, # 341 + None, # 342 + None, # 343 + None, # 344 + None, # 345 + None, # 346 + None, # 347 + None, # 348 + None, # 349 + None, # 350 + None, # 351 + None, # 352 + None, # 353 + None, # 354 + None, # 355 + None, # 356 + None, # 357 + None, # 358 + None, # 359 + None, # 360 + None, # 361 + None, # 362 + None, # 363 + None, # 364 + None, # 365 + None, # 366 + None, # 367 + None, # 368 + None, # 369 + None, # 370 + None, # 371 + None, # 372 + None, # 373 + None, # 374 + None, # 375 + None, # 376 + None, # 377 + None, # 378 + None, # 379 + None, # 380 + None, # 381 + None, # 382 + None, # 383 + None, # 384 + None, # 385 + None, # 386 + None, # 387 + None, # 388 + None, # 389 + None, # 390 + None, # 391 + None, # 392 + None, # 393 + None, # 394 + None, # 395 + None, # 396 + None, # 397 + None, # 398 + None, # 399 + None, # 400 + None, # 401 + None, # 402 + None, # 403 + None, # 404 + None, # 405 + None, # 406 + None, # 407 + None, # 408 + None, # 409 + None, # 410 + None, # 411 + None, # 412 + None, # 413 + None, # 414 + None, # 415 + None, # 416 + None, # 417 + None, # 418 + None, # 419 + None, # 420 + None, # 421 + None, # 422 + None, # 423 + None, # 424 + None, # 425 + None, # 426 + None, # 427 + None, # 428 + None, # 429 + None, # 430 + None, # 431 + None, # 432 + None, # 433 + None, # 434 + None, # 435 + None, # 436 + None, # 437 + None, # 438 + None, # 439 + None, # 440 + None, # 441 + None, # 442 + None, # 443 + None, # 444 + None, # 445 + None, # 446 + None, # 447 + None, # 448 + None, # 449 + None, # 450 + None, # 451 + None, # 452 + None, # 453 + None, # 454 + None, # 455 + None, # 456 + None, # 457 + None, # 458 + None, # 459 + None, # 460 + None, # 461 + None, # 462 + None, # 463 + None, # 464 + None, # 465 + None, # 466 + None, # 467 + None, # 468 + None, # 469 + None, # 470 + None, # 471 + None, # 472 + None, # 473 + None, # 474 + None, # 475 + None, # 476 + None, # 477 + None, # 478 + None, # 479 + None, # 480 + None, # 481 + None, # 482 + None, # 483 + None, # 484 + None, # 485 + None, # 486 + None, # 487 + None, # 488 + None, # 489 + None, # 490 + None, # 491 + None, # 492 + None, # 493 + None, # 494 + None, # 495 + None, # 496 + None, # 497 + None, # 498 + None, # 499 + None, # 500 + None, # 501 + None, # 502 + None, # 503 + None, # 504 + None, # 505 + None, # 506 + None, # 507 + None, # 508 + None, # 509 + None, # 510 + None, # 511 + None, # 512 + None, # 513 + None, # 514 + None, # 515 + None, # 516 + None, # 517 + None, # 518 + None, # 519 + None, # 520 + None, # 521 + None, # 522 + None, # 523 + None, # 524 + None, # 525 + None, # 526 + None, # 527 + None, # 528 + None, # 529 + None, # 530 + None, # 531 + None, # 532 + None, # 533 + None, # 534 + None, # 535 + None, # 536 + None, # 537 + None, # 538 + None, # 539 + None, # 540 + None, # 541 + None, # 542 + None, # 543 + None, # 544 + None, # 545 + None, # 546 + None, # 547 + None, # 548 + None, # 549 + None, # 550 + None, # 551 + None, # 552 + None, # 553 + None, # 554 + None, # 555 + None, # 556 + None, # 557 + None, # 558 + None, # 559 + None, # 560 + None, # 561 + None, # 562 + None, # 563 + None, # 564 + None, # 565 + None, # 566 + None, # 567 + None, # 568 + None, # 569 + None, # 570 + None, # 571 + None, # 572 + None, # 573 + None, # 574 + None, # 575 + None, # 576 + None, # 577 + None, # 578 + None, # 579 + None, # 580 + None, # 581 + None, # 582 + None, # 583 + None, # 584 + None, # 585 + None, # 586 + None, # 587 + None, # 588 + None, # 589 + None, # 590 + None, # 591 + None, # 592 + None, # 593 + None, # 594 + None, # 595 + None, # 596 + None, # 597 + None, # 598 + None, # 599 + None, # 600 + None, # 601 + None, # 602 + None, # 603 + None, # 604 + None, # 605 + None, # 606 + None, # 607 + None, # 608 + None, # 609 + None, # 610 + None, # 611 + None, # 612 + None, # 613 + None, # 614 + None, # 615 + None, # 616 + None, # 617 + None, # 618 + None, # 619 + None, # 620 + None, # 621 + None, # 622 + None, # 623 + None, # 624 + None, # 625 + None, # 626 + None, # 627 + None, # 628 + None, # 629 + None, # 630 + None, # 631 + None, # 632 + None, # 633 + None, # 634 + None, # 635 + None, # 636 + None, # 637 + None, # 638 + None, # 639 + None, # 640 + None, # 641 + None, # 642 + None, # 643 + None, # 644 + None, # 645 + None, # 646 + None, # 647 + None, # 648 + None, # 649 + None, # 650 + None, # 651 + None, # 652 + None, # 653 + None, # 654 + None, # 655 + None, # 656 + None, # 657 + None, # 658 + None, # 659 + None, # 660 + None, # 661 + None, # 662 + None, # 663 + None, # 664 + None, # 665 + None, # 666 + None, # 667 + None, # 668 + None, # 669 + None, # 670 + None, # 671 + None, # 672 + None, # 673 + None, # 674 + None, # 675 + None, # 676 + None, # 677 + None, # 678 + None, # 679 + None, # 680 + None, # 681 + None, # 682 + None, # 683 + None, # 684 + None, # 685 + None, # 686 + None, # 687 + None, # 688 + None, # 689 + None, # 690 + None, # 691 + None, # 692 + None, # 693 + None, # 694 + None, # 695 + None, # 696 + None, # 697 + None, # 698 + None, # 699 + None, # 700 + None, # 701 + None, # 702 + None, # 703 + None, # 704 + None, # 705 + None, # 706 + None, # 707 + None, # 708 + None, # 709 + None, # 710 + None, # 711 + None, # 712 + None, # 713 + None, # 714 + None, # 715 + None, # 716 + None, # 717 + None, # 718 + None, # 719 + None, # 720 + None, # 721 + None, # 722 + None, # 723 + None, # 724 + None, # 725 + None, # 726 + None, # 727 + None, # 728 + None, # 729 + None, # 730 + None, # 731 + None, # 732 + None, # 733 + None, # 734 + None, # 735 + None, # 736 + None, # 737 + None, # 738 + None, # 739 + None, # 740 + None, # 741 + None, # 742 + None, # 743 + None, # 744 + None, # 745 + None, # 746 + None, # 747 + None, # 748 + None, # 749 + None, # 750 + None, # 751 + None, # 752 + None, # 753 + None, # 754 + None, # 755 + None, # 756 + None, # 757 + None, # 758 + None, # 759 + None, # 760 + None, # 761 + None, # 762 + None, # 763 + None, # 764 + None, # 765 + None, # 766 + None, # 767 + None, # 768 + None, # 769 + None, # 770 + None, # 771 + None, # 772 + None, # 773 + None, # 774 + None, # 775 + None, # 776 + None, # 777 + None, # 778 + None, # 779 + None, # 780 + None, # 781 + None, # 782 + None, # 783 + None, # 784 + None, # 785 + None, # 786 + None, # 787 + None, # 788 + None, # 789 + None, # 790 + None, # 791 + None, # 792 + None, # 793 + None, # 794 + None, # 795 + None, # 796 + None, # 797 + None, # 798 + None, # 799 + None, # 800 + None, # 801 + None, # 802 + None, # 803 + None, # 804 + None, # 805 + None, # 806 + None, # 807 + None, # 808 + None, # 809 + None, # 810 + None, # 811 + None, # 812 + None, # 813 + None, # 814 + None, # 815 + None, # 816 + None, # 817 + None, # 818 + None, # 819 + None, # 820 + None, # 821 + None, # 822 + None, # 823 + None, # 824 + None, # 825 + None, # 826 + None, # 827 + None, # 828 + None, # 829 + None, # 830 + None, # 831 + None, # 832 + None, # 833 + None, # 834 + None, # 835 + None, # 836 + None, # 837 + None, # 838 + None, # 839 + None, # 840 + None, # 841 + None, # 842 + None, # 843 + None, # 844 + None, # 845 + None, # 846 + None, # 847 + None, # 848 + None, # 849 + None, # 850 + None, # 851 + None, # 852 + None, # 853 + None, # 854 + None, # 855 + None, # 856 + None, # 857 + None, # 858 + None, # 859 + None, # 860 + None, # 861 + None, # 862 + None, # 863 + None, # 864 + None, # 865 + None, # 866 + None, # 867 + None, # 868 + None, # 869 + None, # 870 + None, # 871 + None, # 872 + None, # 873 + None, # 874 + None, # 875 + None, # 876 + None, # 877 + None, # 878 + None, # 879 + None, # 880 + None, # 881 + None, # 882 + None, # 883 + None, # 884 + None, # 885 + None, # 886 + None, # 887 + None, # 888 + None, # 889 + None, # 890 + None, # 891 + None, # 892 + None, # 893 + None, # 894 + None, # 895 + None, # 896 + None, # 897 + None, # 898 + None, # 899 + None, # 900 + None, # 901 + None, # 902 + None, # 903 + None, # 904 + None, # 905 + None, # 906 + None, # 907 + None, # 908 + None, # 909 + None, # 910 + None, # 911 + None, # 912 + None, # 913 + None, # 914 + None, # 915 + None, # 916 + None, # 917 + None, # 918 + None, # 919 + None, # 920 + None, # 921 + None, # 922 + None, # 923 + None, # 924 + None, # 925 + None, # 926 + None, # 927 + None, # 928 + None, # 929 + None, # 930 + None, # 931 + None, # 932 + None, # 933 + None, # 934 + None, # 935 + None, # 936 + None, # 937 + None, # 938 + None, # 939 + None, # 940 + None, # 941 + None, # 942 + None, # 943 + None, # 944 + None, # 945 + None, # 946 + None, # 947 + None, # 948 + None, # 949 + None, # 950 + None, # 951 + None, # 952 + None, # 953 + None, # 954 + None, # 955 + None, # 956 + None, # 957 + None, # 958 + None, # 959 + None, # 960 + None, # 961 + None, # 962 + None, # 963 + None, # 964 + None, # 965 + None, # 966 + None, # 967 + None, # 968 + None, # 969 + None, # 970 + None, # 971 + None, # 972 + None, # 973 + None, # 974 + None, # 975 + None, # 976 + None, # 977 + None, # 978 + None, # 979 + None, # 980 + None, # 981 + None, # 982 + None, # 983 + None, # 984 + None, # 985 + None, # 986 + None, # 987 + None, # 988 + None, # 989 + None, # 990 + None, # 991 + None, # 992 + None, # 993 + None, # 994 + None, # 995 + None, # 996 + None, # 997 + None, # 998 + None, # 999 + None, # 1000 + None, # 1001 + None, # 1002 + None, # 1003 + None, # 1004 + None, # 1005 + None, # 1006 + None, # 1007 + None, # 1008 + None, # 1009 + None, # 1010 + None, # 1011 + None, # 1012 + None, # 1013 + None, # 1014 + None, # 1015 + None, # 1016 + None, # 1017 + None, # 1018 + None, # 1019 + None, # 1020 + None, # 1021 + None, # 1022 + None, # 1023 + None, # 1024 + None, # 1025 + None, # 1026 + None, # 1027 + None, # 1028 + None, # 1029 + None, # 1030 + None, # 1031 + None, # 1032 + None, # 1033 + None, # 1034 + None, # 1035 + None, # 1036 + None, # 1037 + None, # 1038 + None, # 1039 + None, # 1040 + None, # 1041 + None, # 1042 + None, # 1043 + None, # 1044 + None, # 1045 + None, # 1046 + None, # 1047 + None, # 1048 + None, # 1049 + None, # 1050 + None, # 1051 + None, # 1052 + None, # 1053 + None, # 1054 + None, # 1055 + None, # 1056 + None, # 1057 + None, # 1058 + None, # 1059 + None, # 1060 + None, # 1061 + None, # 1062 + None, # 1063 + None, # 1064 + None, # 1065 + None, # 1066 + None, # 1067 + None, # 1068 + None, # 1069 + None, # 1070 + None, # 1071 + None, # 1072 + None, # 1073 + None, # 1074 + None, # 1075 + None, # 1076 + None, # 1077 + None, # 1078 + None, # 1079 + None, # 1080 + None, # 1081 + None, # 1082 + None, # 1083 + None, # 1084 + None, # 1085 + None, # 1086 + None, # 1087 + None, # 1088 + None, # 1089 + None, # 1090 + None, # 1091 + None, # 1092 + None, # 1093 + None, # 1094 + None, # 1095 + None, # 1096 + None, # 1097 + None, # 1098 + None, # 1099 + None, # 1100 + None, # 1101 + None, # 1102 + None, # 1103 + None, # 1104 + None, # 1105 + None, # 1106 + None, # 1107 + None, # 1108 + None, # 1109 + None, # 1110 + None, # 1111 + None, # 1112 + None, # 1113 + None, # 1114 + None, # 1115 + None, # 1116 + None, # 1117 + None, # 1118 + None, # 1119 + None, # 1120 + None, # 1121 + None, # 1122 + None, # 1123 + None, # 1124 + None, # 1125 + None, # 1126 + None, # 1127 + None, # 1128 + None, # 1129 + None, # 1130 + None, # 1131 + None, # 1132 + None, # 1133 + None, # 1134 + None, # 1135 + None, # 1136 + None, # 1137 + None, # 1138 + None, # 1139 + None, # 1140 + None, # 1141 + None, # 1142 + None, # 1143 + None, # 1144 + None, # 1145 + None, # 1146 + None, # 1147 + None, # 1148 + None, # 1149 + None, # 1150 + None, # 1151 + None, # 1152 + None, # 1153 + None, # 1154 + None, # 1155 + None, # 1156 + None, # 1157 + None, # 1158 + None, # 1159 + None, # 1160 + None, # 1161 + None, # 1162 + None, # 1163 + None, # 1164 + None, # 1165 + None, # 1166 + None, # 1167 + None, # 1168 + None, # 1169 + None, # 1170 + None, # 1171 + None, # 1172 + None, # 1173 + None, # 1174 + None, # 1175 + None, # 1176 + None, # 1177 + None, # 1178 + None, # 1179 + None, # 1180 + None, # 1181 + None, # 1182 + None, # 1183 + None, # 1184 + None, # 1185 + None, # 1186 + None, # 1187 + None, # 1188 + None, # 1189 + None, # 1190 + None, # 1191 + None, # 1192 + None, # 1193 + None, # 1194 + None, # 1195 + None, # 1196 + None, # 1197 + None, # 1198 + None, # 1199 + None, # 1200 + None, # 1201 + None, # 1202 + None, # 1203 + None, # 1204 + None, # 1205 + None, # 1206 + None, # 1207 + None, # 1208 + None, # 1209 + None, # 1210 + None, # 1211 + None, # 1212 + None, # 1213 + None, # 1214 + None, # 1215 + None, # 1216 + None, # 1217 + None, # 1218 + None, # 1219 + None, # 1220 + None, # 1221 + None, # 1222 + None, # 1223 + None, # 1224 + None, # 1225 + None, # 1226 + None, # 1227 + None, # 1228 + None, # 1229 + None, # 1230 + None, # 1231 + None, # 1232 + None, # 1233 + None, # 1234 + None, # 1235 + None, # 1236 + None, # 1237 + None, # 1238 + None, # 1239 + None, # 1240 + None, # 1241 + None, # 1242 + None, # 1243 + None, # 1244 + None, # 1245 + None, # 1246 + None, # 1247 + None, # 1248 + None, # 1249 + None, # 1250 + None, # 1251 + None, # 1252 + None, # 1253 + None, # 1254 + None, # 1255 + None, # 1256 + None, # 1257 + None, # 1258 + None, # 1259 + None, # 1260 + None, # 1261 + None, # 1262 + None, # 1263 + None, # 1264 + None, # 1265 + None, # 1266 + None, # 1267 + None, # 1268 + None, # 1269 + None, # 1270 + None, # 1271 + None, # 1272 + None, # 1273 + None, # 1274 + None, # 1275 + None, # 1276 + None, # 1277 + None, # 1278 + None, # 1279 + None, # 1280 + (1281, TType.STRUCT, 'directResults', [TSparkDirectResults, None], None, ), # 1281 +) +all_structs.append(TGetOperationStatusReq) +TGetOperationStatusReq.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 1 + (2, TType.BOOL, 'getProgressUpdate', None, None, ), # 2 +) +all_structs.append(TGetOperationStatusResp) +TGetOperationStatusResp.thrift_spec = ( None, # 0 (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 2 - None, # 3 - None, # 4 - None, # 5 - None, # 6 - None, # 7 - None, # 8 - None, # 9 - None, # 10 - None, # 11 + (2, TType.I32, 'operationState', None, None, ), # 2 + (3, TType.STRING, 'sqlState', 'UTF8', None, ), # 3 + (4, TType.I32, 'errorCode', None, None, ), # 4 + (5, TType.STRING, 'errorMessage', 'UTF8', None, ), # 5 + (6, TType.STRING, 'taskStatus', 'UTF8', None, ), # 6 + (7, TType.I64, 'operationStarted', None, None, ), # 7 + (8, TType.I64, 'operationCompleted', None, None, ), # 8 + (9, TType.BOOL, 'hasResultSet', None, None, ), # 9 + (10, TType.STRUCT, 'progressUpdateResponse', [TProgressUpdateResp, None], None, ), # 10 + (11, TType.I64, 'numModifiedRows', None, None, ), # 11 None, # 12 None, # 13 None, # 14 @@ -77706,28 +81713,2073 @@ def __ne__(self, other): None, # 1278 None, # 1279 None, # 1280 - (1281, TType.STRUCT, 'directResults', [TSparkDirectResults, None], None, ), # 1281 + (1281, TType.STRING, 'displayMessage', 'UTF8', None, ), # 1281 + (1282, TType.STRING, 'diagnosticInfo', 'UTF8', None, ), # 1282 + (1283, TType.STRING, 'errorDetailsJson', 'UTF8', None, ), # 1283 + None, # 1284 + None, # 1285 + None, # 1286 + None, # 1287 + None, # 1288 + None, # 1289 + None, # 1290 + None, # 1291 + None, # 1292 + None, # 1293 + None, # 1294 + None, # 1295 + None, # 1296 + None, # 1297 + None, # 1298 + None, # 1299 + None, # 1300 + None, # 1301 + None, # 1302 + None, # 1303 + None, # 1304 + None, # 1305 + None, # 1306 + None, # 1307 + None, # 1308 + None, # 1309 + None, # 1310 + None, # 1311 + None, # 1312 + None, # 1313 + None, # 1314 + None, # 1315 + None, # 1316 + None, # 1317 + None, # 1318 + None, # 1319 + None, # 1320 + None, # 1321 + None, # 1322 + None, # 1323 + None, # 1324 + None, # 1325 + None, # 1326 + None, # 1327 + None, # 1328 + None, # 1329 + None, # 1330 + None, # 1331 + None, # 1332 + None, # 1333 + None, # 1334 + None, # 1335 + None, # 1336 + None, # 1337 + None, # 1338 + None, # 1339 + None, # 1340 + None, # 1341 + None, # 1342 + None, # 1343 + None, # 1344 + None, # 1345 + None, # 1346 + None, # 1347 + None, # 1348 + None, # 1349 + None, # 1350 + None, # 1351 + None, # 1352 + None, # 1353 + None, # 1354 + None, # 1355 + None, # 1356 + None, # 1357 + None, # 1358 + None, # 1359 + None, # 1360 + None, # 1361 + None, # 1362 + None, # 1363 + None, # 1364 + None, # 1365 + None, # 1366 + None, # 1367 + None, # 1368 + None, # 1369 + None, # 1370 + None, # 1371 + None, # 1372 + None, # 1373 + None, # 1374 + None, # 1375 + None, # 1376 + None, # 1377 + None, # 1378 + None, # 1379 + None, # 1380 + None, # 1381 + None, # 1382 + None, # 1383 + None, # 1384 + None, # 1385 + None, # 1386 + None, # 1387 + None, # 1388 + None, # 1389 + None, # 1390 + None, # 1391 + None, # 1392 + None, # 1393 + None, # 1394 + None, # 1395 + None, # 1396 + None, # 1397 + None, # 1398 + None, # 1399 + None, # 1400 + None, # 1401 + None, # 1402 + None, # 1403 + None, # 1404 + None, # 1405 + None, # 1406 + None, # 1407 + None, # 1408 + None, # 1409 + None, # 1410 + None, # 1411 + None, # 1412 + None, # 1413 + None, # 1414 + None, # 1415 + None, # 1416 + None, # 1417 + None, # 1418 + None, # 1419 + None, # 1420 + None, # 1421 + None, # 1422 + None, # 1423 + None, # 1424 + None, # 1425 + None, # 1426 + None, # 1427 + None, # 1428 + None, # 1429 + None, # 1430 + None, # 1431 + None, # 1432 + None, # 1433 + None, # 1434 + None, # 1435 + None, # 1436 + None, # 1437 + None, # 1438 + None, # 1439 + None, # 1440 + None, # 1441 + None, # 1442 + None, # 1443 + None, # 1444 + None, # 1445 + None, # 1446 + None, # 1447 + None, # 1448 + None, # 1449 + None, # 1450 + None, # 1451 + None, # 1452 + None, # 1453 + None, # 1454 + None, # 1455 + None, # 1456 + None, # 1457 + None, # 1458 + None, # 1459 + None, # 1460 + None, # 1461 + None, # 1462 + None, # 1463 + None, # 1464 + None, # 1465 + None, # 1466 + None, # 1467 + None, # 1468 + None, # 1469 + None, # 1470 + None, # 1471 + None, # 1472 + None, # 1473 + None, # 1474 + None, # 1475 + None, # 1476 + None, # 1477 + None, # 1478 + None, # 1479 + None, # 1480 + None, # 1481 + None, # 1482 + None, # 1483 + None, # 1484 + None, # 1485 + None, # 1486 + None, # 1487 + None, # 1488 + None, # 1489 + None, # 1490 + None, # 1491 + None, # 1492 + None, # 1493 + None, # 1494 + None, # 1495 + None, # 1496 + None, # 1497 + None, # 1498 + None, # 1499 + None, # 1500 + None, # 1501 + None, # 1502 + None, # 1503 + None, # 1504 + None, # 1505 + None, # 1506 + None, # 1507 + None, # 1508 + None, # 1509 + None, # 1510 + None, # 1511 + None, # 1512 + None, # 1513 + None, # 1514 + None, # 1515 + None, # 1516 + None, # 1517 + None, # 1518 + None, # 1519 + None, # 1520 + None, # 1521 + None, # 1522 + None, # 1523 + None, # 1524 + None, # 1525 + None, # 1526 + None, # 1527 + None, # 1528 + None, # 1529 + None, # 1530 + None, # 1531 + None, # 1532 + None, # 1533 + None, # 1534 + None, # 1535 + None, # 1536 + None, # 1537 + None, # 1538 + None, # 1539 + None, # 1540 + None, # 1541 + None, # 1542 + None, # 1543 + None, # 1544 + None, # 1545 + None, # 1546 + None, # 1547 + None, # 1548 + None, # 1549 + None, # 1550 + None, # 1551 + None, # 1552 + None, # 1553 + None, # 1554 + None, # 1555 + None, # 1556 + None, # 1557 + None, # 1558 + None, # 1559 + None, # 1560 + None, # 1561 + None, # 1562 + None, # 1563 + None, # 1564 + None, # 1565 + None, # 1566 + None, # 1567 + None, # 1568 + None, # 1569 + None, # 1570 + None, # 1571 + None, # 1572 + None, # 1573 + None, # 1574 + None, # 1575 + None, # 1576 + None, # 1577 + None, # 1578 + None, # 1579 + None, # 1580 + None, # 1581 + None, # 1582 + None, # 1583 + None, # 1584 + None, # 1585 + None, # 1586 + None, # 1587 + None, # 1588 + None, # 1589 + None, # 1590 + None, # 1591 + None, # 1592 + None, # 1593 + None, # 1594 + None, # 1595 + None, # 1596 + None, # 1597 + None, # 1598 + None, # 1599 + None, # 1600 + None, # 1601 + None, # 1602 + None, # 1603 + None, # 1604 + None, # 1605 + None, # 1606 + None, # 1607 + None, # 1608 + None, # 1609 + None, # 1610 + None, # 1611 + None, # 1612 + None, # 1613 + None, # 1614 + None, # 1615 + None, # 1616 + None, # 1617 + None, # 1618 + None, # 1619 + None, # 1620 + None, # 1621 + None, # 1622 + None, # 1623 + None, # 1624 + None, # 1625 + None, # 1626 + None, # 1627 + None, # 1628 + None, # 1629 + None, # 1630 + None, # 1631 + None, # 1632 + None, # 1633 + None, # 1634 + None, # 1635 + None, # 1636 + None, # 1637 + None, # 1638 + None, # 1639 + None, # 1640 + None, # 1641 + None, # 1642 + None, # 1643 + None, # 1644 + None, # 1645 + None, # 1646 + None, # 1647 + None, # 1648 + None, # 1649 + None, # 1650 + None, # 1651 + None, # 1652 + None, # 1653 + None, # 1654 + None, # 1655 + None, # 1656 + None, # 1657 + None, # 1658 + None, # 1659 + None, # 1660 + None, # 1661 + None, # 1662 + None, # 1663 + None, # 1664 + None, # 1665 + None, # 1666 + None, # 1667 + None, # 1668 + None, # 1669 + None, # 1670 + None, # 1671 + None, # 1672 + None, # 1673 + None, # 1674 + None, # 1675 + None, # 1676 + None, # 1677 + None, # 1678 + None, # 1679 + None, # 1680 + None, # 1681 + None, # 1682 + None, # 1683 + None, # 1684 + None, # 1685 + None, # 1686 + None, # 1687 + None, # 1688 + None, # 1689 + None, # 1690 + None, # 1691 + None, # 1692 + None, # 1693 + None, # 1694 + None, # 1695 + None, # 1696 + None, # 1697 + None, # 1698 + None, # 1699 + None, # 1700 + None, # 1701 + None, # 1702 + None, # 1703 + None, # 1704 + None, # 1705 + None, # 1706 + None, # 1707 + None, # 1708 + None, # 1709 + None, # 1710 + None, # 1711 + None, # 1712 + None, # 1713 + None, # 1714 + None, # 1715 + None, # 1716 + None, # 1717 + None, # 1718 + None, # 1719 + None, # 1720 + None, # 1721 + None, # 1722 + None, # 1723 + None, # 1724 + None, # 1725 + None, # 1726 + None, # 1727 + None, # 1728 + None, # 1729 + None, # 1730 + None, # 1731 + None, # 1732 + None, # 1733 + None, # 1734 + None, # 1735 + None, # 1736 + None, # 1737 + None, # 1738 + None, # 1739 + None, # 1740 + None, # 1741 + None, # 1742 + None, # 1743 + None, # 1744 + None, # 1745 + None, # 1746 + None, # 1747 + None, # 1748 + None, # 1749 + None, # 1750 + None, # 1751 + None, # 1752 + None, # 1753 + None, # 1754 + None, # 1755 + None, # 1756 + None, # 1757 + None, # 1758 + None, # 1759 + None, # 1760 + None, # 1761 + None, # 1762 + None, # 1763 + None, # 1764 + None, # 1765 + None, # 1766 + None, # 1767 + None, # 1768 + None, # 1769 + None, # 1770 + None, # 1771 + None, # 1772 + None, # 1773 + None, # 1774 + None, # 1775 + None, # 1776 + None, # 1777 + None, # 1778 + None, # 1779 + None, # 1780 + None, # 1781 + None, # 1782 + None, # 1783 + None, # 1784 + None, # 1785 + None, # 1786 + None, # 1787 + None, # 1788 + None, # 1789 + None, # 1790 + None, # 1791 + None, # 1792 + None, # 1793 + None, # 1794 + None, # 1795 + None, # 1796 + None, # 1797 + None, # 1798 + None, # 1799 + None, # 1800 + None, # 1801 + None, # 1802 + None, # 1803 + None, # 1804 + None, # 1805 + None, # 1806 + None, # 1807 + None, # 1808 + None, # 1809 + None, # 1810 + None, # 1811 + None, # 1812 + None, # 1813 + None, # 1814 + None, # 1815 + None, # 1816 + None, # 1817 + None, # 1818 + None, # 1819 + None, # 1820 + None, # 1821 + None, # 1822 + None, # 1823 + None, # 1824 + None, # 1825 + None, # 1826 + None, # 1827 + None, # 1828 + None, # 1829 + None, # 1830 + None, # 1831 + None, # 1832 + None, # 1833 + None, # 1834 + None, # 1835 + None, # 1836 + None, # 1837 + None, # 1838 + None, # 1839 + None, # 1840 + None, # 1841 + None, # 1842 + None, # 1843 + None, # 1844 + None, # 1845 + None, # 1846 + None, # 1847 + None, # 1848 + None, # 1849 + None, # 1850 + None, # 1851 + None, # 1852 + None, # 1853 + None, # 1854 + None, # 1855 + None, # 1856 + None, # 1857 + None, # 1858 + None, # 1859 + None, # 1860 + None, # 1861 + None, # 1862 + None, # 1863 + None, # 1864 + None, # 1865 + None, # 1866 + None, # 1867 + None, # 1868 + None, # 1869 + None, # 1870 + None, # 1871 + None, # 1872 + None, # 1873 + None, # 1874 + None, # 1875 + None, # 1876 + None, # 1877 + None, # 1878 + None, # 1879 + None, # 1880 + None, # 1881 + None, # 1882 + None, # 1883 + None, # 1884 + None, # 1885 + None, # 1886 + None, # 1887 + None, # 1888 + None, # 1889 + None, # 1890 + None, # 1891 + None, # 1892 + None, # 1893 + None, # 1894 + None, # 1895 + None, # 1896 + None, # 1897 + None, # 1898 + None, # 1899 + None, # 1900 + None, # 1901 + None, # 1902 + None, # 1903 + None, # 1904 + None, # 1905 + None, # 1906 + None, # 1907 + None, # 1908 + None, # 1909 + None, # 1910 + None, # 1911 + None, # 1912 + None, # 1913 + None, # 1914 + None, # 1915 + None, # 1916 + None, # 1917 + None, # 1918 + None, # 1919 + None, # 1920 + None, # 1921 + None, # 1922 + None, # 1923 + None, # 1924 + None, # 1925 + None, # 1926 + None, # 1927 + None, # 1928 + None, # 1929 + None, # 1930 + None, # 1931 + None, # 1932 + None, # 1933 + None, # 1934 + None, # 1935 + None, # 1936 + None, # 1937 + None, # 1938 + None, # 1939 + None, # 1940 + None, # 1941 + None, # 1942 + None, # 1943 + None, # 1944 + None, # 1945 + None, # 1946 + None, # 1947 + None, # 1948 + None, # 1949 + None, # 1950 + None, # 1951 + None, # 1952 + None, # 1953 + None, # 1954 + None, # 1955 + None, # 1956 + None, # 1957 + None, # 1958 + None, # 1959 + None, # 1960 + None, # 1961 + None, # 1962 + None, # 1963 + None, # 1964 + None, # 1965 + None, # 1966 + None, # 1967 + None, # 1968 + None, # 1969 + None, # 1970 + None, # 1971 + None, # 1972 + None, # 1973 + None, # 1974 + None, # 1975 + None, # 1976 + None, # 1977 + None, # 1978 + None, # 1979 + None, # 1980 + None, # 1981 + None, # 1982 + None, # 1983 + None, # 1984 + None, # 1985 + None, # 1986 + None, # 1987 + None, # 1988 + None, # 1989 + None, # 1990 + None, # 1991 + None, # 1992 + None, # 1993 + None, # 1994 + None, # 1995 + None, # 1996 + None, # 1997 + None, # 1998 + None, # 1999 + None, # 2000 + None, # 2001 + None, # 2002 + None, # 2003 + None, # 2004 + None, # 2005 + None, # 2006 + None, # 2007 + None, # 2008 + None, # 2009 + None, # 2010 + None, # 2011 + None, # 2012 + None, # 2013 + None, # 2014 + None, # 2015 + None, # 2016 + None, # 2017 + None, # 2018 + None, # 2019 + None, # 2020 + None, # 2021 + None, # 2022 + None, # 2023 + None, # 2024 + None, # 2025 + None, # 2026 + None, # 2027 + None, # 2028 + None, # 2029 + None, # 2030 + None, # 2031 + None, # 2032 + None, # 2033 + None, # 2034 + None, # 2035 + None, # 2036 + None, # 2037 + None, # 2038 + None, # 2039 + None, # 2040 + None, # 2041 + None, # 2042 + None, # 2043 + None, # 2044 + None, # 2045 + None, # 2046 + None, # 2047 + None, # 2048 + None, # 2049 + None, # 2050 + None, # 2051 + None, # 2052 + None, # 2053 + None, # 2054 + None, # 2055 + None, # 2056 + None, # 2057 + None, # 2058 + None, # 2059 + None, # 2060 + None, # 2061 + None, # 2062 + None, # 2063 + None, # 2064 + None, # 2065 + None, # 2066 + None, # 2067 + None, # 2068 + None, # 2069 + None, # 2070 + None, # 2071 + None, # 2072 + None, # 2073 + None, # 2074 + None, # 2075 + None, # 2076 + None, # 2077 + None, # 2078 + None, # 2079 + None, # 2080 + None, # 2081 + None, # 2082 + None, # 2083 + None, # 2084 + None, # 2085 + None, # 2086 + None, # 2087 + None, # 2088 + None, # 2089 + None, # 2090 + None, # 2091 + None, # 2092 + None, # 2093 + None, # 2094 + None, # 2095 + None, # 2096 + None, # 2097 + None, # 2098 + None, # 2099 + None, # 2100 + None, # 2101 + None, # 2102 + None, # 2103 + None, # 2104 + None, # 2105 + None, # 2106 + None, # 2107 + None, # 2108 + None, # 2109 + None, # 2110 + None, # 2111 + None, # 2112 + None, # 2113 + None, # 2114 + None, # 2115 + None, # 2116 + None, # 2117 + None, # 2118 + None, # 2119 + None, # 2120 + None, # 2121 + None, # 2122 + None, # 2123 + None, # 2124 + None, # 2125 + None, # 2126 + None, # 2127 + None, # 2128 + None, # 2129 + None, # 2130 + None, # 2131 + None, # 2132 + None, # 2133 + None, # 2134 + None, # 2135 + None, # 2136 + None, # 2137 + None, # 2138 + None, # 2139 + None, # 2140 + None, # 2141 + None, # 2142 + None, # 2143 + None, # 2144 + None, # 2145 + None, # 2146 + None, # 2147 + None, # 2148 + None, # 2149 + None, # 2150 + None, # 2151 + None, # 2152 + None, # 2153 + None, # 2154 + None, # 2155 + None, # 2156 + None, # 2157 + None, # 2158 + None, # 2159 + None, # 2160 + None, # 2161 + None, # 2162 + None, # 2163 + None, # 2164 + None, # 2165 + None, # 2166 + None, # 2167 + None, # 2168 + None, # 2169 + None, # 2170 + None, # 2171 + None, # 2172 + None, # 2173 + None, # 2174 + None, # 2175 + None, # 2176 + None, # 2177 + None, # 2178 + None, # 2179 + None, # 2180 + None, # 2181 + None, # 2182 + None, # 2183 + None, # 2184 + None, # 2185 + None, # 2186 + None, # 2187 + None, # 2188 + None, # 2189 + None, # 2190 + None, # 2191 + None, # 2192 + None, # 2193 + None, # 2194 + None, # 2195 + None, # 2196 + None, # 2197 + None, # 2198 + None, # 2199 + None, # 2200 + None, # 2201 + None, # 2202 + None, # 2203 + None, # 2204 + None, # 2205 + None, # 2206 + None, # 2207 + None, # 2208 + None, # 2209 + None, # 2210 + None, # 2211 + None, # 2212 + None, # 2213 + None, # 2214 + None, # 2215 + None, # 2216 + None, # 2217 + None, # 2218 + None, # 2219 + None, # 2220 + None, # 2221 + None, # 2222 + None, # 2223 + None, # 2224 + None, # 2225 + None, # 2226 + None, # 2227 + None, # 2228 + None, # 2229 + None, # 2230 + None, # 2231 + None, # 2232 + None, # 2233 + None, # 2234 + None, # 2235 + None, # 2236 + None, # 2237 + None, # 2238 + None, # 2239 + None, # 2240 + None, # 2241 + None, # 2242 + None, # 2243 + None, # 2244 + None, # 2245 + None, # 2246 + None, # 2247 + None, # 2248 + None, # 2249 + None, # 2250 + None, # 2251 + None, # 2252 + None, # 2253 + None, # 2254 + None, # 2255 + None, # 2256 + None, # 2257 + None, # 2258 + None, # 2259 + None, # 2260 + None, # 2261 + None, # 2262 + None, # 2263 + None, # 2264 + None, # 2265 + None, # 2266 + None, # 2267 + None, # 2268 + None, # 2269 + None, # 2270 + None, # 2271 + None, # 2272 + None, # 2273 + None, # 2274 + None, # 2275 + None, # 2276 + None, # 2277 + None, # 2278 + None, # 2279 + None, # 2280 + None, # 2281 + None, # 2282 + None, # 2283 + None, # 2284 + None, # 2285 + None, # 2286 + None, # 2287 + None, # 2288 + None, # 2289 + None, # 2290 + None, # 2291 + None, # 2292 + None, # 2293 + None, # 2294 + None, # 2295 + None, # 2296 + None, # 2297 + None, # 2298 + None, # 2299 + None, # 2300 + None, # 2301 + None, # 2302 + None, # 2303 + None, # 2304 + None, # 2305 + None, # 2306 + None, # 2307 + None, # 2308 + None, # 2309 + None, # 2310 + None, # 2311 + None, # 2312 + None, # 2313 + None, # 2314 + None, # 2315 + None, # 2316 + None, # 2317 + None, # 2318 + None, # 2319 + None, # 2320 + None, # 2321 + None, # 2322 + None, # 2323 + None, # 2324 + None, # 2325 + None, # 2326 + None, # 2327 + None, # 2328 + None, # 2329 + None, # 2330 + None, # 2331 + None, # 2332 + None, # 2333 + None, # 2334 + None, # 2335 + None, # 2336 + None, # 2337 + None, # 2338 + None, # 2339 + None, # 2340 + None, # 2341 + None, # 2342 + None, # 2343 + None, # 2344 + None, # 2345 + None, # 2346 + None, # 2347 + None, # 2348 + None, # 2349 + None, # 2350 + None, # 2351 + None, # 2352 + None, # 2353 + None, # 2354 + None, # 2355 + None, # 2356 + None, # 2357 + None, # 2358 + None, # 2359 + None, # 2360 + None, # 2361 + None, # 2362 + None, # 2363 + None, # 2364 + None, # 2365 + None, # 2366 + None, # 2367 + None, # 2368 + None, # 2369 + None, # 2370 + None, # 2371 + None, # 2372 + None, # 2373 + None, # 2374 + None, # 2375 + None, # 2376 + None, # 2377 + None, # 2378 + None, # 2379 + None, # 2380 + None, # 2381 + None, # 2382 + None, # 2383 + None, # 2384 + None, # 2385 + None, # 2386 + None, # 2387 + None, # 2388 + None, # 2389 + None, # 2390 + None, # 2391 + None, # 2392 + None, # 2393 + None, # 2394 + None, # 2395 + None, # 2396 + None, # 2397 + None, # 2398 + None, # 2399 + None, # 2400 + None, # 2401 + None, # 2402 + None, # 2403 + None, # 2404 + None, # 2405 + None, # 2406 + None, # 2407 + None, # 2408 + None, # 2409 + None, # 2410 + None, # 2411 + None, # 2412 + None, # 2413 + None, # 2414 + None, # 2415 + None, # 2416 + None, # 2417 + None, # 2418 + None, # 2419 + None, # 2420 + None, # 2421 + None, # 2422 + None, # 2423 + None, # 2424 + None, # 2425 + None, # 2426 + None, # 2427 + None, # 2428 + None, # 2429 + None, # 2430 + None, # 2431 + None, # 2432 + None, # 2433 + None, # 2434 + None, # 2435 + None, # 2436 + None, # 2437 + None, # 2438 + None, # 2439 + None, # 2440 + None, # 2441 + None, # 2442 + None, # 2443 + None, # 2444 + None, # 2445 + None, # 2446 + None, # 2447 + None, # 2448 + None, # 2449 + None, # 2450 + None, # 2451 + None, # 2452 + None, # 2453 + None, # 2454 + None, # 2455 + None, # 2456 + None, # 2457 + None, # 2458 + None, # 2459 + None, # 2460 + None, # 2461 + None, # 2462 + None, # 2463 + None, # 2464 + None, # 2465 + None, # 2466 + None, # 2467 + None, # 2468 + None, # 2469 + None, # 2470 + None, # 2471 + None, # 2472 + None, # 2473 + None, # 2474 + None, # 2475 + None, # 2476 + None, # 2477 + None, # 2478 + None, # 2479 + None, # 2480 + None, # 2481 + None, # 2482 + None, # 2483 + None, # 2484 + None, # 2485 + None, # 2486 + None, # 2487 + None, # 2488 + None, # 2489 + None, # 2490 + None, # 2491 + None, # 2492 + None, # 2493 + None, # 2494 + None, # 2495 + None, # 2496 + None, # 2497 + None, # 2498 + None, # 2499 + None, # 2500 + None, # 2501 + None, # 2502 + None, # 2503 + None, # 2504 + None, # 2505 + None, # 2506 + None, # 2507 + None, # 2508 + None, # 2509 + None, # 2510 + None, # 2511 + None, # 2512 + None, # 2513 + None, # 2514 + None, # 2515 + None, # 2516 + None, # 2517 + None, # 2518 + None, # 2519 + None, # 2520 + None, # 2521 + None, # 2522 + None, # 2523 + None, # 2524 + None, # 2525 + None, # 2526 + None, # 2527 + None, # 2528 + None, # 2529 + None, # 2530 + None, # 2531 + None, # 2532 + None, # 2533 + None, # 2534 + None, # 2535 + None, # 2536 + None, # 2537 + None, # 2538 + None, # 2539 + None, # 2540 + None, # 2541 + None, # 2542 + None, # 2543 + None, # 2544 + None, # 2545 + None, # 2546 + None, # 2547 + None, # 2548 + None, # 2549 + None, # 2550 + None, # 2551 + None, # 2552 + None, # 2553 + None, # 2554 + None, # 2555 + None, # 2556 + None, # 2557 + None, # 2558 + None, # 2559 + None, # 2560 + None, # 2561 + None, # 2562 + None, # 2563 + None, # 2564 + None, # 2565 + None, # 2566 + None, # 2567 + None, # 2568 + None, # 2569 + None, # 2570 + None, # 2571 + None, # 2572 + None, # 2573 + None, # 2574 + None, # 2575 + None, # 2576 + None, # 2577 + None, # 2578 + None, # 2579 + None, # 2580 + None, # 2581 + None, # 2582 + None, # 2583 + None, # 2584 + None, # 2585 + None, # 2586 + None, # 2587 + None, # 2588 + None, # 2589 + None, # 2590 + None, # 2591 + None, # 2592 + None, # 2593 + None, # 2594 + None, # 2595 + None, # 2596 + None, # 2597 + None, # 2598 + None, # 2599 + None, # 2600 + None, # 2601 + None, # 2602 + None, # 2603 + None, # 2604 + None, # 2605 + None, # 2606 + None, # 2607 + None, # 2608 + None, # 2609 + None, # 2610 + None, # 2611 + None, # 2612 + None, # 2613 + None, # 2614 + None, # 2615 + None, # 2616 + None, # 2617 + None, # 2618 + None, # 2619 + None, # 2620 + None, # 2621 + None, # 2622 + None, # 2623 + None, # 2624 + None, # 2625 + None, # 2626 + None, # 2627 + None, # 2628 + None, # 2629 + None, # 2630 + None, # 2631 + None, # 2632 + None, # 2633 + None, # 2634 + None, # 2635 + None, # 2636 + None, # 2637 + None, # 2638 + None, # 2639 + None, # 2640 + None, # 2641 + None, # 2642 + None, # 2643 + None, # 2644 + None, # 2645 + None, # 2646 + None, # 2647 + None, # 2648 + None, # 2649 + None, # 2650 + None, # 2651 + None, # 2652 + None, # 2653 + None, # 2654 + None, # 2655 + None, # 2656 + None, # 2657 + None, # 2658 + None, # 2659 + None, # 2660 + None, # 2661 + None, # 2662 + None, # 2663 + None, # 2664 + None, # 2665 + None, # 2666 + None, # 2667 + None, # 2668 + None, # 2669 + None, # 2670 + None, # 2671 + None, # 2672 + None, # 2673 + None, # 2674 + None, # 2675 + None, # 2676 + None, # 2677 + None, # 2678 + None, # 2679 + None, # 2680 + None, # 2681 + None, # 2682 + None, # 2683 + None, # 2684 + None, # 2685 + None, # 2686 + None, # 2687 + None, # 2688 + None, # 2689 + None, # 2690 + None, # 2691 + None, # 2692 + None, # 2693 + None, # 2694 + None, # 2695 + None, # 2696 + None, # 2697 + None, # 2698 + None, # 2699 + None, # 2700 + None, # 2701 + None, # 2702 + None, # 2703 + None, # 2704 + None, # 2705 + None, # 2706 + None, # 2707 + None, # 2708 + None, # 2709 + None, # 2710 + None, # 2711 + None, # 2712 + None, # 2713 + None, # 2714 + None, # 2715 + None, # 2716 + None, # 2717 + None, # 2718 + None, # 2719 + None, # 2720 + None, # 2721 + None, # 2722 + None, # 2723 + None, # 2724 + None, # 2725 + None, # 2726 + None, # 2727 + None, # 2728 + None, # 2729 + None, # 2730 + None, # 2731 + None, # 2732 + None, # 2733 + None, # 2734 + None, # 2735 + None, # 2736 + None, # 2737 + None, # 2738 + None, # 2739 + None, # 2740 + None, # 2741 + None, # 2742 + None, # 2743 + None, # 2744 + None, # 2745 + None, # 2746 + None, # 2747 + None, # 2748 + None, # 2749 + None, # 2750 + None, # 2751 + None, # 2752 + None, # 2753 + None, # 2754 + None, # 2755 + None, # 2756 + None, # 2757 + None, # 2758 + None, # 2759 + None, # 2760 + None, # 2761 + None, # 2762 + None, # 2763 + None, # 2764 + None, # 2765 + None, # 2766 + None, # 2767 + None, # 2768 + None, # 2769 + None, # 2770 + None, # 2771 + None, # 2772 + None, # 2773 + None, # 2774 + None, # 2775 + None, # 2776 + None, # 2777 + None, # 2778 + None, # 2779 + None, # 2780 + None, # 2781 + None, # 2782 + None, # 2783 + None, # 2784 + None, # 2785 + None, # 2786 + None, # 2787 + None, # 2788 + None, # 2789 + None, # 2790 + None, # 2791 + None, # 2792 + None, # 2793 + None, # 2794 + None, # 2795 + None, # 2796 + None, # 2797 + None, # 2798 + None, # 2799 + None, # 2800 + None, # 2801 + None, # 2802 + None, # 2803 + None, # 2804 + None, # 2805 + None, # 2806 + None, # 2807 + None, # 2808 + None, # 2809 + None, # 2810 + None, # 2811 + None, # 2812 + None, # 2813 + None, # 2814 + None, # 2815 + None, # 2816 + None, # 2817 + None, # 2818 + None, # 2819 + None, # 2820 + None, # 2821 + None, # 2822 + None, # 2823 + None, # 2824 + None, # 2825 + None, # 2826 + None, # 2827 + None, # 2828 + None, # 2829 + None, # 2830 + None, # 2831 + None, # 2832 + None, # 2833 + None, # 2834 + None, # 2835 + None, # 2836 + None, # 2837 + None, # 2838 + None, # 2839 + None, # 2840 + None, # 2841 + None, # 2842 + None, # 2843 + None, # 2844 + None, # 2845 + None, # 2846 + None, # 2847 + None, # 2848 + None, # 2849 + None, # 2850 + None, # 2851 + None, # 2852 + None, # 2853 + None, # 2854 + None, # 2855 + None, # 2856 + None, # 2857 + None, # 2858 + None, # 2859 + None, # 2860 + None, # 2861 + None, # 2862 + None, # 2863 + None, # 2864 + None, # 2865 + None, # 2866 + None, # 2867 + None, # 2868 + None, # 2869 + None, # 2870 + None, # 2871 + None, # 2872 + None, # 2873 + None, # 2874 + None, # 2875 + None, # 2876 + None, # 2877 + None, # 2878 + None, # 2879 + None, # 2880 + None, # 2881 + None, # 2882 + None, # 2883 + None, # 2884 + None, # 2885 + None, # 2886 + None, # 2887 + None, # 2888 + None, # 2889 + None, # 2890 + None, # 2891 + None, # 2892 + None, # 2893 + None, # 2894 + None, # 2895 + None, # 2896 + None, # 2897 + None, # 2898 + None, # 2899 + None, # 2900 + None, # 2901 + None, # 2902 + None, # 2903 + None, # 2904 + None, # 2905 + None, # 2906 + None, # 2907 + None, # 2908 + None, # 2909 + None, # 2910 + None, # 2911 + None, # 2912 + None, # 2913 + None, # 2914 + None, # 2915 + None, # 2916 + None, # 2917 + None, # 2918 + None, # 2919 + None, # 2920 + None, # 2921 + None, # 2922 + None, # 2923 + None, # 2924 + None, # 2925 + None, # 2926 + None, # 2927 + None, # 2928 + None, # 2929 + None, # 2930 + None, # 2931 + None, # 2932 + None, # 2933 + None, # 2934 + None, # 2935 + None, # 2936 + None, # 2937 + None, # 2938 + None, # 2939 + None, # 2940 + None, # 2941 + None, # 2942 + None, # 2943 + None, # 2944 + None, # 2945 + None, # 2946 + None, # 2947 + None, # 2948 + None, # 2949 + None, # 2950 + None, # 2951 + None, # 2952 + None, # 2953 + None, # 2954 + None, # 2955 + None, # 2956 + None, # 2957 + None, # 2958 + None, # 2959 + None, # 2960 + None, # 2961 + None, # 2962 + None, # 2963 + None, # 2964 + None, # 2965 + None, # 2966 + None, # 2967 + None, # 2968 + None, # 2969 + None, # 2970 + None, # 2971 + None, # 2972 + None, # 2973 + None, # 2974 + None, # 2975 + None, # 2976 + None, # 2977 + None, # 2978 + None, # 2979 + None, # 2980 + None, # 2981 + None, # 2982 + None, # 2983 + None, # 2984 + None, # 2985 + None, # 2986 + None, # 2987 + None, # 2988 + None, # 2989 + None, # 2990 + None, # 2991 + None, # 2992 + None, # 2993 + None, # 2994 + None, # 2995 + None, # 2996 + None, # 2997 + None, # 2998 + None, # 2999 + None, # 3000 + None, # 3001 + None, # 3002 + None, # 3003 + None, # 3004 + None, # 3005 + None, # 3006 + None, # 3007 + None, # 3008 + None, # 3009 + None, # 3010 + None, # 3011 + None, # 3012 + None, # 3013 + None, # 3014 + None, # 3015 + None, # 3016 + None, # 3017 + None, # 3018 + None, # 3019 + None, # 3020 + None, # 3021 + None, # 3022 + None, # 3023 + None, # 3024 + None, # 3025 + None, # 3026 + None, # 3027 + None, # 3028 + None, # 3029 + None, # 3030 + None, # 3031 + None, # 3032 + None, # 3033 + None, # 3034 + None, # 3035 + None, # 3036 + None, # 3037 + None, # 3038 + None, # 3039 + None, # 3040 + None, # 3041 + None, # 3042 + None, # 3043 + None, # 3044 + None, # 3045 + None, # 3046 + None, # 3047 + None, # 3048 + None, # 3049 + None, # 3050 + None, # 3051 + None, # 3052 + None, # 3053 + None, # 3054 + None, # 3055 + None, # 3056 + None, # 3057 + None, # 3058 + None, # 3059 + None, # 3060 + None, # 3061 + None, # 3062 + None, # 3063 + None, # 3064 + None, # 3065 + None, # 3066 + None, # 3067 + None, # 3068 + None, # 3069 + None, # 3070 + None, # 3071 + None, # 3072 + None, # 3073 + None, # 3074 + None, # 3075 + None, # 3076 + None, # 3077 + None, # 3078 + None, # 3079 + None, # 3080 + None, # 3081 + None, # 3082 + None, # 3083 + None, # 3084 + None, # 3085 + None, # 3086 + None, # 3087 + None, # 3088 + None, # 3089 + None, # 3090 + None, # 3091 + None, # 3092 + None, # 3093 + None, # 3094 + None, # 3095 + None, # 3096 + None, # 3097 + None, # 3098 + None, # 3099 + None, # 3100 + None, # 3101 + None, # 3102 + None, # 3103 + None, # 3104 + None, # 3105 + None, # 3106 + None, # 3107 + None, # 3108 + None, # 3109 + None, # 3110 + None, # 3111 + None, # 3112 + None, # 3113 + None, # 3114 + None, # 3115 + None, # 3116 + None, # 3117 + None, # 3118 + None, # 3119 + None, # 3120 + None, # 3121 + None, # 3122 + None, # 3123 + None, # 3124 + None, # 3125 + None, # 3126 + None, # 3127 + None, # 3128 + None, # 3129 + None, # 3130 + None, # 3131 + None, # 3132 + None, # 3133 + None, # 3134 + None, # 3135 + None, # 3136 + None, # 3137 + None, # 3138 + None, # 3139 + None, # 3140 + None, # 3141 + None, # 3142 + None, # 3143 + None, # 3144 + None, # 3145 + None, # 3146 + None, # 3147 + None, # 3148 + None, # 3149 + None, # 3150 + None, # 3151 + None, # 3152 + None, # 3153 + None, # 3154 + None, # 3155 + None, # 3156 + None, # 3157 + None, # 3158 + None, # 3159 + None, # 3160 + None, # 3161 + None, # 3162 + None, # 3163 + None, # 3164 + None, # 3165 + None, # 3166 + None, # 3167 + None, # 3168 + None, # 3169 + None, # 3170 + None, # 3171 + None, # 3172 + None, # 3173 + None, # 3174 + None, # 3175 + None, # 3176 + None, # 3177 + None, # 3178 + None, # 3179 + None, # 3180 + None, # 3181 + None, # 3182 + None, # 3183 + None, # 3184 + None, # 3185 + None, # 3186 + None, # 3187 + None, # 3188 + None, # 3189 + None, # 3190 + None, # 3191 + None, # 3192 + None, # 3193 + None, # 3194 + None, # 3195 + None, # 3196 + None, # 3197 + None, # 3198 + None, # 3199 + None, # 3200 + None, # 3201 + None, # 3202 + None, # 3203 + None, # 3204 + None, # 3205 + None, # 3206 + None, # 3207 + None, # 3208 + None, # 3209 + None, # 3210 + None, # 3211 + None, # 3212 + None, # 3213 + None, # 3214 + None, # 3215 + None, # 3216 + None, # 3217 + None, # 3218 + None, # 3219 + None, # 3220 + None, # 3221 + None, # 3222 + None, # 3223 + None, # 3224 + None, # 3225 + None, # 3226 + None, # 3227 + None, # 3228 + None, # 3229 + None, # 3230 + None, # 3231 + None, # 3232 + None, # 3233 + None, # 3234 + None, # 3235 + None, # 3236 + None, # 3237 + None, # 3238 + None, # 3239 + None, # 3240 + None, # 3241 + None, # 3242 + None, # 3243 + None, # 3244 + None, # 3245 + None, # 3246 + None, # 3247 + None, # 3248 + None, # 3249 + None, # 3250 + None, # 3251 + None, # 3252 + None, # 3253 + None, # 3254 + None, # 3255 + None, # 3256 + None, # 3257 + None, # 3258 + None, # 3259 + None, # 3260 + None, # 3261 + None, # 3262 + None, # 3263 + None, # 3264 + None, # 3265 + None, # 3266 + None, # 3267 + None, # 3268 + None, # 3269 + None, # 3270 + None, # 3271 + None, # 3272 + None, # 3273 + None, # 3274 + None, # 3275 + None, # 3276 + None, # 3277 + None, # 3278 + None, # 3279 + None, # 3280 + None, # 3281 + None, # 3282 + None, # 3283 + None, # 3284 + None, # 3285 + None, # 3286 + None, # 3287 + None, # 3288 + None, # 3289 + None, # 3290 + None, # 3291 + None, # 3292 + None, # 3293 + None, # 3294 + None, # 3295 + None, # 3296 + None, # 3297 + None, # 3298 + None, # 3299 + None, # 3300 + None, # 3301 + None, # 3302 + None, # 3303 + None, # 3304 + None, # 3305 + None, # 3306 + None, # 3307 + None, # 3308 + None, # 3309 + None, # 3310 + None, # 3311 + None, # 3312 + None, # 3313 + None, # 3314 + None, # 3315 + None, # 3316 + None, # 3317 + None, # 3318 + None, # 3319 + None, # 3320 + None, # 3321 + None, # 3322 + None, # 3323 + None, # 3324 + None, # 3325 + None, # 3326 + None, # 3327 + None, # 3328 + (3329, TType.STRING, 'responseValidation', 'BINARY', None, ), # 3329 + (3330, TType.I32, 'idempotencyType', None, None, ), # 3330 + (3331, TType.I64, 'statementTimeout', None, None, ), # 3331 + (3332, TType.I32, 'statementTimeoutLevel', None, None, ), # 3332 ) -all_structs.append(TGetOperationStatusReq) -TGetOperationStatusReq.thrift_spec = ( +all_structs.append(TCancelOperationReq) +TCancelOperationReq.thrift_spec = ( None, # 0 (1, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 1 - (2, TType.BOOL, 'getProgressUpdate', None, None, ), # 2 -) -all_structs.append(TGetOperationStatusResp) -TGetOperationStatusResp.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.I32, 'operationState', None, None, ), # 2 - (3, TType.STRING, 'sqlState', 'UTF8', None, ), # 3 - (4, TType.I32, 'errorCode', None, None, ), # 4 - (5, TType.STRING, 'errorMessage', 'UTF8', None, ), # 5 - (6, TType.STRING, 'taskStatus', 'UTF8', None, ), # 6 - (7, TType.I64, 'operationStarted', None, None, ), # 7 - (8, TType.I64, 'operationCompleted', None, None, ), # 8 - (9, TType.BOOL, 'hasResultSet', None, None, ), # 9 - (10, TType.STRUCT, 'progressUpdateResponse', [TProgressUpdateResp, None], None, ), # 10 - (11, TType.I64, 'numModifiedRows', None, None, ), # 11 + None, # 2 + None, # 3 + None, # 4 + None, # 5 + None, # 6 + None, # 7 + None, # 8 + None, # 9 + None, # 10 + None, # 11 None, # 12 None, # 13 None, # 14 @@ -78997,8 +85049,8 @@ def __ne__(self, other): None, # 1278 None, # 1279 None, # 1280 - (1281, TType.STRING, 'displayMessage', 'UTF8', None, ), # 1281 - (1282, TType.STRING, 'diagnosticInfo', 'UTF8', None, ), # 1282 + None, # 1281 + None, # 1282 None, # 1283 None, # 1284 None, # 1285 @@ -81045,13 +87097,16 @@ def __ne__(self, other): None, # 3326 None, # 3327 None, # 3328 - (3329, TType.STRING, 'responseValidation', 'BINARY', None, ), # 3329 - (3330, TType.I32, 'idempotencyType', None, None, ), # 3330 - (3331, TType.I64, 'statementTimeout', None, None, ), # 3331 - (3332, TType.I32, 'statementTimeoutLevel', None, None, ), # 3332 + (3329, TType.I16, 'executionVersion', None, None, ), # 3329 + (3330, TType.BOOL, 'replacedByNextAttempt', None, None, ), # 3330 ) -all_structs.append(TCancelOperationReq) -TCancelOperationReq.thrift_spec = ( +all_structs.append(TCancelOperationResp) +TCancelOperationResp.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 +) +all_structs.append(TCloseOperationReq) +TCloseOperationReq.thrift_spec = ( None, # 0 (1, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 1 None, # 2 @@ -84381,18 +90436,7 @@ def __ne__(self, other): None, # 3326 None, # 3327 None, # 3328 - (3329, TType.I16, 'executionVersion', None, None, ), # 3329 - (3330, TType.BOOL, 'replacedByNextAttempt', None, None, ), # 3330 -) -all_structs.append(TCancelOperationResp) -TCancelOperationResp.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 -) -all_structs.append(TCloseOperationReq) -TCloseOperationReq.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 1 + (3329, TType.I32, 'closeReason', None, 0, ), # 3329 ) all_structs.append(TCloseOperationResp) TCloseOperationResp.thrift_spec = ( @@ -91066,7 +97110,21 @@ def __ne__(self, other): (3329, TType.I32, 'reasonForNoCloudFetch', None, None, ), # 3329 (3330, TType.LIST, 'resultFiles', (TType.STRUCT, [TDBSqlCloudResultFile, None], False), None, ), # 3330 (3331, TType.STRING, 'manifestFile', 'UTF8', None, ), # 3331 - (3332, TType.STRING, 'manifestFileFormat', 'UTF8', None, ), # 3332 + (3332, TType.I32, 'manifestFileFormat', None, None, ), # 3332 + (3333, TType.I64, 'cacheLookupLatency', None, None, ), # 3333 + (3334, TType.STRING, 'remoteCacheMissReason', 'UTF8', None, ), # 3334 + (3335, TType.I32, 'fetchDisposition', None, None, ), # 3335 + (3336, TType.BOOL, 'remoteResultCacheEnabled', None, None, ), # 3336 + (3337, TType.BOOL, 'isServerless', None, None, ), # 3337 + None, # 3338 + None, # 3339 + None, # 3340 + None, # 3341 + None, # 3342 + None, # 3343 + (3344, TType.STRUCT, 'resultDataFormat', [TDBSqlResultFormat, None], None, ), # 3344 + (3345, TType.BOOL, 'truncatedByThriftLimit', None, None, ), # 3345 + (3346, TType.I64, 'resultByteLimit', None, None, ), # 3346 ) all_structs.append(TFetchResultsReq) TFetchResultsReq.thrift_spec = ( @@ -105713,50 +111771,5 @@ def __ne__(self, other): (5, TType.STRING, 'footerSummary', 'UTF8', None, ), # 5 (6, TType.I64, 'startTime', None, None, ), # 6 ) -all_structs.append(TDBSqlClusterMetrics) -TDBSqlClusterMetrics.thrift_spec = ( - None, # 0 - (1, TType.I32, 'clusterCapacity', None, None, ), # 1 - (2, TType.I32, 'numRunningTasks', None, None, ), # 2 - (3, TType.I32, 'numPendingTasks', None, None, ), # 3 - (4, TType.DOUBLE, 'rejectionThreshold', None, None, ), # 4 - (5, TType.DOUBLE, 'tasksCompletedPerMinute', None, None, ), # 5 -) -all_structs.append(TDBSqlQueryLaneMetrics) -TDBSqlQueryLaneMetrics.thrift_spec = ( - None, # 0 - (1, TType.I32, 'fastLaneReservation', None, None, ), # 1 - (2, TType.I32, 'numFastLaneRunningTasks', None, None, ), # 2 - (3, TType.I32, 'numFastLanePendingTasks', None, None, ), # 3 - (4, TType.I32, 'slowLaneReservation', None, None, ), # 4 - (5, TType.I32, 'numSlowLaneRunningTasks', None, None, ), # 5 - (6, TType.I32, 'numSlowLanePendingTasks', None, None, ), # 6 -) -all_structs.append(TDBSqlQueryMetrics) -TDBSqlQueryMetrics.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 2 - (3, TType.I32, 'idempotencyType', None, None, ), # 3 - (4, TType.STRUCT, 'sessionHandle', [TSessionHandle, None], None, ), # 4 - (5, TType.I64, 'operationStarted', None, None, ), # 5 - (6, TType.DOUBLE, 'queryCost', None, None, ), # 6 - (7, TType.I32, 'numRunningTasks', None, None, ), # 7 - (8, TType.I32, 'numPendingTasks', None, None, ), # 8 - (9, TType.I32, 'numCompletedTasks', None, None, ), # 9 -) -all_structs.append(TDBSqlGetLoadInformationReq) -TDBSqlGetLoadInformationReq.thrift_spec = ( - None, # 0 - (1, TType.BOOL, 'includeQueryMetrics', None, False, ), # 1 -) -all_structs.append(TDBSqlGetLoadInformationResp) -TDBSqlGetLoadInformationResp.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.STRUCT, 'clusterMetrics', [TDBSqlClusterMetrics, None], None, ), # 2 - (3, TType.STRUCT, 'queryLaneMetrics', [TDBSqlQueryLaneMetrics, None], None, ), # 3 - (4, TType.LIST, 'queryMetrics', (TType.STRUCT, [TDBSqlQueryMetrics, None], False), None, ), # 4 -) fix_spec(all_structs) del all_structs diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index 935c7711..f76350a2 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -3,47 +3,70 @@ import logging import math import time +import uuid import threading -import lz4.frame -from ssl import CERT_NONE, CERT_REQUIRED, create_default_context from typing import List, Union -import pyarrow +from databricks.sql.thrift_api.TCLIService.ttypes import TOperationState + +try: + import pyarrow +except ImportError: + pyarrow = None import thrift.transport.THttpClient import thrift.protocol.TBinaryProtocol import thrift.transport.TSocket import thrift.transport.TTransport +import urllib3.exceptions + import databricks.sql.auth.thrift_http_client +from databricks.sql.auth.thrift_http_client import CommandType from databricks.sql.auth.authenticators import AuthProvider from databricks.sql.thrift_api.TCLIService import TCLIService, ttypes from databricks.sql import * +from databricks.sql.exc import MaxRetryDurationError from databricks.sql.thrift_api.TCLIService.TCLIService import ( Client as TCLIServiceClient, ) from databricks.sql.utils import ( - ArrowQueue, ExecuteResponse, _bound, RequestErrorInfo, NoRetryReason, + ResultSetQueueFactory, + convert_arrow_based_set_to_arrow_table, + convert_decimals_in_arrow_table, + convert_column_based_set_to_arrow_table, ) +from databricks.sql.types import SSLOptions logger = logging.getLogger(__name__) +unsafe_logger = logging.getLogger("databricks.sql.unsafe") +unsafe_logger.setLevel(logging.DEBUG) + +# To capture these logs in client code, add a non-NullHandler. +# See our e2e test suite for an example with logging.FileHandler +unsafe_logger.addHandler(logging.NullHandler()) + +# Disable propagation so that handlers for `databricks.sql` don't pick up these messages +unsafe_logger.propagate = False + THRIFT_ERROR_MESSAGE_HEADER = "x-thriftserver-error-message" DATABRICKS_ERROR_OR_REDIRECT_HEADER = "x-databricks-error-or-redirect-message" DATABRICKS_REASON_HEADER = "x-databricks-reason-phrase" TIMESTAMP_AS_STRING_CONFIG = "spark.thriftserver.arrowBasedRowSet.timestampAsString" +DEFAULT_SOCKET_TIMEOUT = float(900) # see Connection.__init__ for parameter descriptions. # - Min/Max avoids unsustainable configs (sane values are far more constrained) # - 900s attempts-duration lines up w ODBC/JDBC drivers (for cluster startup > 10 mins) _retry_policy = { # (type, default, min, max) "_retry_delay_min": (float, 1, 0.1, 60), - "_retry_delay_max": (float, 60, 5, 3600), + "_retry_delay_max": (float, 30, 5, 3600), "_retry_stop_after_attempts_count": (int, 30, 1, 60), "_retry_stop_after_attempts_duration": (float, 900, 1, 86400), "_retry_delay_default": (float, 5, 1, 60), @@ -53,7 +76,12 @@ class ThriftBackend: CLOSED_OP_STATE = ttypes.TOperationState.CLOSED_STATE ERROR_OP_STATE = ttypes.TOperationState.ERROR_STATE - BIT_MASKS = [1, 2, 4, 8, 16, 32, 64, 128] + + _retry_delay_min: float + _retry_delay_max: float + _retry_stop_after_attempts_count: int + _retry_stop_after_attempts_duration: float + _retry_delay_default: float def __init__( self, @@ -62,6 +90,7 @@ def __init__( http_path: str, http_headers, auth_provider: AuthProvider, + ssl_options: SSLOptions, staging_allowed_local_path: Union[None, str, List[str]] = None, **kwargs, ): @@ -70,16 +99,6 @@ def __init__( # Tag to add to User-Agent header. For use by partners. # _username, _password # Username and password Basic authentication (no official support) - # _tls_no_verify - # Set to True (Boolean) to completely disable SSL verification. - # _tls_verify_hostname - # Set to False (Boolean) to disable SSL hostname verification, but check certificate. - # _tls_trusted_ca_file - # Set to the path of the file containing trusted CA certificates for server certificate - # verification. If not provide, uses system truststore. - # _tls_client_cert_file, _tls_client_cert_key_file, _tls_client_cert_key_password - # Set client SSL certificate. - # See https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_cert_chain # _connection_uri # Overrides server_hostname and http_path. # RETRY/ATTEMPT POLICY @@ -98,17 +117,31 @@ def __init__( # # _retry_stop_after_attempts_count # The maximum number of times we should retry retryable requests (defaults to 24) + # _retry_dangerous_codes + # An iterable of integer HTTP status codes. ExecuteStatement commands will be retried if these codes are received. + # (defaults to []) # _socket_timeout - # The timeout in seconds for socket send, recv and connect operations. Defaults to None for - # no timeout. Should be a positive float or integer. + # The timeout in seconds for socket send, recv and connect operations. Should be a positive float or integer. + # (defaults to 900) + # _enable_v3_retries + # Whether to use the DatabricksRetryPolicy implemented in urllib3 + # (defaults to True) + # _retry_max_redirects + # An integer representing the maximum number of redirects to follow for a request. + # This number must be <= _retry_stop_after_attempts_count. + # (defaults to None) + # max_download_threads + # Number of threads for handling cloud fetch downloads. Defaults to 10 port = port or 443 if kwargs.get("_connection_uri"): uri = kwargs.get("_connection_uri") elif server_hostname and http_path: - uri = "https://{host}:{port}/{path}".format( - host=server_hostname, port=port, path=http_path.lstrip("/") + uri = "{host}:{port}/{path}".format( + host=server_hostname.rstrip("/"), port=port, path=http_path.lstrip("/") ) + if not uri.startswith("https://"): + uri = "https://" + uri else: raise ValueError("No valid connection settings.") @@ -122,38 +155,56 @@ def __init__( "_use_arrow_native_timestamps", True ) - # Configure tls context - ssl_context = create_default_context(cafile=kwargs.get("_tls_trusted_ca_file")) - if kwargs.get("_tls_no_verify") is True: - ssl_context.check_hostname = False - ssl_context.verify_mode = CERT_NONE - elif kwargs.get("_tls_verify_hostname") is False: - ssl_context.check_hostname = False - ssl_context.verify_mode = CERT_REQUIRED + # Cloud fetch + self.max_download_threads = kwargs.get("max_download_threads", 10) + + self._ssl_options = ssl_options + + self._auth_provider = auth_provider + + # Connector version 3 retry approach + self.enable_v3_retries = kwargs.get("_enable_v3_retries", True) + + if not self.enable_v3_retries: + logger.warning( + "Legacy retry behavior is enabled for this connection." + " This behaviour is deprecated and will be removed in a future release." + ) + self.force_dangerous_codes = kwargs.get("_retry_dangerous_codes", []) + + additional_transport_args = {} + _max_redirects: Union[None, int] = kwargs.get("_retry_max_redirects") + + if _max_redirects: + if _max_redirects > self._retry_stop_after_attempts_count: + logger.warn( + "_retry_max_redirects > _retry_stop_after_attempts_count so it will have no affect!" + ) + urllib3_kwargs = {"redirect": _max_redirects} else: - ssl_context.check_hostname = True - ssl_context.verify_mode = CERT_REQUIRED - - tls_client_cert_file = kwargs.get("_tls_client_cert_file") - tls_client_cert_key_file = kwargs.get("_tls_client_cert_key_file") - tls_client_cert_key_password = kwargs.get("_tls_client_cert_key_password") - if tls_client_cert_file: - ssl_context.load_cert_chain( - certfile=tls_client_cert_file, - keyfile=tls_client_cert_key_file, - password=tls_client_cert_key_password, + urllib3_kwargs = {} + if self.enable_v3_retries: + self.retry_policy = databricks.sql.auth.thrift_http_client.DatabricksRetryPolicy( + delay_min=self._retry_delay_min, + delay_max=self._retry_delay_max, + stop_after_attempts_count=self._retry_stop_after_attempts_count, + stop_after_attempts_duration=self._retry_stop_after_attempts_duration, + delay_default=self._retry_delay_default, + force_dangerous_codes=self.force_dangerous_codes, + urllib3_kwargs=urllib3_kwargs, ) - self._auth_provider = auth_provider + additional_transport_args["retry_policy"] = self.retry_policy self._transport = databricks.sql.auth.thrift_http_client.THttpClient( auth_provider=self._auth_provider, uri_or_host=uri, - ssl_context=ssl_context, + ssl_options=self._ssl_options, + **additional_transport_args, # type: ignore ) - timeout = kwargs.get("_socket_timeout") - # setTimeout defaults to None (i.e. no timeout), and is expected in ms + timeout = kwargs.get("_socket_timeout", DEFAULT_SOCKET_TIMEOUT) + # setTimeout defaults to 15 minutes and is expected in ms self._transport.setTimeout(timeout and (float(timeout) * 1000.0)) self._transport.setCustomHeaders(dict(http_headers)) @@ -168,10 +219,11 @@ def __init__( self._request_lock = threading.RLock() + # TODO: Move this bounding logic into DatabricksRetryPolicy for v3 (PECO-918) def _initialize_retry_args(self, kwargs): # Configure retries & timing: use user-settings or defaults, and bound # by policy. Log.warn when given param gets restricted. - for (key, (type_, default, min, max)) in _retry_policy.items(): + for key, (type_, default, min, max) in _retry_policy.items(): given_or_default = type_(kwargs.get(key, default)) bound = _bound(min, max, given_or_default) setattr(self, key, bound) @@ -269,7 +321,7 @@ def _handle_request_error(self, error_info, attempt, elapsed): # FUTURE: Consider moving to https://github.com/litl/backoff or # https://github.com/jd/tenacity for retry logic. - def make_request(self, method, request): + def make_request(self, method, request, retryable=True): """Execute given request, attempting retries when 1. Receiving HTTP 429/503 from server 2. OSError is raised during a GetOperationStatus @@ -300,8 +352,8 @@ def extract_retry_delay(attempt): # encapsulate retry checks, returns None || delay-in-secs # Retry IFF 429/503 code + Retry-After header set http_code = getattr(self._transport, "code", None) - retry_after = getattr(self._transport, "headers", {}).get("Retry-After") - if http_code in [429, 503] and retry_after: + retry_after = getattr(self._transport, "headers", {}).get("Retry-After", 1) + if http_code in [429, 503]: # bound delay (seconds) by [min_delay*1.5^(attempt-1), max_delay] return bound_retry_delay(attempt, int(retry_after)) return None @@ -315,10 +367,44 @@ def attempt_request(attempt): error, error_message, retry_delay = None, None, None try: - logger.debug("Sending request: {}".format(request)) + this_method_name = getattr(method, "__name__") + + logger.debug("Sending request: {}()".format(this_method_name)) + unsafe_logger.debug("Sending request: {}".format(request)) + + # These three lines are no-ops if the v3 retry policy is not in use + if self.enable_v3_retries: + this_command_type = CommandType.get(this_method_name) + self._transport.set_retry_command_type(this_command_type) + self._transport.startRetryTimer() + response = method(request) - logger.debug("Received response: {}".format(response)) + + # We need to call type(response) here because thrift doesn't implement __name__ attributes for thrift responses + logger.debug( + "Received response: {}()".format(type(response).__name__) + ) + unsafe_logger.debug("Received response: {}".format(response)) return response + + except urllib3.exceptions.HTTPError as err: + # retry on timeout. Happens a lot in Azure and it is safe as data has not been sent to server yet + + # TODO: don't use exception handling for GOS polling... + + gos_name = TCLIServiceClient.GetOperationStatus.__name__ + if method.__name__ == gos_name: + delay_default = ( + self.enable_v3_retries + and self.retry_policy.delay_default + or self._retry_delay_default + ) + retry_delay = bound_retry_delay(attempt, delay_default) + logger.info( + f"GetOperationStatus failed with HTTP error and will be retried: {str(err)}" + ) + else: + raise err except OSError as err: error = err error_message = str(err) @@ -327,11 +413,11 @@ def attempt_request(attempt): # log.info for errors we believe are not unusual or unexpected. log.warn for # for others like EEXIST, EBADF, ERANGE which are not expected in this context. # - # I manually tested this retry behaviour using mitmweb and confirmed that + # I manually tested this retry behaviour using mitmweb and confirmed that # GetOperationStatus requests are retried when I forced network connection # interruptions / timeouts / reconnects. See #24 for more info. # | Debian | Darwin | - info_errs = [ # |--------|--------| + info_errs = [ # |--------|--------| errno.ESHUTDOWN, # | 32 | 32 | errno.EAFNOSUPPORT, # | 97 | 47 | errno.ECONNRESET, # | 104 | 54 | @@ -355,6 +441,10 @@ def attempt_request(attempt): error_message = ThriftBackend._extract_error_message_from_headers( getattr(self._transport, "headers", {}) ) + finally: + # Calling `close()` here releases the active HTTP connection back to the pool + self._transport.close() + return RequestErrorInfo( error=error, error_message=error_message, @@ -370,7 +460,7 @@ def attempt_request(attempt): # return on success # if available: bounded delay and retry # if not: raise error - max_attempts = self._retry_stop_after_attempts_count + max_attempts = self._retry_stop_after_attempts_count if retryable else 1 # use index-1 counting for logging/human consistency for attempt in range(1, max_attempts + 1): @@ -464,7 +554,7 @@ def open_session(self, session_configuration, catalog, schema): response = self.make_request(self._client.OpenSession, open_session_req) self._check_initial_namespace(catalog, schema, response) self._check_protocol_version(response) - return response.sessionHandle + return response except: self._transport.close() raise @@ -484,7 +574,8 @@ def _check_command_not_in_error_or_closed_state( raise ServerOperationError( get_operations_resp.displayMessage, { - "operation-id": op_handle and op_handle.operationId.guid, + "operation-id": op_handle + and self.guid_to_hex_id(op_handle.operationId.guid), "diagnostic-info": get_operations_resp.diagnosticInfo, }, ) @@ -492,16 +583,20 @@ def _check_command_not_in_error_or_closed_state( raise ServerOperationError( get_operations_resp.errorMessage, { - "operation-id": op_handle and op_handle.operationId.guid, + "operation-id": op_handle + and self.guid_to_hex_id(op_handle.operationId.guid), "diagnostic-info": None, }, ) elif get_operations_resp.operationState == ttypes.TOperationState.CLOSED_STATE: raise DatabaseError( "Command {} unexpectedly closed server side".format( - op_handle and op_handle.operationId.guid + op_handle and self.guid_to_hex_id(op_handle.operationId.guid) ), - {"operation-id": op_handle and op_handle.operationId.guid}, + { + "operation-id": op_handle + and self.guid_to_hex_id(op_handle.operationId.guid) + }, ) def _poll_for_status(self, op_handle): @@ -516,108 +611,14 @@ def _create_arrow_table(self, t_row_set, lz4_compressed, schema_bytes, descripti ( arrow_table, num_rows, - ) = ThriftBackend._convert_column_based_set_to_arrow_table( - t_row_set.columns, description - ) + ) = convert_column_based_set_to_arrow_table(t_row_set.columns, description) elif t_row_set.arrowBatches is not None: - ( - arrow_table, - num_rows, - ) = ThriftBackend._convert_arrow_based_set_to_arrow_table( + (arrow_table, num_rows,) = convert_arrow_based_set_to_arrow_table( t_row_set.arrowBatches, lz4_compressed, schema_bytes ) else: raise OperationalError("Unsupported TRowSet instance {}".format(t_row_set)) - return self._convert_decimals_in_arrow_table(arrow_table, description), num_rows - - @staticmethod - def _convert_decimals_in_arrow_table(table, description): - for (i, col) in enumerate(table.itercolumns()): - if description[i][1] == "decimal": - decimal_col = col.to_pandas().apply( - lambda v: v if v is None else Decimal(v) - ) - precision, scale = description[i][4], description[i][5] - assert scale is not None - assert precision is not None - # Spark limits decimal to a maximum scale of 38, - # so 128 is guaranteed to be big enough - dtype = pyarrow.decimal128(precision, scale) - col_data = pyarrow.array(decimal_col, type=dtype) - field = table.field(i).with_type(dtype) - table = table.set_column(i, field, col_data) - return table - - @staticmethod - def _convert_arrow_based_set_to_arrow_table( - arrow_batches, lz4_compressed, schema_bytes - ): - ba = bytearray() - ba += schema_bytes - n_rows = 0 - if lz4_compressed: - for arrow_batch in arrow_batches: - n_rows += arrow_batch.rowCount - ba += lz4.frame.decompress(arrow_batch.batch) - else: - for arrow_batch in arrow_batches: - n_rows += arrow_batch.rowCount - ba += arrow_batch.batch - arrow_table = pyarrow.ipc.open_stream(ba).read_all() - return arrow_table, n_rows - - @staticmethod - def _convert_column_based_set_to_arrow_table(columns, description): - arrow_table = pyarrow.Table.from_arrays( - [ThriftBackend._convert_column_to_arrow_array(c) for c in columns], - # Only use the column names from the schema, the types are determined by the - # physical types used in column based set, as they can differ from the - # mapping used in _hive_schema_to_arrow_schema. - names=[c[0] for c in description], - ) - return arrow_table, arrow_table.num_rows - - @staticmethod - def _convert_column_to_arrow_array(t_col): - """ - Return a pyarrow array from the values in a TColumn instance. - Note that ColumnBasedSet has no native support for complex types, so they will be converted - to strings server-side. - """ - field_name_to_arrow_type = { - "boolVal": pyarrow.bool_(), - "byteVal": pyarrow.int8(), - "i16Val": pyarrow.int16(), - "i32Val": pyarrow.int32(), - "i64Val": pyarrow.int64(), - "doubleVal": pyarrow.float64(), - "stringVal": pyarrow.string(), - "binaryVal": pyarrow.binary(), - } - for field in field_name_to_arrow_type.keys(): - wrapper = getattr(t_col, field) - if wrapper: - return ThriftBackend._create_arrow_array( - wrapper, field_name_to_arrow_type[field] - ) - - raise OperationalError("Empty TColumn instance {}".format(t_col)) - - @staticmethod - def _create_arrow_array(t_col_value_wrapper, arrow_type): - result = t_col_value_wrapper.values - nulls = t_col_value_wrapper.nulls # bitfield describing which values are null - assert isinstance(nulls, bytes) - - # The number of bits in nulls can be both larger or smaller than the number of - # elements in result, so take the minimum of both to iterate over. - length = min(len(result), len(nulls) * 8) - - for i in range(length): - if nulls[i >> 3] & ThriftBackend.BIT_MASKS[i & 0x7]: - result[i] = None - - return pyarrow.array(result, type=arrow_type) + return convert_decimals_in_arrow_table(arrow_table, description), num_rows def _get_metadata_resp(self, op_handle): req = ttypes.TGetResultSetMetadataReq(operationHandle=op_handle) @@ -710,6 +711,7 @@ def _results_message_to_execute_response(self, resp, operation_state): if t_result_set_metadata_resp.resultFormat not in [ ttypes.TSparkRowSetType.ARROW_BASED_SET, ttypes.TSparkRowSetType.COLUMN_BASED_SET, + ttypes.TSparkRowSetType.URL_BASED_SET, ]: raise OperationalError( "Expected results to be in Arrow or column based format, " @@ -729,25 +731,32 @@ def _results_message_to_execute_response(self, resp, operation_state): description = self._hive_schema_to_description( t_result_set_metadata_resp.schema ) - schema_bytes = ( - t_result_set_metadata_resp.arrowSchema - or self._hive_schema_to_arrow_schema(t_result_set_metadata_resp.schema) - .serialize() - .to_pybytes() - ) + + if pyarrow: + schema_bytes = ( + t_result_set_metadata_resp.arrowSchema + or self._hive_schema_to_arrow_schema(t_result_set_metadata_resp.schema) + .serialize() + .to_pybytes() + ) + else: + schema_bytes = None + lz4_compressed = t_result_set_metadata_resp.lz4Compressed is_staging_operation = t_result_set_metadata_resp.isStagingOperation if direct_results and direct_results.resultSet: assert direct_results.resultSet.results.startRowOffset == 0 assert direct_results.resultSetMetadata - arrow_results, n_rows = self._create_arrow_table( - direct_results.resultSet.results, - lz4_compressed, - schema_bytes, - description, + arrow_queue_opt = ResultSetQueueFactory.build_queue( + row_set_type=t_result_set_metadata_resp.resultFormat, + t_row_set=direct_results.resultSet.results, + arrow_schema_bytes=schema_bytes, + max_download_threads=self.max_download_threads, + lz4_compressed=lz4_compressed, + description=description, + ssl_options=self._ssl_options, ) - arrow_queue_opt = ArrowQueue(arrow_results, n_rows, 0) else: arrow_queue_opt = None return ExecuteResponse( @@ -762,6 +771,63 @@ def _results_message_to_execute_response(self, resp, operation_state): arrow_schema_bytes=schema_bytes, ) + def get_execution_result(self, op_handle, cursor): + + assert op_handle is not None + + req = ttypes.TFetchResultsReq( + operationHandle=ttypes.TOperationHandle( + op_handle.operationId, + op_handle.operationType, + False, + op_handle.modifiedRowCount, + ), + maxRows=cursor.arraysize, + maxBytes=cursor.buffer_size_bytes, + orientation=ttypes.TFetchOrientation.FETCH_NEXT, + includeResultSetMetadata=True, + ) + + resp = self.make_request(self._client.FetchResults, req) + + t_result_set_metadata_resp = resp.resultSetMetadata + + lz4_compressed = t_result_set_metadata_resp.lz4Compressed + is_staging_operation = t_result_set_metadata_resp.isStagingOperation + has_more_rows = resp.hasMoreRows + description = self._hive_schema_to_description( + t_result_set_metadata_resp.schema + ) + + schema_bytes = ( + t_result_set_metadata_resp.arrowSchema + or self._hive_schema_to_arrow_schema(t_result_set_metadata_resp.schema) + .serialize() + .to_pybytes() + ) + + queue = ResultSetQueueFactory.build_queue( + row_set_type=resp.resultSetMetadata.resultFormat, + t_row_set=resp.results, + arrow_schema_bytes=schema_bytes, + max_download_threads=self.max_download_threads, + lz4_compressed=lz4_compressed, + description=description, + ssl_options=self._ssl_options, + ) + + return ExecuteResponse( + arrow_queue=queue, + status=resp.status, + has_been_closed_server_side=False, + has_more_rows=has_more_rows, + lz4_compressed=lz4_compressed, + is_staging_operation=is_staging_operation, + command_handle=op_handle, + description=description, + arrow_schema_bytes=schema_bytes, + ) + def _wait_until_command_done(self, op_handle, initial_operation_status_resp): if initial_operation_status_resp: self._check_command_not_in_error_or_closed_state( @@ -780,6 +846,12 @@ def _wait_until_command_done(self, op_handle, initial_operation_status_resp): self._check_command_not_in_error_or_closed_state(op_handle, poll_resp) return operation_state + def get_query_state(self, op_handle) -> "TOperationState": + poll_resp = self._poll_for_status(op_handle) + operation_state = poll_resp.operationState + self._check_command_not_in_error_or_closed_state(op_handle, poll_resp) + return operation_state + @staticmethod def _check_direct_results_for_error(t_spark_direct_results): if t_spark_direct_results: @@ -801,7 +873,16 @@ def _check_direct_results_for_error(t_spark_direct_results): ) def execute_command( - self, operation, session_handle, max_rows, max_bytes, lz4_compression, cursor + self, + operation, + session_handle, + max_rows, + max_bytes, + lz4_compression, + cursor, + use_cloud_fetch=True, + parameters=[], + async_op=False, ): assert session_handle is not None @@ -820,17 +901,22 @@ def execute_command( getDirectResults=ttypes.TSparkGetDirectResults( maxRows=max_rows, maxBytes=max_bytes ), - canReadArrowResult=True, + canReadArrowResult=True if pyarrow else False, canDecompressLZ4Result=lz4_compression, - canDownloadResult=False, + canDownloadResult=use_cloud_fetch, confOverlay={ # We want to receive proper Timestamp arrow types. "spark.thriftserver.arrowBasedRowSet.timestampAsString": "false" }, useArrowNativeTypes=spark_arrow_types, + parameters=parameters, ) resp = self.make_request(self._client.ExecuteStatement, req) - return self._handle_execute_response(resp, cursor) + + if async_op: + self._handle_execute_response_async(resp, cursor) + else: + return self._handle_execute_response(resp, cursor) def get_catalogs(self, session_handle, max_rows, max_bytes, cursor): assert session_handle is not None @@ -929,6 +1015,10 @@ def _handle_execute_response(self, resp, cursor): return self._results_message_to_execute_response(resp, final_operation_state) + def _handle_execute_response_async(self, resp, cursor): + cursor.active_op_handle = resp.operationHandle + self._check_direct_results_for_error(resp.directResults) + def fetch_results( self, op_handle, @@ -938,6 +1028,7 @@ def fetch_results( lz4_compressed, arrow_schema_bytes, description, + use_cloud_fetch=True, ): assert op_handle is not None @@ -951,21 +1042,29 @@ def fetch_results( maxRows=max_rows, maxBytes=max_bytes, orientation=ttypes.TFetchOrientation.FETCH_NEXT, + includeResultSetMetadata=True, ) - resp = self.make_request(self._client.FetchResults, req) + # Fetch results in Inline mode with FETCH_NEXT orientation are not idempotent and hence not retried + resp = self.make_request(self._client.FetchResults, req, use_cloud_fetch) if resp.results.startRowOffset > expected_row_start_offset: - logger.warning( - "Expected results to start from {} but they instead start at {}".format( + raise DataError( + "fetch_results failed due to inconsistency in the state between the client and the server. Expected results to start from {} but they instead start at {}, some result batches must have been skipped".format( expected_row_start_offset, resp.results.startRowOffset ) ) - arrow_results, n_rows = self._create_arrow_table( - resp.results, lz4_compressed, arrow_schema_bytes, description + + queue = ResultSetQueueFactory.build_queue( + row_set_type=resp.resultSetMetadata.resultFormat, + t_row_set=resp.results, + arrow_schema_bytes=arrow_schema_bytes, + max_download_threads=self.max_download_threads, + lz4_compressed=lz4_compressed, + description=description, + ssl_options=self._ssl_options, ) - arrow_queue = ArrowQueue(arrow_results, n_rows) - return arrow_queue, resp.hasMoreRows + return queue, resp.hasMoreRows def close_command(self, op_handle): req = ttypes.TCloseOperationReq(operationHandle=op_handle) @@ -973,10 +1072,39 @@ def close_command(self, op_handle): return resp.status def cancel_command(self, active_op_handle): - logger.debug("Cancelling command {}".format(active_op_handle.operationId.guid)) + logger.debug( + "Cancelling command {}".format( + self.guid_to_hex_id(active_op_handle.operationId.guid) + ) + ) req = ttypes.TCancelOperationReq(active_op_handle) self.make_request(self._client.CancelOperation, req) @staticmethod def handle_to_id(session_handle): return session_handle.sessionId.guid + + @staticmethod + def handle_to_hex_id(session_handle: TCLIService.TSessionHandle): + this_uuid = uuid.UUID(bytes=session_handle.sessionId.guid) + return str(this_uuid) + + @staticmethod + def guid_to_hex_id(guid: bytes) -> str: + """Return a hexadecimal string instead of bytes + + Example: + IN b'\x01\xee\x1d)\xa4\x19\x1d\xb6\xa9\xc0\x8d\xf1\xfe\xbaB\xdd' + OUT '01ee1d29-a419-1db6-a9c0-8df1feba42dd' + + If conversion to hexadecimal fails, the original bytes are returned + """ + + this_uuid: Union[bytes, uuid.UUID] + + try: + this_uuid = uuid.UUID(bytes=guid) + except Exception as e: + logger.debug(f"Unable to convert bytes to UUID: {bytes} -- {str(e)}") + this_uuid = guid + return str(this_uuid) diff --git a/src/databricks/sql/types.py b/src/databricks/sql/types.py index b44704cd..fef22cd9 100644 --- a/src/databricks/sql/types.py +++ b/src/databricks/sql/types.py @@ -16,7 +16,57 @@ # # Row class was taken from Apache Spark pyspark. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union, TypeVar +import datetime +import decimal +from ssl import SSLContext, CERT_NONE, CERT_REQUIRED, create_default_context + + +class SSLOptions: + tls_verify: bool + tls_verify_hostname: bool + tls_trusted_ca_file: Optional[str] + tls_client_cert_file: Optional[str] + tls_client_cert_key_file: Optional[str] + tls_client_cert_key_password: Optional[str] + + def __init__( + self, + tls_verify: bool = True, + tls_verify_hostname: bool = True, + tls_trusted_ca_file: Optional[str] = None, + tls_client_cert_file: Optional[str] = None, + tls_client_cert_key_file: Optional[str] = None, + tls_client_cert_key_password: Optional[str] = None, + ): + self.tls_verify = tls_verify + self.tls_verify_hostname = tls_verify_hostname + self.tls_trusted_ca_file = tls_trusted_ca_file + self.tls_client_cert_file = tls_client_cert_file + self.tls_client_cert_key_file = tls_client_cert_key_file + self.tls_client_cert_key_password = tls_client_cert_key_password + + def create_ssl_context(self) -> SSLContext: + ssl_context = create_default_context(cafile=self.tls_trusted_ca_file) + + if self.tls_verify is False: + ssl_context.check_hostname = False + ssl_context.verify_mode = CERT_NONE + elif self.tls_verify_hostname is False: + ssl_context.check_hostname = False + ssl_context.verify_mode = CERT_REQUIRED + else: + ssl_context.check_hostname = True + ssl_context.verify_mode = CERT_REQUIRED + + if self.tls_client_cert_file: + ssl_context.load_cert_chain( + certfile=self.tls_client_cert_file, + keyfile=self.tls_client_cert_key_file, + password=self.tls_client_cert_key_password, + ) + + return ssl_context class Row(tuple): diff --git a/src/databricks/sql/utils.py b/src/databricks/sql/utils.py index ed558136..cd655c4e 100644 --- a/src/databricks/sql/utils.py +++ b/src/databricks/sql/utils.py @@ -1,16 +1,169 @@ -from collections import namedtuple, OrderedDict +from __future__ import annotations + +import pytz +import datetime +import decimal +from abc import ABC, abstractmethod +from collections import OrderedDict, namedtuple from collections.abc import Iterable -import datetime, decimal +from decimal import Decimal from enum import Enum -from typing import Dict -import pyarrow +from typing import Any, Dict, List, Optional, Union +import re + +import lz4.frame + +try: + import pyarrow +except ImportError: + pyarrow = None + +from databricks.sql import OperationalError, exc +from databricks.sql.cloudfetch.download_manager import ResultFileDownloadManager +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TRowSet, + TSparkArrowResultLink, + TSparkRowSetType, +) +from databricks.sql.types import SSLOptions + +from databricks.sql.parameters.native import ParameterStructure, TDbsqlParameter + +import logging + +BIT_MASKS = [1, 2, 4, 8, 16, 32, 64, 128] +DEFAULT_ERROR_CONTEXT = "Unknown error" + +logger = logging.getLogger(__name__) + + +class ResultSetQueue(ABC): + @abstractmethod + def next_n_rows(self, num_rows: int): + pass -from databricks.sql import exc + @abstractmethod + def remaining_rows(self): + pass -class ArrowQueue: +class ResultSetQueueFactory(ABC): + @staticmethod + def build_queue( + row_set_type: TSparkRowSetType, + t_row_set: TRowSet, + arrow_schema_bytes: bytes, + max_download_threads: int, + ssl_options: SSLOptions, + lz4_compressed: bool = True, + description: Optional[List[List[Any]]] = None, + ) -> ResultSetQueue: + """ + Factory method to build a result set queue. + + Args: + row_set_type (enum): Row set type (Arrow, Column, or URL). + t_row_set (TRowSet): Result containing arrow batches, columns, or cloud fetch links. + arrow_schema_bytes (bytes): Bytes representing the arrow schema. + lz4_compressed (bool): Whether result data has been lz4 compressed. + description (List[List[Any]]): Hive table schema description. + max_download_threads (int): Maximum number of downloader thread pool threads. + ssl_options (SSLOptions): SSLOptions object for CloudFetchQueue + + Returns: + ResultSetQueue + """ + if row_set_type == TSparkRowSetType.ARROW_BASED_SET: + arrow_table, n_valid_rows = convert_arrow_based_set_to_arrow_table( + t_row_set.arrowBatches, lz4_compressed, arrow_schema_bytes + ) + converted_arrow_table = convert_decimals_in_arrow_table( + arrow_table, description + ) + return ArrowQueue(converted_arrow_table, n_valid_rows) + elif row_set_type == TSparkRowSetType.COLUMN_BASED_SET: + column_table, column_names = convert_column_based_set_to_column_table( + t_row_set.columns, description + ) + + converted_column_table = convert_to_assigned_datatypes_in_column_table( + column_table, description + ) + + return ColumnQueue(ColumnTable(converted_column_table, column_names)) + elif row_set_type == TSparkRowSetType.URL_BASED_SET: + return CloudFetchQueue( + schema_bytes=arrow_schema_bytes, + start_row_offset=t_row_set.startRowOffset, + result_links=t_row_set.resultLinks, + lz4_compressed=lz4_compressed, + description=description, + max_download_threads=max_download_threads, + ssl_options=ssl_options, + ) + else: + raise AssertionError("Row set type is not valid") + + +class ColumnTable: + def __init__(self, column_table, column_names): + self.column_table = column_table + self.column_names = column_names + + @property + def num_rows(self): + if len(self.column_table) == 0: + return 0 + else: + return len(self.column_table[0]) + + @property + def num_columns(self): + return len(self.column_names) + + def get_item(self, col_index, row_index): + return self.column_table[col_index][row_index] + + def slice(self, curr_index, length): + sliced_column_table = [ + column[curr_index : curr_index + length] for column in self.column_table + ] + return ColumnTable(sliced_column_table, self.column_names) + + def __eq__(self, other): + return ( + self.column_table == other.column_table + and self.column_names == other.column_names + ) + + +class ColumnQueue(ResultSetQueue): + def __init__(self, column_table: ColumnTable): + self.column_table = column_table + self.cur_row_index = 0 + self.n_valid_rows = column_table.num_rows + + def next_n_rows(self, num_rows): + length = min(num_rows, self.n_valid_rows - self.cur_row_index) + + slice = self.column_table.slice(self.cur_row_index, length) + self.cur_row_index += slice.num_rows + return slice + + def remaining_rows(self): + slice = self.column_table.slice( + self.cur_row_index, self.n_valid_rows - self.cur_row_index + ) + self.cur_row_index += slice.num_rows + return slice + + +class ArrowQueue(ResultSetQueue): def __init__( - self, arrow_table: pyarrow.Table, n_valid_rows: int, start_row_index: int = 0 + self, + arrow_table: "pyarrow.Table", + n_valid_rows: int, + start_row_index: int = 0, ): """ A queue-like wrapper over an Arrow table @@ -23,7 +176,7 @@ def __init__( self.arrow_table = arrow_table self.n_valid_rows = n_valid_rows - def next_n_rows(self, num_rows: int) -> pyarrow.Table: + def next_n_rows(self, num_rows: int) -> "pyarrow.Table": """Get upto the next n rows of the Arrow dataframe""" length = min(num_rows, self.n_valid_rows - self.cur_row_index) # Note that the table.slice API is not the same as Python's slice @@ -32,7 +185,7 @@ def next_n_rows(self, num_rows: int) -> pyarrow.Table: self.cur_row_index += slice.num_rows return slice - def remaining_rows(self) -> pyarrow.Table: + def remaining_rows(self) -> "pyarrow.Table": slice = self.arrow_table.slice( self.cur_row_index, self.n_valid_rows - self.cur_row_index ) @@ -40,6 +193,155 @@ def remaining_rows(self) -> pyarrow.Table: return slice +class CloudFetchQueue(ResultSetQueue): + def __init__( + self, + schema_bytes, + max_download_threads: int, + ssl_options: SSLOptions, + start_row_offset: int = 0, + result_links: Optional[List[TSparkArrowResultLink]] = None, + lz4_compressed: bool = True, + description: Optional[List[List[Any]]] = None, + ): + """ + A queue-like wrapper over CloudFetch arrow batches. + + Attributes: + schema_bytes (bytes): Table schema in bytes. + max_download_threads (int): Maximum number of downloader thread pool threads. + start_row_offset (int): The offset of the first row of the cloud fetch links. + result_links (List[TSparkArrowResultLink]): Links containing the downloadable URL and metadata. + lz4_compressed (bool): Whether the files are lz4 compressed. + description (List[List[Any]]): Hive table schema description. + """ + self.schema_bytes = schema_bytes + self.max_download_threads = max_download_threads + self.start_row_index = start_row_offset + self.result_links = result_links + self.lz4_compressed = lz4_compressed + self.description = description + self._ssl_options = ssl_options + + logger.debug( + "Initialize CloudFetch loader, row set start offset: {}, file list:".format( + start_row_offset + ) + ) + if result_links is not None: + for result_link in result_links: + logger.debug( + "- start row offset: {}, row count: {}".format( + result_link.startRowOffset, result_link.rowCount + ) + ) + self.download_manager = ResultFileDownloadManager( + links=result_links or [], + max_download_threads=self.max_download_threads, + lz4_compressed=self.lz4_compressed, + ssl_options=self._ssl_options, + ) + + self.table = self._create_next_table() + self.table_row_index = 0 + + def next_n_rows(self, num_rows: int) -> "pyarrow.Table": + """ + Get up to the next n rows of the cloud fetch Arrow dataframes. + + Args: + num_rows (int): Number of rows to retrieve. + + Returns: + pyarrow.Table + """ + if not self.table: + logger.debug("CloudFetchQueue: no more rows available") + # Return empty pyarrow table to cause retry of fetch + return self._create_empty_table() + logger.debug("CloudFetchQueue: trying to get {} next rows".format(num_rows)) + results = self.table.slice(0, 0) + while num_rows > 0 and self.table: + # Get remaining of num_rows or the rest of the current table, whichever is smaller + length = min(num_rows, self.table.num_rows - self.table_row_index) + table_slice = self.table.slice(self.table_row_index, length) + results = pyarrow.concat_tables([results, table_slice]) + self.table_row_index += table_slice.num_rows + + # Replace current table with the next table if we are at the end of the current table + if self.table_row_index == self.table.num_rows: + self.table = self._create_next_table() + self.table_row_index = 0 + num_rows -= table_slice.num_rows + + logger.debug("CloudFetchQueue: collected {} next rows".format(results.num_rows)) + return results + + def remaining_rows(self) -> "pyarrow.Table": + """ + Get all remaining rows of the cloud fetch Arrow dataframes. + + Returns: + pyarrow.Table + """ + if not self.table: + # Return empty pyarrow table to cause retry of fetch + return self._create_empty_table() + results = self.table.slice(0, 0) + while self.table: + table_slice = self.table.slice( + self.table_row_index, self.table.num_rows - self.table_row_index + ) + results = pyarrow.concat_tables([results, table_slice]) + self.table_row_index += table_slice.num_rows + self.table = self._create_next_table() + self.table_row_index = 0 + return results + + def _create_next_table(self) -> Union["pyarrow.Table", None]: + logger.debug( + "CloudFetchQueue: Trying to get downloaded file for row {}".format( + self.start_row_index + ) + ) + # Create next table by retrieving the logical next downloaded file, or return None to signal end of queue + downloaded_file = self.download_manager.get_next_downloaded_file( + self.start_row_index + ) + if not downloaded_file: + logger.debug( + "CloudFetchQueue: Cannot find downloaded file for row {}".format( + self.start_row_index + ) + ) + # None signals no more Arrow tables can be built from the remaining handlers if any remain + return None + arrow_table = create_arrow_table_from_arrow_file( + downloaded_file.file_bytes, self.description + ) + + # The server rarely prepares the exact number of rows requested by the client in cloud fetch. + # Subsequently, we drop the extraneous rows in the last file if more rows are retrieved than requested + if arrow_table.num_rows > downloaded_file.row_count: + arrow_table = arrow_table.slice(0, downloaded_file.row_count) + + # At this point, whether the file has extraneous rows or not, the arrow table should have the correct num rows + assert downloaded_file.row_count == arrow_table.num_rows + self.start_row_index += arrow_table.num_rows + + logger.debug( + "CloudFetchQueue: Found downloaded file, row count: {}, new start offset: {}".format( + arrow_table.num_rows, self.start_row_index + ) + ) + + return arrow_table + + def _create_empty_table(self) -> "pyarrow.Table": + # Create a 0-row table with just the schema bytes + return create_arrow_table_from_arrow_file(self.schema_bytes, self.description) + + ExecuteResponse = namedtuple( "ExecuteResponse", "status has_been_closed_server_side has_more_rows description lz4_compressed is_staging_operation " @@ -116,7 +418,12 @@ def user_friendly_error_message(self, no_retry_reason, attempt, elapsed): user_friendly_error_message = "{}: {}".format( user_friendly_error_message, self.error_message ) - return user_friendly_error_message + try: + error_context = str(self.error) + except: + error_context = DEFAULT_ERROR_CONTEXT + + return user_friendly_error_message + ". " + error_context # Taken from PyHive @@ -183,3 +490,264 @@ def escape_item(self, item): def inject_parameters(operation: str, parameters: Dict[str, str]): return operation % parameters + + +def _dbsqlparameter_names(params: List[TDbsqlParameter]) -> list[str]: + return [p.name if p.name else "" for p in params] + + +def _generate_named_interpolation_values( + params: List[TDbsqlParameter], +) -> dict[str, str]: + """Returns a dictionary of the form {name: ":name"} for each parameter in params""" + + names = _dbsqlparameter_names(params) + + return {name: f":{name}" for name in names} + + +def _may_contain_inline_positional_markers(operation: str) -> bool: + """Check for the presence of `%s` in the operation string.""" + + interpolated = operation.replace("%s", "?") + return interpolated != operation + + +def _interpolate_named_markers( + operation: str, parameters: List[TDbsqlParameter] +) -> str: + """Replace all instances of `%(param)s` in `operation` with `:param`. + + If `operation` contains no instances of `%(param)s` then the input string is returned unchanged. + + ``` + "SELECT * FROM table WHERE field = %(field)s and other_field = %(other_field)s" + ``` + + Yields + + ``` + SELECT * FROM table WHERE field = :field and other_field = :other_field + ``` + """ + + _output_operation = operation + + PYFORMAT_PARAMSTYLE_REGEX = r"%\((\w+)\)s" + pat = re.compile(PYFORMAT_PARAMSTYLE_REGEX) + NAMED_PARAMSTYLE_FMT = ":{}" + PYFORMAT_PARAMSTYLE_FMT = "%({})s" + + pyformat_markers = pat.findall(operation) + for marker in pyformat_markers: + pyformat_marker = PYFORMAT_PARAMSTYLE_FMT.format(marker) + named_marker = NAMED_PARAMSTYLE_FMT.format(marker) + _output_operation = _output_operation.replace(pyformat_marker, named_marker) + + return _output_operation + + +def transform_paramstyle( + operation: str, + parameters: List[TDbsqlParameter], + param_structure: ParameterStructure, +) -> str: + """ + Performs a Python string interpolation such that any occurence of `%(param)s` will be replaced with `:param` + + This utility function is built to assist users in the transition between the default paramstyle in + this connector prior to version 3.0.0 (`pyformat`) and the new default paramstyle (`named`). + + Args: + operation: The operation or SQL text to transform. + parameters: The parameters to use for the transformation. + + Returns: + str + """ + output = operation + if ( + param_structure == ParameterStructure.POSITIONAL + and _may_contain_inline_positional_markers(operation) + ): + logger.warning( + "It looks like this query may contain un-named query markers like `%s`" + " This format is not supported when use_inline_params=False." + " Use `?` instead or set use_inline_params=True" + ) + elif param_structure == ParameterStructure.NAMED: + output = _interpolate_named_markers(operation, parameters) + + return output + + +def create_arrow_table_from_arrow_file( + file_bytes: bytes, description +) -> "pyarrow.Table": + arrow_table = convert_arrow_based_file_to_arrow_table(file_bytes) + return convert_decimals_in_arrow_table(arrow_table, description) + + +def convert_arrow_based_file_to_arrow_table(file_bytes: bytes): + try: + return pyarrow.ipc.open_stream(file_bytes).read_all() + except Exception as e: + raise RuntimeError("Failure to convert arrow based file to arrow table", e) + + +def convert_arrow_based_set_to_arrow_table(arrow_batches, lz4_compressed, schema_bytes): + ba = bytearray() + ba += schema_bytes + n_rows = 0 + for arrow_batch in arrow_batches: + n_rows += arrow_batch.rowCount + ba += ( + lz4.frame.decompress(arrow_batch.batch) + if lz4_compressed + else arrow_batch.batch + ) + arrow_table = pyarrow.ipc.open_stream(ba).read_all() + return arrow_table, n_rows + + +def convert_decimals_in_arrow_table(table, description) -> "pyarrow.Table": + for i, col in enumerate(table.itercolumns()): + if description[i][1] == "decimal": + decimal_col = col.to_pandas().apply( + lambda v: v if v is None else Decimal(v) + ) + precision, scale = description[i][4], description[i][5] + assert scale is not None + assert precision is not None + # Spark limits decimal to a maximum scale of 38, + # so 128 is guaranteed to be big enough + dtype = pyarrow.decimal128(precision, scale) + col_data = pyarrow.array(decimal_col, type=dtype) + field = table.field(i).with_type(dtype) + table = table.set_column(i, field, col_data) + return table + + +def convert_to_assigned_datatypes_in_column_table(column_table, description): + + converted_column_table = [] + for i, col in enumerate(column_table): + if description[i][1] == "decimal": + converted_column_table.append( + tuple(v if v is None else Decimal(v) for v in col) + ) + elif description[i][1] == "date": + converted_column_table.append( + tuple(v if v is None else datetime.date.fromisoformat(v) for v in col) + ) + elif description[i][1] == "timestamp": + converted_column_table.append( + tuple( + ( + v + if v is None + else datetime.datetime.strptime( + v, "%Y-%m-%d %H:%M:%S.%f" + ).replace(tzinfo=pytz.UTC) + ) + for v in col + ) + ) + else: + converted_column_table.append(col) + + return converted_column_table + + +def convert_column_based_set_to_arrow_table(columns, description): + arrow_table = pyarrow.Table.from_arrays( + [_convert_column_to_arrow_array(c) for c in columns], + # Only use the column names from the schema, the types are determined by the + # physical types used in column based set, as they can differ from the + # mapping used in _hive_schema_to_arrow_schema. + names=[c[0] for c in description], + ) + return arrow_table, arrow_table.num_rows + + +def convert_column_based_set_to_column_table(columns, description): + column_names = [c[0] for c in description] + column_table = [_convert_column_to_list(c) for c in columns] + + return column_table, column_names + + +def _convert_column_to_arrow_array(t_col): + """ + Return a pyarrow array from the values in a TColumn instance. + Note that ColumnBasedSet has no native support for complex types, so they will be converted + to strings server-side. + """ + field_name_to_arrow_type = { + "boolVal": pyarrow.bool_(), + "byteVal": pyarrow.int8(), + "i16Val": pyarrow.int16(), + "i32Val": pyarrow.int32(), + "i64Val": pyarrow.int64(), + "doubleVal": pyarrow.float64(), + "stringVal": pyarrow.string(), + "binaryVal": pyarrow.binary(), + } + for field in field_name_to_arrow_type.keys(): + wrapper = getattr(t_col, field) + if wrapper: + return _create_arrow_array(wrapper, field_name_to_arrow_type[field]) + + raise OperationalError("Empty TColumn instance {}".format(t_col)) + + +def _convert_column_to_list(t_col): + SUPPORTED_FIELD_TYPES = ( + "boolVal", + "byteVal", + "i16Val", + "i32Val", + "i64Val", + "doubleVal", + "stringVal", + "binaryVal", + ) + + for field in SUPPORTED_FIELD_TYPES: + wrapper = getattr(t_col, field) + if wrapper: + return _create_python_tuple(wrapper) + + raise OperationalError("Empty TColumn instance {}".format(t_col)) + + +def _create_arrow_array(t_col_value_wrapper, arrow_type): + result = t_col_value_wrapper.values + nulls = t_col_value_wrapper.nulls # bitfield describing which values are null + assert isinstance(nulls, bytes) + + # The number of bits in nulls can be both larger or smaller than the number of + # elements in result, so take the minimum of both to iterate over. + length = min(len(result), len(nulls) * 8) + + for i in range(length): + if nulls[i >> 3] & BIT_MASKS[i & 0x7]: + result[i] = None + + return pyarrow.array(result, type=arrow_type) + + +def _create_python_tuple(t_col_value_wrapper): + result = t_col_value_wrapper.values + nulls = t_col_value_wrapper.nulls # bitfield describing which values are null + assert isinstance(nulls, bytes) + + # The number of bits in nulls can be both larger or smaller than the number of + # elements in result, so take the minimum of both to iterate over. + length = min(len(result), len(nulls) * 8) + + for i in range(length): + if nulls[i >> 3] & BIT_MASKS[i & 0x7]: + result[i] = None + + return tuple(result) diff --git a/src/databricks/sqlalchemy/__init__.py b/src/databricks/sqlalchemy/__init__.py deleted file mode 100644 index 1df1e1d4..00000000 --- a/src/databricks/sqlalchemy/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from databricks.sqlalchemy.dialect import DatabricksDialect diff --git a/src/databricks/sqlalchemy/dialect/__init__.py b/src/databricks/sqlalchemy/dialect/__init__.py deleted file mode 100644 index da508bb0..00000000 --- a/src/databricks/sqlalchemy/dialect/__init__.py +++ /dev/null @@ -1,316 +0,0 @@ -"""This module's layout loosely follows example of SQLAlchemy's postgres dialect -""" - -import decimal, re, datetime -from dateutil.parser import parse - -from sqlalchemy import types, processors, event -from sqlalchemy.engine import default, Engine -from sqlalchemy.exc import DatabaseError -from sqlalchemy.engine import reflection - -from databricks import sql - - -from databricks.sqlalchemy.dialect.base import ( - DatabricksDDLCompiler, - DatabricksIdentifierPreparer, -) -from databricks.sqlalchemy.dialect.compiler import DatabricksTypeCompiler - -try: - import alembic -except ImportError: - pass -else: - from alembic.ddl import DefaultImpl - - class DatabricksImpl(DefaultImpl): - __dialect__ = "databricks" - - -class DatabricksDecimal(types.TypeDecorator): - """Translates strings to decimals""" - - impl = types.DECIMAL - - def process_result_value(self, value, dialect): - if value is not None: - return decimal.Decimal(value) - else: - return None - - -class DatabricksTimestamp(types.TypeDecorator): - """Translates timestamp strings to datetime objects""" - - impl = types.TIMESTAMP - - def process_result_value(self, value, dialect): - return value - - def adapt(self, impltype, **kwargs): - return self.impl - - -class DatabricksDate(types.TypeDecorator): - """Translates date strings to date objects""" - - impl = types.DATE - - def process_result_value(self, value, dialect): - return value - - def adapt(self, impltype, **kwargs): - return self.impl - - -class DatabricksDialect(default.DefaultDialect): - """This dialect implements only those methods required to pass our e2e tests""" - - # Possible attributes are defined here: https://docs.sqlalchemy.org/en/14/core/internals.html#sqlalchemy.engine.Dialect - name: str = "databricks" - driver: str = "databricks-sql-python" - default_schema_name: str = "default" - - preparer = DatabricksIdentifierPreparer - type_compiler = DatabricksTypeCompiler - ddl_compiler = DatabricksDDLCompiler - supports_statement_cache: bool = True - supports_multivalues_insert: bool = True - supports_native_decimal: bool = True - supports_sane_rowcount: bool = False - - @classmethod - def dbapi(cls): - return sql - - def create_connect_args(self, url): - # TODO: can schema be provided after HOST? - # Expected URI format is: databricks+thrift://token:dapi***@***.cloud.databricks.com?http_path=/sql/*** - - kwargs = { - "server_hostname": url.host, - "access_token": url.password, - "http_path": url.query.get("http_path"), - "catalog": url.query.get("catalog"), - "schema": url.query.get("schema"), - } - - self.schema = kwargs["schema"] - self.catalog = kwargs["catalog"] - - return [], kwargs - - def get_columns(self, connection, table_name, schema=None, **kwargs): - """Return information about columns in `table_name`. - - Given a :class:`_engine.Connection`, a string - `table_name`, and an optional string `schema`, return column - information as a list of dictionaries with these keys: - - name - the column's name - - type - [sqlalchemy.types#TypeEngine] - - nullable - boolean - - default - the column's default value - - autoincrement - boolean - - sequence - a dictionary of the form - {'name' : str, 'start' :int, 'increment': int, 'minvalue': int, - 'maxvalue': int, 'nominvalue': bool, 'nomaxvalue': bool, - 'cycle': bool, 'cache': int, 'order': bool} - - Additional column attributes may be present. - """ - - _type_map = { - "boolean": types.Boolean, - "smallint": types.SmallInteger, - "int": types.Integer, - "bigint": types.BigInteger, - "float": types.Float, - "double": types.Float, - "string": types.String, - "varchar": types.String, - "char": types.String, - "binary": types.String, - "array": types.String, - "map": types.String, - "struct": types.String, - "uniontype": types.String, - "decimal": DatabricksDecimal, - "timestamp": DatabricksTimestamp, - "date": DatabricksDate, - } - - with self.get_driver_connection( - connection - )._dbapi_connection.dbapi_connection.cursor() as cur: - resp = cur.columns( - catalog_name=self.catalog, - schema_name=schema or self.schema, - table_name=table_name, - ).fetchall() - - columns = [] - - for col in resp: - - # Taken from PyHive. This removes added type info from decimals and maps - _col_type = re.search(r"^\w+", col.TYPE_NAME).group(0) - this_column = { - "name": col.COLUMN_NAME, - "type": _type_map[_col_type.lower()], - "nullable": bool(col.NULLABLE), - "default": col.COLUMN_DEF, - "autoincrement": False if col.IS_AUTO_INCREMENT == "NO" else True, - } - columns.append(this_column) - - return columns - - def get_pk_constraint(self, connection, table_name, schema=None, **kw): - """Return information about the primary key constraint on - table_name`. - - Given a :class:`_engine.Connection`, a string - `table_name`, and an optional string `schema`, return primary - key information as a dictionary with these keys: - - constrained_columns - a list of column names that make up the primary key - - name - optional name of the primary key constraint. - - """ - # TODO: implement this behaviour - return {"constrained_columns": []} - - def get_foreign_keys(self, connection, table_name, schema=None, **kw): - """Return information about foreign_keys in `table_name`. - - Given a :class:`_engine.Connection`, a string - `table_name`, and an optional string `schema`, return foreign - key information as a list of dicts with these keys: - - name - the constraint's name - - constrained_columns - a list of column names that make up the foreign key - - referred_schema - the name of the referred schema - - referred_table - the name of the referred table - - referred_columns - a list of column names in the referred table that correspond to - constrained_columns - """ - # TODO: Implement this behaviour - return [] - - def get_indexes(self, connection, table_name, schema=None, **kw): - """Return information about indexes in `table_name`. - - Given a :class:`_engine.Connection`, a string - `table_name` and an optional string `schema`, return index - information as a list of dictionaries with these keys: - - name - the index's name - - column_names - list of column names in order - - unique - boolean - """ - # TODO: Implement this behaviour - return [] - - def get_table_names(self, connection, schema=None, **kwargs): - TABLE_NAME = 1 - with self.get_driver_connection( - connection - )._dbapi_connection.dbapi_connection.cursor() as cur: - sql_str = "SHOW TABLES FROM {}".format( - ".".join([self.catalog, schema or self.schema]) - ) - data = cur.execute(sql_str).fetchall() - _tables = [i[TABLE_NAME] for i in data] - - return _tables - - def get_view_names(self, connection, schema=None, **kwargs): - VIEW_NAME = 1 - with self.get_driver_connection( - connection - )._dbapi_connection.dbapi_connection.cursor() as cur: - sql_str = "SHOW VIEWS FROM {}".format( - ".".join([self.catalog, schema or self.schema]) - ) - data = cur.execute(sql_str).fetchall() - _tables = [i[VIEW_NAME] for i in data] - - return _tables - - def do_rollback(self, dbapi_connection): - # Databricks SQL Does not support transactions - pass - - def has_table(self, connection, table_name, schema=None, **kwargs) -> bool: - """SQLAlchemy docstrings say dialect providers must implement this method""" - - schema = schema or "default" - - # DBR >12.x uses underscores in error messages - DBR_LTE_12_NOT_FOUND_STRING = "Table or view not found" - DBR_GT_12_NOT_FOUND_STRING = "TABLE_OR_VIEW_NOT_FOUND" - - try: - res = connection.execute(f"DESCRIBE TABLE {table_name}") - return True - except DatabaseError as e: - if DBR_GT_12_NOT_FOUND_STRING in str( - e - ) or DBR_LTE_12_NOT_FOUND_STRING in str(e): - return False - else: - raise e - - @reflection.cache - def get_schema_names(self, connection, **kw): - # Equivalent to SHOW DATABASES - - # TODO: replace with call to cursor.schemas() once its performance matches raw SQL - return [row[0] for row in connection.execute("SHOW SCHEMAS")] - - -@event.listens_for(Engine, "do_connect") -def receive_do_connect(dialect, conn_rec, cargs, cparams): - """Helpful for DS on traffic from clients using SQLAlchemy in particular""" - - # Ignore connect invocations that don't use our dialect - if not dialect.name == "databricks": - return - - if "_user_agent_entry" in cparams: - new_user_agent = f"sqlalchemy + {cparams['_user_agent_entry']}" - else: - new_user_agent = "sqlalchemy" - - cparams["_user_agent_entry"] = new_user_agent diff --git a/src/databricks/sqlalchemy/dialect/base.py b/src/databricks/sqlalchemy/dialect/base.py deleted file mode 100644 index 080f0410..00000000 --- a/src/databricks/sqlalchemy/dialect/base.py +++ /dev/null @@ -1,17 +0,0 @@ -import re -from sqlalchemy.sql import compiler - - -class DatabricksIdentifierPreparer(compiler.IdentifierPreparer): - # SparkSQL identifier specification: - # ref: https://spark.apache.org/docs/latest/sql-ref-identifier.html - - legal_characters = re.compile(r"^[A-Z0-9_]+$", re.I) - - def __init__(self, dialect): - super().__init__(dialect, initial_quote="`") - - -class DatabricksDDLCompiler(compiler.DDLCompiler): - def post_create_table(self, table): - return " USING DELTA" diff --git a/src/databricks/sqlalchemy/dialect/compiler.py b/src/databricks/sqlalchemy/dialect/compiler.py deleted file mode 100644 index f77807ed..00000000 --- a/src/databricks/sqlalchemy/dialect/compiler.py +++ /dev/null @@ -1,38 +0,0 @@ -from sqlalchemy.sql import compiler - - -class DatabricksTypeCompiler(compiler.GenericTypeCompiler): - """Originally forked from pyhive""" - - def visit_INTEGER(self, type_): - return "INT" - - def visit_NUMERIC(self, type_): - return "DECIMAL" - - def visit_CHAR(self, type_): - return "STRING" - - def visit_VARCHAR(self, type_): - return "STRING" - - def visit_NCHAR(self, type_): - return "STRING" - - def visit_TEXT(self, type_): - return "STRING" - - def visit_CLOB(self, type_): - return "STRING" - - def visit_BLOB(self, type_): - return "BINARY" - - def visit_TIME(self, type_): - return "TIMESTAMP" - - def visit_DATE(self, type_): - return "DATE" - - def visit_DATETIME(self, type_): - return "TIMESTAMP" diff --git a/test.env.example b/test.env.example new file mode 100644 index 00000000..f99abc9d --- /dev/null +++ b/test.env.example @@ -0,0 +1,11 @@ +# Authentication details for running e2e tests +DATABRICKS_SERVER_HOSTNAME= +DATABRICKS_HTTP_PATH= +DATABRICKS_TOKEN= + +# Only required to run the PySQLStagingIngestionTestSuite +DATABRICKS_USER= + +# Only required to run SQLAlchemy tests +DATABRICKS_CATALOG= +DATABRICKS_SCHEMA= diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/common/core_tests.py b/tests/e2e/common/core_tests.py index cd325e8d..3f0fdc05 100644 --- a/tests/e2e/common/core_tests.py +++ b/tests/e2e/common/core_tests.py @@ -3,14 +3,20 @@ from collections import namedtuple TypeFailure = namedtuple( - "TypeFailure", "query,columnType,resultType,resultValue," - "actualValue,actualType,description,conf") + "TypeFailure", + "query,columnType,resultType,resultValue," + "actualValue,actualType,description,conf", +) ResultFailure = namedtuple( - "ResultFailure", "query,columnType,resultType,resultValue," - "actualValue,actualType,description,conf") + "ResultFailure", + "query,columnType,resultType,resultValue," + "actualValue,actualType,description,conf", +) ExecFailure = namedtuple( - "ExecFailure", "query,columnType,resultType,resultValue," - "actualValue,actualType,description,conf,error") + "ExecFailure", + "query,columnType,resultType,resultValue," + "actualValue,actualType,description,conf,error", +) class SmokeTestMixin: @@ -18,8 +24,8 @@ def test_smoke_test(self): with self.cursor() as cursor: cursor.execute("select 0") rows = cursor.fetchall() - self.assertEqual(len(rows), 1) - self.assertEqual(rows[0][0], 0) + assert len(rows) == 1 + assert rows[0][0] == 0 class CoreTestMixin: @@ -32,69 +38,115 @@ class CoreTestMixin: # A list of (subquery, column_type, python_type, expected_result) # To be executed as "SELECT {} FROM RANGE(...)" and "SELECT {}" range_queries = [ - ("TRUE", 'boolean', bool, True), - ("cast(1 AS TINYINT)", 'byte', int, 1), - ("cast(1000 AS SMALLINT)", 'short', int, 1000), - ("cast(100000 AS INTEGER)", 'integer', int, 100000), - ("cast(10000000000000 AS BIGINT)", 'long', int, 10000000000000), - ("cast(100.001 AS DECIMAL(6, 3))", 'decimal', decimal.Decimal, 100.001), - ("date '2020-02-20'", 'date', datetime.date, datetime.date(2020, 2, 20)), - ("unhex('f000')", 'binary', bytes, b'\xf0\x00'), # pyodbc internal mismatch - ("'foo'", 'string', str, 'foo'), + ("TRUE", "boolean", bool, True), + ("cast(1 AS TINYINT)", "byte", int, 1), + ("cast(1000 AS SMALLINT)", "short", int, 1000), + ("cast(100000 AS INTEGER)", "integer", int, 100000), + ("cast(10000000000000 AS BIGINT)", "long", int, 10000000000000), + ("cast(100.001 AS DECIMAL(6, 3))", "decimal", decimal.Decimal, 100.001), + ("date '2020-02-20'", "date", datetime.date, datetime.date(2020, 2, 20)), + ("unhex('f000')", "binary", bytes, b"\xf0\x00"), # pyodbc internal mismatch + ("'foo'", "string", str, "foo"), # SPARK-32130: 6.x: "4 weeks 2 days" vs 7.x: "30 days" # ("interval 30 days", str, str, "interval 4 weeks 2 days"), # ("interval 3 days", str, str, "interval 3 days"), - ("CAST(NULL AS DOUBLE)", 'double', type(None), None), + ("CAST(NULL AS DOUBLE)", "double", type(None), None), ] # Full queries, only the first column of the first row is checked - queries = [("NULL UNION (SELECT 1) order by 1", 'integer', type(None), None)] + queries = [("NULL UNION (SELECT 1) order by 1", "integer", type(None), None)] def run_tests_on_queries(self, default_conf): failures = [] - for (query, columnType, rowValueType, answer) in self.range_queries: + for query, columnType, rowValueType, answer in self.range_queries: with self.cursor(default_conf) as cursor: failures.extend( - self.run_query(cursor, query, columnType, rowValueType, answer, default_conf)) + self.run_query( + cursor, query, columnType, rowValueType, answer, default_conf + ) + ) failures.extend( - self.run_range_query(cursor, query, columnType, rowValueType, answer, - default_conf)) + self.run_range_query( + cursor, query, columnType, rowValueType, answer, default_conf + ) + ) - for (query, columnType, rowValueType, answer) in self.queries: + for query, columnType, rowValueType, answer in self.queries: with self.cursor(default_conf) as cursor: failures.extend( - self.run_query(cursor, query, columnType, rowValueType, answer, default_conf)) + self.run_query( + cursor, query, columnType, rowValueType, answer, default_conf + ) + ) if failures: - self.fail("Failed testing result set with Arrow. " - "Failed queries: {}".format("\n\n".join([str(f) for f in failures]))) + self.fail( + "Failed testing result set with Arrow. " + "Failed queries: {}".format("\n\n".join([str(f) for f in failures])) + ) def run_query(self, cursor, query, columnType, rowValueType, answer, conf): full_query = "SELECT {}".format(query) expected_column_types = self.expected_column_types(columnType) try: cursor.execute(full_query) - (result, ) = cursor.fetchone() - if not all(cursor.description[0][1] == type for type in expected_column_types): + (result,) = cursor.fetchone() + if not all( + cursor.description[0][1] == type for type in expected_column_types + ): return [ - TypeFailure(full_query, expected_column_types, rowValueType, answer, result, - type(result), cursor.description, conf) + TypeFailure( + full_query, + expected_column_types, + rowValueType, + answer, + result, + type(result), + cursor.description, + conf, + ) ] if self.validate_row_value_type and type(result) is not rowValueType: return [ - TypeFailure(full_query, expected_column_types, rowValueType, answer, result, - type(result), cursor.description, conf) + TypeFailure( + full_query, + expected_column_types, + rowValueType, + answer, + result, + type(result), + cursor.description, + conf, + ) ] if self.validate_result and str(answer) != str(result): return [ - ResultFailure(full_query, query, expected_column_types, rowValueType, answer, - result, type(result), cursor.description, conf) + ResultFailure( + full_query, + query, + expected_column_types, + rowValueType, + answer, + result, + type(result), + cursor.description, + conf, + ) ] return [] except Exception as e: return [ - ExecFailure(full_query, columnType, rowValueType, None, None, None, - cursor.description, conf, e) + ExecFailure( + full_query, + columnType, + rowValueType, + None, + None, + None, + cursor.description, + conf, + e, + ) ] def run_range_query(self, cursor, query, columnType, rowValueType, expected, conf): @@ -107,25 +159,63 @@ def run_range_query(self, cursor, query, columnType, rowValueType, expected, con if len(rows) <= 0: break for index, (result, id) in enumerate(rows): - if not all(cursor.description[0][1] == type for type in expected_column_types): + if not all( + cursor.description[0][1] == type + for type in expected_column_types + ): return [ - TypeFailure(full_query, expected_column_types, rowValueType, expected, - result, type(result), cursor.description, conf) + TypeFailure( + full_query, + expected_column_types, + rowValueType, + expected, + result, + type(result), + cursor.description, + conf, + ) ] - if self.validate_row_value_type and type(result) \ - is not rowValueType: + if ( + self.validate_row_value_type + and type(result) is not rowValueType + ): return [ - TypeFailure(full_query, expected_column_types, rowValueType, expected, - result, type(result), cursor.description, conf) + TypeFailure( + full_query, + expected_column_types, + rowValueType, + expected, + result, + type(result), + cursor.description, + conf, + ) ] if self.validate_result and str(expected) != str(result): return [ - ResultFailure(full_query, expected_column_types, rowValueType, expected, - result, type(result), cursor.description, conf) + ResultFailure( + full_query, + expected_column_types, + rowValueType, + expected, + result, + type(result), + cursor.description, + conf, + ) ] return [] except Exception as e: return [ - ExecFailure(full_query, columnType, rowValueType, None, None, None, - cursor.description, conf, e) + ExecFailure( + full_query, + columnType, + rowValueType, + None, + None, + None, + cursor.description, + conf, + e, + ) ] diff --git a/tests/e2e/common/decimal_tests.py b/tests/e2e/common/decimal_tests.py index 8051d2a1..0029f30c 100644 --- a/tests/e2e/common/decimal_tests.py +++ b/tests/e2e/common/decimal_tests.py @@ -1,15 +1,24 @@ from decimal import Decimal import pyarrow +import pytest class DecimalTestsMixin: decimal_and_expected_results = [ ("100.001 AS DECIMAL(6, 3)", Decimal("100.001"), pyarrow.decimal128(6, 3)), - ("1000000.0000 AS DECIMAL(11, 4)", Decimal("1000000.0000"), pyarrow.decimal128(11, 4)), - ("-10.2343 AS DECIMAL(10, 6)", Decimal("-10.234300"), pyarrow.decimal128(10, 6)), + ( + "1000000.0000 AS DECIMAL(11, 4)", + Decimal("1000000.0000"), + pyarrow.decimal128(11, 4), + ), + ( + "-10.2343 AS DECIMAL(10, 6)", + Decimal("-10.234300"), + pyarrow.decimal128(10, 6), + ), # TODO(SC-90767): Re-enable this test after we have a way of passing `ansi_mode` = False - #("-13872347.2343 AS DECIMAL(10, 10)", None, pyarrow.decimal128(10, 10)), + # ("-13872347.2343 AS DECIMAL(10, 10)", None, pyarrow.decimal128(10, 10)), ("NULL AS DECIMAL(1, 1)", None, pyarrow.decimal128(1, 1)), ("1 AS DECIMAL(1, 0)", Decimal("1"), pyarrow.decimal128(1, 0)), ("0.00000 AS DECIMAL(5, 3)", Decimal("0.000"), pyarrow.decimal128(5, 3)), @@ -17,32 +26,40 @@ class DecimalTestsMixin: ] multi_decimals_and_expected_results = [ - (["1 AS DECIMAL(6, 3)", "100.001 AS DECIMAL(6, 3)", "NULL AS DECIMAL(6, 3)"], - [Decimal("1.00"), Decimal("100.001"), None], pyarrow.decimal128(6, 3)), - (["1 AS DECIMAL(6, 3)", "2 AS DECIMAL(5, 2)"], [Decimal('1.000'), - Decimal('2.000')], pyarrow.decimal128(6, - 3)), + ( + ["1 AS DECIMAL(6, 3)", "100.001 AS DECIMAL(6, 3)", "NULL AS DECIMAL(6, 3)"], + [Decimal("1.00"), Decimal("100.001"), None], + pyarrow.decimal128(6, 3), + ), + ( + ["1 AS DECIMAL(6, 3)", "2 AS DECIMAL(5, 2)"], + [Decimal("1.000"), Decimal("2.000")], + pyarrow.decimal128(6, 3), + ), ] - def test_decimals(self): + @pytest.mark.parametrize( + "decimal, expected_value, expected_type", decimal_and_expected_results + ) + def test_decimals(self, decimal, expected_value, expected_type): with self.cursor({}) as cursor: - for (decimal, expected_value, expected_type) in self.decimal_and_expected_results: - query = "SELECT CAST ({})".format(decimal) - with self.subTest(query=query): - cursor.execute(query) - table = cursor.fetchmany_arrow(1) - self.assertEqual(table.field(0).type, expected_type) - self.assertEqual(table.to_pydict().popitem()[1][0], expected_value) + query = "SELECT CAST ({})".format(decimal) + cursor.execute(query) + table = cursor.fetchmany_arrow(1) + assert table.field(0).type == expected_type + assert table.to_pydict().popitem()[1][0] == expected_value - def test_multi_decimals(self): + @pytest.mark.parametrize( + "decimals, expected_values, expected_type", multi_decimals_and_expected_results + ) + def test_multi_decimals(self, decimals, expected_values, expected_type): with self.cursor({}) as cursor: - for (decimals, expected_values, - expected_type) in self.multi_decimals_and_expected_results: - union_str = " UNION ".join(["(SELECT CAST ({}))".format(dec) for dec in decimals]) - query = "SELECT * FROM ({}) ORDER BY 1 NULLS LAST".format(union_str) + union_str = " UNION ".join( + ["(SELECT CAST ({}))".format(dec) for dec in decimals] + ) + query = "SELECT * FROM ({}) ORDER BY 1 NULLS LAST".format(union_str) - with self.subTest(query=query): - cursor.execute(query) - table = cursor.fetchall_arrow() - self.assertEqual(table.field(0).type, expected_type) - self.assertEqual(table.to_pydict().popitem()[1], expected_values) + cursor.execute(query) + table = cursor.fetchall_arrow() + assert table.field(0).type == expected_type + assert table.to_pydict().popitem()[1] == expected_values diff --git a/tests/e2e/common/large_queries_mixin.py b/tests/e2e/common/large_queries_mixin.py index 3e1e45bc..41ef029b 100644 --- a/tests/e2e/common/large_queries_mixin.py +++ b/tests/e2e/common/large_queries_mixin.py @@ -35,8 +35,12 @@ def fetch_rows(self, cursor, row_count, fetchmany_size): num_fetches = max(math.ceil(n / 10000), 1) latency_ms = int((time.time() - start_time) * 1000 / num_fetches), 1 - print('Fetched {} rows with an avg latency of {} per fetch, '.format(n, latency_ms) + - 'assuming 10K fetch size.') + print( + "Fetched {} rows with an avg latency of {} per fetch, ".format( + n, latency_ms + ) + + "assuming 10K fetch size." + ) def test_query_with_large_wide_result_set(self): resultSize = 300 * 1000 * 1000 # 300 MB @@ -50,14 +54,19 @@ def test_query_with_large_wide_result_set(self): self.arraysize = 1000 with self.cursor() as cursor: for lz4_compression in [False, True]: - cursor.connection.lz4_compression=lz4_compression + cursor.connection.lz4_compression = lz4_compression uuids = ", ".join(["uuid() uuid{}".format(i) for i in range(cols)]) - cursor.execute("SELECT id, {uuids} FROM RANGE({rows})".format(uuids=uuids, rows=rows)) - self.assertEqual(lz4_compression, cursor.active_result_set.lz4_compressed) - for row_id, row in enumerate(self.fetch_rows(cursor, rows, fetchmany_size)): - self.assertEqual(row[0], row_id) # Verify no rows are dropped in the middle. - self.assertEqual(len(row[1]), 36) - + cursor.execute( + "SELECT id, {uuids} FROM RANGE({rows})".format( + uuids=uuids, rows=rows + ) + ) + assert lz4_compression == cursor.active_result_set.lz4_compressed + for row_id, row in enumerate( + self.fetch_rows(cursor, rows, fetchmany_size) + ): + assert row[0] == row_id # Verify no rows are dropped in the middle. + assert len(row[1]) == 36 def test_query_with_large_narrow_result_set(self): resultSize = 300 * 1000 * 1000 # 300 MB @@ -71,10 +80,10 @@ def test_query_with_large_narrow_result_set(self): with self.cursor() as cursor: cursor.execute("SELECT * FROM RANGE({rows})".format(rows=rows)) for row_id, row in enumerate(self.fetch_rows(cursor, rows, fetchmany_size)): - self.assertEqual(row[0], row_id) + assert row[0] == row_id def test_long_running_query(self): - """ Incrementally increase query size until it takes at least 5 minutes, + """Incrementally increase query size until it takes at least 5 minutes, and asserts that the query completes successfully. """ minutes = 60 @@ -85,20 +94,24 @@ def test_long_running_query(self): scale_factor = 1 with self.cursor() as cursor: while duration < min_duration: - self.assertLess(scale_factor, 512, msg="Detected infinite loop") + assert scale_factor < 512, "Detected infinite loop" start = time.time() - cursor.execute("""SELECT count(*) + cursor.execute( + """SELECT count(*) FROM RANGE({scale}) x JOIN RANGE({scale0}) y - ON from_unixtime(x.id * y.id, "yyyy-MM-dd") LIKE "%not%a%date%" - """.format(scale=scale_factor * scale0, scale0=scale0)) + ON from_unixtime(x.id * y.id, "yyyy-MM-dd") LIKE "%not%a%date%" + """.format( + scale=scale_factor * scale0, scale0=scale0 + ) + ) - n, = cursor.fetchone() - self.assertEqual(n, 0) + (n,) = cursor.fetchone() + assert n == 0 duration = time.time() - start current_fraction = duration / min_duration - print('Took {} s with scale factor={}'.format(duration, scale_factor)) + print("Took {} s with scale factor={}".format(duration, scale_factor)) # Extrapolate linearly to reach 5 min and add 50% padding to push over the limit scale_factor = math.ceil(1.5 * scale_factor / current_fraction) diff --git a/tests/e2e/common/predicates.py b/tests/e2e/common/predicates.py index 3450087f..61de69fd 100644 --- a/tests/e2e/common/predicates.py +++ b/tests/e2e/common/predicates.py @@ -10,7 +10,8 @@ def pysql_supports_arrow(): """Import databricks.sql and test whether Cursor has fetchall_arrow.""" from databricks.sql.client import Cursor - return hasattr(Cursor, 'fetchall_arrow') + + return hasattr(Cursor, "fetchall_arrow") def pysql_has_version(compare, version): @@ -25,20 +26,21 @@ def test_some_pyhive_v1_stuff(): ... """ from databricks import sql + return compare_module_version(sql, compare, version) def is_endpoint_test(cli_args=None): - + # Currently only supporting tests against DBSQL Endpoints # So we don't read `is_endpoint_test` from the CLI args - return True + return True def compare_dbr_versions(cli_args, compare, major_version, minor_version): if MAJOR_DBR_V_KEY in cli_args and MINOR_DBR_V_KEY in cli_args: if cli_args[MINOR_DBR_V_KEY] == "x": - actual_minor_v = float('inf') + actual_minor_v = float("inf") else: actual_minor_v = int(cli_args[MINOR_DBR_V_KEY]) dbr_version = (int(cli_args[MAJOR_DBR_V_KEY]), actual_minor_v) @@ -47,8 +49,10 @@ def compare_dbr_versions(cli_args, compare, major_version, minor_version): if not is_endpoint_test(): raise ValueError( - "DBR version not provided for non-endpoint test. Please pass the {} and {} params". - format(MAJOR_DBR_V_KEY, MINOR_DBR_V_KEY)) + "DBR version not provided for non-endpoint test. Please pass the {} and {} params".format( + MAJOR_DBR_V_KEY, MINOR_DBR_V_KEY + ) + ) def is_thrift_v5_plus(cli_args): @@ -56,18 +60,18 @@ def is_thrift_v5_plus(cli_args): _compare_fns = { - '<': '__lt__', - '<=': '__le__', - '>': '__gt__', - '>=': '__ge__', - '==': '__eq__', - '!=': '__ne__', + "<": "__lt__", + "<=": "__le__", + ">": "__gt__", + ">=": "__ge__", + "==": "__eq__", + "!=": "__ne__", } def compare_versions(compare, v1_tuple, v2_tuple): compare_fn_name = _compare_fns.get(compare) - assert compare_fn_name, 'Received invalid compare string: ' + compare + assert compare_fn_name, "Received invalid compare string: " + compare return getattr(v1_tuple, compare_fn_name)(v2_tuple) @@ -87,13 +91,15 @@ def test_some_pyhive_v1_stuff(): NOTE: This comparison leverages packaging.version.parse, and compares _release_ versions, thus ignoring pre/post release tags (eg -rc1, -dev, etc). """ - assert module, 'Received invalid module: ' + module - assert getattr(module, '__version__'), 'Received module with no version: ' + module + assert module, "Received invalid module: " + module + assert getattr(module, "__version__"), "Received module with no version: " + module def validate_version(version): v = parse_version(str(version)) # assert that we get a PEP-440 Version back -- LegacyVersion doesn't have major/minor. - assert hasattr(v, 'major'), 'Module has incompatible "Legacy" version: ' + version + assert hasattr(v, "major"), ( + 'Module has incompatible "Legacy" version: ' + version + ) return (v.major, v.minor, v.micro) mod_version = validate_version(module.__version__) diff --git a/tests/e2e/common/retry_test_mixins.py b/tests/e2e/common/retry_test_mixins.py old mode 100644 new mode 100755 index a088ba1e..942955ca --- a/tests/e2e/common/retry_test_mixins.py +++ b/tests/e2e/common/retry_test_mixins.py @@ -1,3 +1,21 @@ +from contextlib import contextmanager +import time +from typing import Optional, List +from unittest.mock import MagicMock, PropertyMock, patch + +import pytest +from urllib3.exceptions import MaxRetryError + +from databricks.sql.auth.retry import DatabricksRetryPolicy +from databricks.sql.exc import ( + MaxRetryDurationError, + NonRecoverableNetworkError, + RequestError, + SessionAlreadyClosedError, + UnsafeToRetryError, +) + + class Client429ResponseMixin: def test_client_should_retry_automatically_when_getting_429(self): with self.cursor() as cursor: @@ -8,15 +26,17 @@ def test_client_should_retry_automatically_when_getting_429(self): self.assertEqual(rows[0][0], 1) def test_client_should_not_retry_429_if_RateLimitRetry_is_0(self): - with self.assertRaises(self.error_type) as cm: + with pytest.raises(self.error_type) as cm: with self.cursor(self.conf_to_disable_rate_limit_retries) as cursor: for _ in range(10): cursor.execute("SELECT 1") rows = cursor.fetchall() self.assertEqual(len(rows), 1) self.assertEqual(rows[0][0], 1) - expected = "Maximum rate of 1 requests per SECOND has been exceeded. " \ - "Please reduce the rate of requests and try again after 1 seconds." + expected = ( + "Maximum rate of 1 requests per SECOND has been exceeded. " + "Please reduce the rate of requests and try again after 1 seconds." + ) exception_str = str(cm.exception) # FIXME (Ali Smesseim, 7-Jul-2020): ODBC driver does not always return the @@ -32,7 +52,404 @@ def test_wait_cluster_startup(self): cursor.fetchall() def _test_retry_disabled_with_message(self, error_msg_substring, exception_type): - with self.assertRaises(exception_type) as cm: + with pytest.raises(exception_type) as cm: with self.connection(self.conf_to_disable_temporarily_unavailable_retries): pass - self.assertIn(error_msg_substring, str(cm.exception)) + assert error_msg_substring in str(cm.exception) + + +@contextmanager +def mocked_server_response( + status: int = 200, headers: dict = {}, redirect_location: Optional[str] = None +): + """Context manager for patching urllib3 responses""" + + # When mocking mocking a BaseHTTPResponse for urllib3 the mock must include + # 1. A status code + # 2. A headers dict + # 3. mock.get_redirect_location() return falsy by default + + # `msg` is included for testing when urllib3~=1.0.0 is installed + mock_response = MagicMock(headers=headers, msg=headers, status=status) + mock_response.get_redirect_location.return_value = ( + False if redirect_location is None else redirect_location + ) + + with patch("urllib3.connectionpool.HTTPSConnectionPool._get_conn") as getconn_mock: + getconn_mock.return_value.getresponse.return_value = mock_response + try: + yield getconn_mock + finally: + pass + + +@contextmanager +def mock_sequential_server_responses(responses: List[dict]): + """Same as the mocked_server_response context manager but it will yield + the provided responses in the order received + + `responses` should be a list of dictionaries containing these members: + - status: int + - headers: dict + - redirect_location: str + """ + + mock_responses = [] + + # Each resp should have these members: + + for resp in responses: + _mock = MagicMock( + headers=resp["headers"], msg=resp["headers"], status=resp["status"] + ) + _mock.get_redirect_location.return_value = ( + False if resp["redirect_location"] is None else resp["redirect_location"] + ) + mock_responses.append(_mock) + + with patch("urllib3.connectionpool.HTTPSConnectionPool._get_conn") as getconn_mock: + getconn_mock.return_value.getresponse.side_effect = mock_responses + try: + yield getconn_mock + finally: + pass + + +class PySQLRetryTestsMixin: + """Home for retry tests where we patch urllib to return different codes and monitor that it tries to retry""" + + # For testing purposes + _retry_policy = { + "_retry_delay_min": 0.1, + "_retry_delay_max": 5, + "_retry_stop_after_attempts_count": 5, + "_retry_stop_after_attempts_duration": 10, + "_retry_delay_default": 0.5, + } + + def test_retry_urllib3_settings_are_honored(self): + """Databricks overrides some of urllib3's configuration. This tests confirms that what configuration + we DON'T override is preserved in urllib3's internals + """ + + urllib3_config = {"connect": 10, "read": 11, "redirect": 12} + rp = DatabricksRetryPolicy( + delay_min=0.1, + delay_max=10.0, + stop_after_attempts_count=10, + stop_after_attempts_duration=10.0, + delay_default=1.0, + force_dangerous_codes=[], + urllib3_kwargs=urllib3_config, + ) + + assert rp.connect == 10 + assert rp.read == 11 + assert rp.redirect == 12 + + def test_oserror_retries(self): + """If a network error occurs during make_request, the request is retried according to policy""" + with patch( + "urllib3.connectionpool.HTTPSConnectionPool._validate_conn", + ) as mock_validate_conn: + mock_validate_conn.side_effect = OSError("Some arbitrary network error") + with pytest.raises(MaxRetryError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + + assert mock_validate_conn.call_count == 6 + + def test_retry_max_count_not_exceeded(self): + """GIVEN the max_attempts_count is 5 + WHEN the server sends nothing but 429 responses + THEN the connector issues six request (original plus five retries) + before raising an exception + """ + with mocked_server_response(status=404) as mock_obj: + with pytest.raises(MaxRetryError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + assert mock_obj.return_value.getresponse.call_count == 6 + + def test_retry_exponential_backoff(self): + """GIVEN the retry policy is configured for reasonable exponential backoff + WHEN the server sends nothing but 429 responses with retry-afters + THEN the connector will use those retry-afters values as delay + """ + retry_policy = self._retry_policy.copy() + retry_policy["_retry_delay_min"] = 1 + + time_start = time.time() + with mocked_server_response( + status=429, headers={"Retry-After": "3"} + ) as mock_obj: + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=retry_policy) as conn: + pass + + duration = time.time() - time_start + assert isinstance(cm.value.args[1], MaxRetryDurationError) + + # With setting delay_min to 1, the expected retry delays should be: + # 3, 3, 3, 3 + # The first 3 retries are allowed, the 4th retry puts the total duration over the limit + # of 10 seconds + assert mock_obj.return_value.getresponse.call_count == 4 + assert duration > 6 + + # Should be less than 7, but this is a safe margin for CI/CD slowness + assert duration < 10 + + def test_retry_max_duration_not_exceeded(self): + """GIVEN the max attempt duration of 10 seconds + WHEN the server sends a Retry-After header of 60 seconds + THEN the connector raises a MaxRetryDurationError + """ + with mocked_server_response(status=429, headers={"Retry-After": "60"}): + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + assert isinstance(cm.value.args[1], MaxRetryDurationError) + + def test_retry_abort_non_recoverable_error(self): + """GIVEN the server returns a code 501 + WHEN the connector receives this response + THEN nothing is retried and an exception is raised + """ + + # Code 501 is a Not Implemented error + with mocked_server_response(status=501): + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + assert isinstance(cm.value.args[1], NonRecoverableNetworkError) + + def test_retry_abort_unsafe_execute_statement_retry_condition(self): + """GIVEN the server sends a code other than 429 or 503 + WHEN the connector sent an ExecuteStatement command + THEN nothing is retried because it's idempotent + """ + with self.connection(extra_params=self._retry_policy) as conn: + with conn.cursor() as cursor: + # Code 502 is a Bad Gateway, which we commonly see in production under heavy load + with mocked_server_response(status=502): + with pytest.raises(RequestError) as cm: + cursor.execute("Not a real query") + assert isinstance(cm.value.args[1], UnsafeToRetryError) + + def test_retry_dangerous_codes(self): + """GIVEN the server sends a dangerous code and the user forced this to be retryable + WHEN the connector sent an ExecuteStatement command + THEN the command is retried + """ + + # These http codes are not retried by default + # For some applications, idempotency is not important so we give users a way to force retries anyway + DANGEROUS_CODES = [502, 504, 400] + + additional_settings = { + "_retry_dangerous_codes": DANGEROUS_CODES, + "_retry_stop_after_attempts_count": 1, + } + + # Prove that these codes are not retried by default + with self.connection(extra_params={**self._retry_policy}) as conn: + with conn.cursor() as cursor: + for dangerous_code in DANGEROUS_CODES: + with mocked_server_response(status=dangerous_code): + with pytest.raises(RequestError) as cm: + cursor.execute("Not a real query") + assert isinstance(cm.value.args[1], UnsafeToRetryError) + + # Prove that these codes are retried if forced by the user + with self.connection( + extra_params={**self._retry_policy, **additional_settings} + ) as conn: + with conn.cursor() as cursor: + for dangerous_code in DANGEROUS_CODES: + with mocked_server_response(status=dangerous_code): + with pytest.raises(MaxRetryError) as cm: + cursor.execute("Not a real query") + + def test_retry_safe_execute_statement_retry_condition(self): + """GIVEN the server sends either code 429 or 503 + WHEN the connector sent an ExecuteStatement command + THEN the request is retried because these are idempotent + """ + + responses = [ + {"status": 429, "headers": {"Retry-After": "1"}, "redirect_location": None}, + {"status": 503, "headers": {}, "redirect_location": None}, + ] + + with self.connection( + extra_params={**self._retry_policy, "_retry_stop_after_attempts_count": 1} + ) as conn: + with conn.cursor() as cursor: + # Code 502 is a Bad Gateway, which we commonly see in production under heavy load + with mock_sequential_server_responses(responses) as mock_obj: + with pytest.raises(MaxRetryError): + cursor.execute("This query never reaches the server") + assert mock_obj.return_value.getresponse.call_count == 2 + + def test_retry_abort_close_session_on_404(self, caplog): + """GIVEN the connector sends a CloseSession command + WHEN server sends a 404 (which is normally retried) + THEN nothing is retried because 404 means the session already closed + """ + + # First response is a Bad Gateway -> Result is the command actually goes through + # Second response is a 404 because the session is no longer found + responses = [ + {"status": 502, "headers": {"Retry-After": "1"}, "redirect_location": None}, + {"status": 404, "headers": {}, "redirect_location": None}, + ] + + with self.connection(extra_params={**self._retry_policy}) as conn: + with mock_sequential_server_responses(responses): + conn.close() + assert "Session was closed by a prior request" in caplog.text + + def test_retry_abort_close_operation_on_404(self, caplog): + """GIVEN the connector sends a CancelOperation command + WHEN server sends a 404 (which is normally retried) + THEN nothing is retried because 404 means the operation was already canceled + """ + + # First response is a Bad Gateway -> Result is the command actually goes through + # Second response is a 404 because the session is no longer found + responses = [ + {"status": 502, "headers": {"Retry-After": "1"}, "redirect_location": None}, + {"status": 404, "headers": {}, "redirect_location": None}, + ] + + with self.connection(extra_params={**self._retry_policy}) as conn: + with conn.cursor() as curs: + with patch( + "databricks.sql.utils.ExecuteResponse.has_been_closed_server_side", + new_callable=PropertyMock, + return_value=False, + ): + # This call guarantees we have an open cursor at the server + curs.execute("SELECT 1") + with mock_sequential_server_responses(responses): + curs.close() + assert ( + "Operation was canceled by a prior request" in caplog.text + ) + + def test_retry_max_redirects_raises_too_many_redirects_exception(self): + """GIVEN the connector is configured with a custom max_redirects + WHEN the DatabricksRetryPolicy is created + THEN the connector raises a MaxRedirectsError if that number is exceeded + """ + + max_redirects, expected_call_count = 1, 2 + + # Code 302 is a redirect + with mocked_server_response( + status=302, redirect_location="/foo.bar" + ) as mock_obj: + with pytest.raises(MaxRetryError) as cm: + with self.connection( + extra_params={ + **self._retry_policy, + "_retry_max_redirects": max_redirects, + } + ): + pass + assert "too many redirects" == str(cm.value.reason) + # Total call count should be 2 (original + 1 retry) + assert mock_obj.return_value.getresponse.call_count == expected_call_count + + def test_retry_max_redirects_unset_doesnt_redirect_forever(self): + """GIVEN the connector is configured without a custom max_redirects + WHEN the DatabricksRetryPolicy is used + THEN the connector raises a MaxRedirectsError if that number is exceeded + + This test effectively guarantees that regardless of _retry_max_redirects, + _stop_after_attempts_count is enforced. + """ + # Code 302 is a redirect + with mocked_server_response( + status=302, redirect_location="/foo.bar/" + ) as mock_obj: + with pytest.raises(MaxRetryError) as cm: + with self.connection( + extra_params={ + **self._retry_policy, + } + ): + pass + + # Total call count should be 6 (original + _retry_stop_after_attempts_count) + assert mock_obj.return_value.getresponse.call_count == 6 + + def test_retry_max_redirects_is_bounded_by_stop_after_attempts_count(self): + # If I add another 503 or 302 here the test will fail with a MaxRetryError + responses = [ + {"status": 302, "headers": {}, "redirect_location": "/foo.bar"}, + {"status": 500, "headers": {}, "redirect_location": None}, + ] + + additional_settings = { + "_retry_max_redirects": 1, + "_retry_stop_after_attempts_count": 2, + } + + with pytest.raises(RequestError) as cm: + with mock_sequential_server_responses(responses): + with self.connection( + extra_params={**self._retry_policy, **additional_settings} + ): + pass + + # The error should be the result of the 500, not because of too many requests. + assert "too many redirects" not in str(cm.value.message) + assert "Error during request to server" in str(cm.value.message) + + def test_retry_max_redirects_exceeds_max_attempts_count_warns_user(self, caplog): + with self.connection( + extra_params={ + **self._retry_policy, + **{ + "_retry_max_redirects": 100, + "_retry_stop_after_attempts_count": 1, + }, + } + ): + assert "it will have no affect!" in caplog.text + + def test_retry_legacy_behavior_warns_user(self, caplog): + with self.connection( + extra_params={**self._retry_policy, "_enable_v3_retries": False} + ): + assert ( + "Legacy retry behavior is enabled for this connection." in caplog.text + ) + + def test_403_not_retried(self): + """GIVEN the server returns a code 403 + WHEN the connector receives this response + THEN nothing is retried and an exception is raised + """ + + # Code 403 is a Forbidden error + with mocked_server_response(status=403): + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + assert isinstance(cm.value.args[1], NonRecoverableNetworkError) + + def test_401_not_retried(self): + """GIVEN the server returns a code 401 + WHEN the connector receives this response + THEN nothing is retried and an exception is raised + """ + + # Code 401 is an Unauthorized error + with mocked_server_response(status=401): + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=self._retry_policy): + pass + assert isinstance(cm.value.args[1], NonRecoverableNetworkError) diff --git a/tests/e2e/common/staging_ingestion_tests.py b/tests/e2e/common/staging_ingestion_tests.py new file mode 100644 index 00000000..008055e3 --- /dev/null +++ b/tests/e2e/common/staging_ingestion_tests.py @@ -0,0 +1,349 @@ +import os +import tempfile + +import pytest +import databricks.sql as sql +from databricks.sql import Error + + +@pytest.fixture(scope="module", autouse=True) +def check_staging_ingestion_user(ingestion_user): + """This fixture verifies that a staging ingestion user email address + is present in the environment and raises an exception if not. The fixture + only evaluates when the test _isn't skipped_. + """ + + if ingestion_user is None: + raise ValueError( + "To run this test you must designate a `DATABRICKS_USER` environment variable. This will be the user associated with the personal access token." + ) + + +class PySQLStagingIngestionTestSuiteMixin: + """Simple namespace for ingestion tests. These should be run against DBR >12.x + + In addition to connection credentials (host, path, token) this suite requires an env var + named staging_ingestion_user""" + + def test_staging_ingestion_life_cycle(self, ingestion_user): + """PUT a file into the staging location + GET the file from the staging location + REMOVE the file from the staging location + Try to GET the file again expecting to raise an exception + """ + + # PUT should succeed + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + with self.connection( + extra_params={"staging_allowed_local_path": temp_path} + ) as conn: + + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + # GET should succeed + + new_fh, new_temp_path = tempfile.mkstemp() + + with self.connection( + extra_params={"staging_allowed_local_path": new_temp_path} + ) as conn: + cursor = conn.cursor() + query = f"GET 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' TO '{new_temp_path}'" + cursor.execute(query) + + with open(new_fh, "rb") as fp: + fetched_text = fp.read() + + assert fetched_text == original_text + + # REMOVE should succeed + + remove_query = f"REMOVE 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv'" + + with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + + # GET after REMOVE should fail + + with pytest.raises( + Error, match="Staging operation over HTTP was unsuccessful: 404" + ): + cursor = conn.cursor() + query = f"GET 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' TO '{new_temp_path}'" + cursor.execute(query) + + os.remove(temp_path) + os.remove(new_temp_path) + + def test_staging_ingestion_put_fails_without_staging_allowed_local_path( + self, ingestion_user + ): + """PUT operations are not supported unless the connection was built with + a parameter called staging_allowed_local_path + """ + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + with pytest.raises( + Error, match="You must provide at least one staging_allowed_local_path" + ): + with self.connection() as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_put_fails_if_localFile_not_in_staging_allowed_local_path( + self, ingestion_user + ): + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + base_path, filename = os.path.split(temp_path) + + # Add junk to base_path + base_path = os.path.join(base_path, "temp") + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + with self.connection( + extra_params={"staging_allowed_local_path": base_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_put_fails_if_file_exists_and_overwrite_not_set( + self, ingestion_user + ): + """PUT a file into the staging location twice. First command should succeed. Second should fail.""" + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + def perform_put(): + with self.connection( + extra_params={"staging_allowed_local_path": temp_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/12/15/file1.csv'" + cursor.execute(query) + + def perform_remove(): + try: + remove_query = ( + f"REMOVE 'stage://tmp/{ingestion_user}/tmp/12/15/file1.csv'" + ) + + with self.connection( + extra_params={"staging_allowed_local_path": "/"} + ) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + except Exception: + pass + + # Make sure file does not exist + perform_remove() + + # Put the file + perform_put() + + # Try to put it again + with pytest.raises( + sql.exc.ServerOperationError, match="FILE_IN_STAGING_PATH_ALREADY_EXISTS" + ): + perform_put() + + # Clean up after ourselves + perform_remove() + + def test_staging_ingestion_fails_to_modify_another_staging_user(self): + """The server should only allow modification of the staging_ingestion_user's files""" + + some_other_user = "mary.poppins@databricks.com" + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + def perform_put(): + with self.connection( + extra_params={"staging_allowed_local_path": temp_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{some_other_user}/tmp/12/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def perform_remove(): + remove_query = f"REMOVE 'stage://tmp/{some_other_user}/tmp/12/15/file1.csv'" + + with self.connection( + extra_params={"staging_allowed_local_path": "/"} + ) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + + def perform_get(): + with self.connection( + extra_params={"staging_allowed_local_path": temp_path} + ) as conn: + cursor = conn.cursor() + query = f"GET 'stage://tmp/{some_other_user}/tmp/11/15/file1.csv' TO '{temp_path}'" + cursor.execute(query) + + # PUT should fail with permissions error + with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): + perform_put() + + # REMOVE should fail with permissions error + with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): + perform_remove() + + # GET should fail with permissions error + with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): + perform_get() + + def test_staging_ingestion_put_fails_if_absolute_localFile_not_in_staging_allowed_local_path( + self, ingestion_user + ): + """ + This test confirms that staging_allowed_local_path and target_file are resolved into absolute paths. + """ + + # If these two paths are not resolved absolutely, they appear to share a common path of /var/www/html + # after resolution their common path is only /var/www which should raise an exception + # Because the common path must always be equal to staging_allowed_local_path + staging_allowed_local_path = "/var/www/html" + target_file = "/var/www/html/../html1/not_allowed.html" + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_empty_local_path_fails_to_parse_at_server( + self, ingestion_user + ): + staging_allowed_local_path = "/var/www/html" + target_file = "" + + with pytest.raises(Error, match="EMPTY_LOCAL_FILE_IN_STAGING_ACCESS_QUERY"): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_invalid_staging_path_fails_at_server( + self, ingestion_user + ): + staging_allowed_local_path = "/var/www/html" + target_file = "index.html" + + with pytest.raises(Error, match="INVALID_STAGING_PATH_IN_STAGING_ACCESS_QUERY"): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO 'stageRANDOMSTRINGOFCHARACTERS://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_supports_multiple_staging_allowed_local_path_values( + self, ingestion_user + ): + """staging_allowed_local_path may be either a path-like object or a list of path-like objects. + + This test confirms that two configured base paths: + 1 - doesn't raise an exception + 2 - allows uploads from both paths + 3 - doesn't allow uploads from a third path + """ + + def generate_file_and_path_and_queries(): + """ + 1. Makes a temp file with some contents. + 2. Write a query to PUT it into a staging location + 3. Write a query to REMOVE it from that location (for cleanup) + """ + fh, temp_path = tempfile.mkstemp() + with open(fh, "wb") as fp: + original_text = "hello world!".encode("utf-8") + fp.write(original_text) + put_query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/{id(temp_path)}.csv' OVERWRITE" + remove_query = ( + f"REMOVE 'stage://tmp/{ingestion_user}/tmp/11/15/{id(temp_path)}.csv'" + ) + return fh, temp_path, put_query, remove_query + + ( + fh1, + temp_path1, + put_query1, + remove_query1, + ) = generate_file_and_path_and_queries() + ( + fh2, + temp_path2, + put_query2, + remove_query2, + ) = generate_file_and_path_and_queries() + ( + fh3, + temp_path3, + put_query3, + remove_query3, + ) = generate_file_and_path_and_queries() + + with self.connection( + extra_params={"staging_allowed_local_path": [temp_path1, temp_path2]} + ) as conn: + cursor = conn.cursor() + + cursor.execute(put_query1) + cursor.execute(put_query2) + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + cursor.execute(put_query3) + + # Then clean up the files we made + cursor.execute(remove_query1) + cursor.execute(remove_query2) diff --git a/tests/e2e/common/timestamp_tests.py b/tests/e2e/common/timestamp_tests.py index 38b14e9e..70ded7d0 100644 --- a/tests/e2e/common/timestamp_tests.py +++ b/tests/e2e/common/timestamp_tests.py @@ -1,29 +1,34 @@ import datetime +import pytest + from .predicates import compare_dbr_versions, is_thrift_v5_plus, pysql_has_version class TimestampTestsMixin: - timestamp_and_expected_results = [ - ('2021-09-30 11:27:35.123+04:00', datetime.datetime(2021, 9, 30, 7, 27, 35, 123000)), - ('2021-09-30 11:27:35+04:00', datetime.datetime(2021, 9, 30, 7, 27, 35)), - ('2021-09-30 11:27:35.123', datetime.datetime(2021, 9, 30, 11, 27, 35, 123000)), - ('2021-09-30 11:27:35', datetime.datetime(2021, 9, 30, 11, 27, 35)), - ('2021-09-30 11:27', datetime.datetime(2021, 9, 30, 11, 27)), - ('2021-09-30 11', datetime.datetime(2021, 9, 30, 11)), - ('2021-09-30', datetime.datetime(2021, 9, 30)), - ('2021-09', datetime.datetime(2021, 9, 1)), - ('2021', datetime.datetime(2021, 1, 1)), - ('9999-12-31T15:59:59', datetime.datetime(9999, 12, 31, 15, 59, 59)), - ('9999-99-31T15:59:59', None), + date_and_expected_results = [ + ("2021-09-30", datetime.date(2021, 9, 30)), + ("2021-09", datetime.date(2021, 9, 1)), + ("2021", datetime.date(2021, 1, 1)), + ("9999-12-31", datetime.date(9999, 12, 31)), + ("9999-99-31", None), ] - date_and_expected_results = [ - ('2021-09-30', datetime.date(2021, 9, 30)), - ('2021-09', datetime.date(2021, 9, 1)), - ('2021', datetime.date(2021, 1, 1)), - ('9999-12-31', datetime.date(9999, 12, 31)), - ('9999-99-31', None), + timestamp_and_expected_results = [ + ( + "2021-09-30 11:27:35.123+04:00", + datetime.datetime(2021, 9, 30, 7, 27, 35, 123000), + ), + ("2021-09-30 11:27:35+04:00", datetime.datetime(2021, 9, 30, 7, 27, 35)), + ("2021-09-30 11:27:35.123", datetime.datetime(2021, 9, 30, 11, 27, 35, 123000)), + ("2021-09-30 11:27:35", datetime.datetime(2021, 9, 30, 11, 27, 35)), + ("2021-09-30 11:27", datetime.datetime(2021, 9, 30, 11, 27)), + ("2021-09-30 11", datetime.datetime(2021, 9, 30, 11)), + ("2021-09-30", datetime.datetime(2021, 9, 30)), + ("2021-09", datetime.datetime(2021, 9, 1)), + ("2021", datetime.datetime(2021, 1, 1)), + ("9999-12-31T15:59:59", datetime.datetime(9999, 12, 31, 15, 59, 59)), + ("9999-99-31T15:59:59", None), ] def should_add_timezone(self): @@ -31,7 +36,7 @@ def should_add_timezone(self): def maybe_add_timezone_to_timestamp(self, ts): """If we're using DBR >= 10.2, then we expect back aware timestamps, so add timezone to `ts` - Otherwise we have naive timestamps, so no change is needed + Otherwise we have naive timestamps, so no change is needed """ if ts and self.should_add_timezone(): return ts.replace(tzinfo=datetime.timezone.utc) @@ -39,20 +44,28 @@ def maybe_add_timezone_to_timestamp(self, ts): return ts def assertTimestampsEqual(self, result, expected): - self.assertEqual(result, self.maybe_add_timezone_to_timestamp(expected)) + assert result == self.maybe_add_timezone_to_timestamp(expected) def multi_query(self, n_rows=10): row_sql = "SELECT " + ", ".join( - ["TIMESTAMP('{}')".format(ts) for (ts, _) in self.timestamp_and_expected_results]) + [ + "TIMESTAMP('{}')".format(ts) + for (ts, _) in self.timestamp_and_expected_results + ] + ) query = " UNION ALL ".join([row_sql for _ in range(n_rows)]) - expected_matrix = [[dt for (_, dt) in self.timestamp_and_expected_results] - for _ in range(n_rows)] + expected_matrix = [ + [dt for (_, dt) in self.timestamp_and_expected_results] + for _ in range(n_rows) + ] return query, expected_matrix def test_timestamps(self): with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: - for (timestamp, expected) in self.timestamp_and_expected_results: - cursor.execute("SELECT TIMESTAMP('{timestamp}')".format(timestamp=timestamp)) + for timestamp, expected in self.timestamp_and_expected_results: + cursor.execute( + "SELECT TIMESTAMP('{timestamp}')".format(timestamp=timestamp) + ) result = cursor.fetchone()[0] self.assertTimestampsEqual(result, expected) @@ -62,13 +75,14 @@ def test_multi_timestamps(self): cursor.execute(query) result = cursor.fetchall() # We list-ify the rows because PyHive will return a tuple for a row - self.assertEqual([list(r) for r in result], - [[self.maybe_add_timezone_to_timestamp(ts) for ts in r] - for r in expected]) + assert [list(r) for r in result] == [ + [self.maybe_add_timezone_to_timestamp(ts) for ts in r] for r in expected + ] - def test_dates(self): + @pytest.mark.parametrize("date, expected", date_and_expected_results) + def test_dates(self, date, expected): with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: - for (date, expected) in self.date_and_expected_results: + for date, expected in self.date_and_expected_results: cursor.execute("SELECT DATE('{date}')".format(date=date)) result = cursor.fetchone()[0] - self.assertEqual(result, expected) + assert result == expected diff --git a/tests/e2e/common/uc_volume_tests.py b/tests/e2e/common/uc_volume_tests.py new file mode 100644 index 00000000..72e2f502 --- /dev/null +++ b/tests/e2e/common/uc_volume_tests.py @@ -0,0 +1,295 @@ +import os +import tempfile + +import pytest +import databricks.sql as sql +from databricks.sql import Error + + +@pytest.fixture(scope="module", autouse=True) +def check_catalog_and_schema(catalog, schema): + """This fixture verifies that a catalog and schema are present in the environment. + The fixture only evaluates when the test _isn't skipped_. + """ + + if catalog is None or schema is None: + raise ValueError( + f"UC Volume tests require values for the `catalog` and `schema` environment variables. Found catalog {_catalog} schema {_schema}" + ) + + +class PySQLUCVolumeTestSuiteMixin: + """Simple namespace for UC Volume tests. + + In addition to connection credentials (host, path, token) this suite requires env vars + named catalog and schema""" + + def test_uc_volume_life_cycle(self, catalog, schema): + """PUT a file into the UC Volume + GET the file from the UC Volume + REMOVE the file from the UC Volume + Try to GET the file again expecting to raise an exception + """ + + # PUT should succeed + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + with self.connection( + extra_params={"staging_allowed_local_path": temp_path} + ) as conn: + + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + # GET should succeed + + new_fh, new_temp_path = tempfile.mkstemp() + + with self.connection( + extra_params={"staging_allowed_local_path": new_temp_path} + ) as conn: + cursor = conn.cursor() + query = f"GET '/Volumes/{catalog}/{schema}/e2etests/file1.csv' TO '{new_temp_path}'" + cursor.execute(query) + + with open(new_fh, "rb") as fp: + fetched_text = fp.read() + + assert fetched_text == original_text + + # REMOVE should succeed + + remove_query = f"REMOVE '/Volumes/{catalog}/{schema}/e2etests/file1.csv'" + + with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + + # GET after REMOVE should fail + + with pytest.raises( + Error, match="Staging operation over HTTP was unsuccessful: 404" + ): + cursor = conn.cursor() + query = f"GET '/Volumes/{catalog}/{schema}/e2etests/file1.csv' TO '{new_temp_path}'" + cursor.execute(query) + + os.remove(temp_path) + os.remove(new_temp_path) + + def test_uc_volume_put_fails_without_staging_allowed_local_path( + self, catalog, schema + ): + """PUT operations are not supported unless the connection was built with + a parameter called staging_allowed_local_path + """ + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + with pytest.raises( + Error, match="You must provide at least one staging_allowed_local_path" + ): + with self.connection() as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_put_fails_if_localFile_not_in_staging_allowed_local_path( + self, catalog, schema + ): + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + base_path, filename = os.path.split(temp_path) + + # Add junk to base_path + base_path = os.path.join(base_path, "temp") + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + with self.connection( + extra_params={"staging_allowed_local_path": base_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_put_fails_if_file_exists_and_overwrite_not_set( + self, catalog, schema + ): + """PUT a file into the staging location twice. First command should succeed. Second should fail.""" + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + def perform_put(): + with self.connection( + extra_params={"staging_allowed_local_path": temp_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv'" + cursor.execute(query) + + def perform_remove(): + try: + remove_query = ( + f"REMOVE '/Volumes/{catalog}/{schema}/e2etests/file1.csv'" + ) + + with self.connection( + extra_params={"staging_allowed_local_path": "/"} + ) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + except Exception: + pass + + # Make sure file does not exist + perform_remove() + + # Put the file + perform_put() + + # Try to put it again + with pytest.raises( + sql.exc.ServerOperationError, match="FILE_IN_STAGING_PATH_ALREADY_EXISTS" + ): + perform_put() + + # Clean up after ourselves + perform_remove() + + def test_uc_volume_put_fails_if_absolute_localFile_not_in_staging_allowed_local_path( + self, catalog, schema + ): + """ + This test confirms that staging_allowed_local_path and target_file are resolved into absolute paths. + """ + + # If these two paths are not resolved absolutely, they appear to share a common path of /var/www/html + # after resolution their common path is only /var/www which should raise an exception + # Because the common path must always be equal to staging_allowed_local_path + staging_allowed_local_path = "/var/www/html" + target_file = "/var/www/html/../html1/not_allowed.html" + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_empty_local_path_fails_to_parse_at_server(self, catalog, schema): + staging_allowed_local_path = "/var/www/html" + target_file = "" + + with pytest.raises(Error, match="EMPTY_LOCAL_FILE_IN_STAGING_ACCESS_QUERY"): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_invalid_volume_path_fails_at_server(self, catalog, schema): + staging_allowed_local_path = "/var/www/html" + target_file = "index.html" + + with pytest.raises(Error, match="NOT_FOUND: Catalog"): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO '/Volumes/RANDOMSTRINGOFCHARACTERS/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_supports_multiple_staging_allowed_local_path_values( + self, catalog, schema + ): + """staging_allowed_local_path may be either a path-like object or a list of path-like objects. + + This test confirms that two configured base paths: + 1 - doesn't raise an exception + 2 - allows uploads from both paths + 3 - doesn't allow uploads from a third path + """ + + def generate_file_and_path_and_queries(): + """ + 1. Makes a temp file with some contents. + 2. Write a query to PUT it into a staging location + 3. Write a query to REMOVE it from that location (for cleanup) + """ + fh, temp_path = tempfile.mkstemp() + with open(fh, "wb") as fp: + original_text = "hello world!".encode("utf-8") + fp.write(original_text) + put_query = f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/{id(temp_path)}.csv' OVERWRITE" + remove_query = ( + f"REMOVE '/Volumes/{catalog}/{schema}/e2etests/{id(temp_path)}.csv'" + ) + return fh, temp_path, put_query, remove_query + + ( + fh1, + temp_path1, + put_query1, + remove_query1, + ) = generate_file_and_path_and_queries() + ( + fh2, + temp_path2, + put_query2, + remove_query2, + ) = generate_file_and_path_and_queries() + ( + fh3, + temp_path3, + put_query3, + remove_query3, + ) = generate_file_and_path_and_queries() + + with self.connection( + extra_params={"staging_allowed_local_path": [temp_path1, temp_path2]} + ) as conn: + cursor = conn.cursor() + + cursor.execute(put_query1) + cursor.execute(put_query2) + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + cursor.execute(put_query3) + + # Then clean up the files we made + cursor.execute(remove_query1) + cursor.execute(remove_query2) diff --git a/tests/e2e/driver_tests.py b/tests/e2e/driver_tests.py deleted file mode 100644 index 1c09d70e..00000000 --- a/tests/e2e/driver_tests.py +++ /dev/null @@ -1,917 +0,0 @@ -from contextlib import contextmanager -from collections import OrderedDict -import datetime -import io -import logging -import os -import sys -import tempfile -import threading -import time -from unittest import loader, skipIf, skipUnless, TestCase -from uuid import uuid4 - -import numpy as np -import pyarrow -import pytz -import thrift -import pytest - -import databricks.sql as sql -from databricks.sql import STRING, BINARY, NUMBER, DATETIME, DATE, DatabaseError, Error, OperationalError -from tests.e2e.common.predicates import pysql_has_version, pysql_supports_arrow, compare_dbr_versions, is_thrift_v5_plus -from tests.e2e.common.core_tests import CoreTestMixin, SmokeTestMixin -from tests.e2e.common.large_queries_mixin import LargeQueriesMixin -from tests.e2e.common.timestamp_tests import TimestampTestsMixin -from tests.e2e.common.decimal_tests import DecimalTestsMixin -from tests.e2e.common.retry_test_mixins import Client429ResponseMixin, Client503ResponseMixin - -log = logging.getLogger(__name__) - -# manually decorate DecimalTestsMixin to need arrow support -for name in loader.getTestCaseNames(DecimalTestsMixin, 'test_'): - fn = getattr(DecimalTestsMixin, name) - decorated = skipUnless(pysql_supports_arrow(), 'Decimal tests need arrow support')(fn) - setattr(DecimalTestsMixin, name, decorated) - -get_args_from_env = True - - -class PySQLTestCase(TestCase): - error_type = Error - conf_to_disable_rate_limit_retries = {"_retry_stop_after_attempts_count": 1} - conf_to_disable_temporarily_unavailable_retries = {"_retry_stop_after_attempts_count": 1} - - def __init__(self, method_name): - super().__init__(method_name) - # If running in local mode, just use environment variables for params. - self.arguments = os.environ if get_args_from_env else {} - self.arraysize = 1000 - - def connection_params(self, arguments): - params = { - "server_hostname": arguments["host"], - "http_path": arguments["http_path"], - **self.auth_params(arguments) - } - - return params - - def auth_params(self, arguments): - return { - "_username": arguments.get("rest_username"), - "_password": arguments.get("rest_password"), - "access_token": arguments.get("access_token") - } - - @contextmanager - def connection(self, extra_params=()): - connection_params = dict(self.connection_params(self.arguments), **dict(extra_params)) - - log.info("Connecting with args: {}".format(connection_params)) - conn = sql.connect(**connection_params) - - try: - yield conn - finally: - conn.close() - - @contextmanager - def cursor(self, extra_params=()): - with self.connection(extra_params) as conn: - cursor = conn.cursor(arraysize=self.arraysize) - try: - yield cursor - finally: - cursor.close() - - def assertEqualRowValues(self, actual, expected): - self.assertEqual(len(actual) if actual else 0, len(expected) if expected else 0) - for act, exp in zip(actual, expected): - self.assertSequenceEqual(act, exp) - - -class PySQLLargeQueriesSuite(PySQLTestCase, LargeQueriesMixin): - def get_some_rows(self, cursor, fetchmany_size): - row = cursor.fetchone() - if row: - return [row] - else: - return None - - -# Exclude Retry tests because they require specific setups, and LargeQueries too slow for core -# tests -class PySQLCoreTestSuite(SmokeTestMixin, CoreTestMixin, DecimalTestsMixin, TimestampTestsMixin, - PySQLTestCase): - validate_row_value_type = True - validate_result = True - - # An output column in description evaluates to equal to multiple types - # - type code returned by the client as string. - # - also potentially a PEP-249 object like NUMBER, DATETIME etc. - def expected_column_types(self, type_): - type_mappings = { - 'boolean': ['boolean', NUMBER], - 'byte': ['tinyint', NUMBER], - 'short': ['smallint', NUMBER], - 'integer': ['int', NUMBER], - 'long': ['bigint', NUMBER], - 'decimal': ['decimal', NUMBER], - 'timestamp': ['timestamp', DATETIME], - 'date': ['date', DATE], - 'binary': ['binary', BINARY], - 'string': ['string', STRING], - 'array': ['array'], - 'struct': ['struct'], - 'map': ['map'], - 'double': ['double', NUMBER], - 'null': ['null'] - } - return type_mappings[type_] - - def test_queries(self): - if not self._should_have_native_complex_types(): - array_type = str - array_val = "[1,2,3]" - struct_type = str - struct_val = "{\"a\":1,\"b\":2}" - map_type = str - map_val = "{1:2,3:4}" - else: - array_type = np.ndarray - array_val = np.array([1, 2, 3]) - struct_type = dict - struct_val = {"a": 1, "b": 2} - map_type = list - map_val = [(1, 2), (3, 4)] - - null_type = "null" if float(sql.__version__[0:2]) < 2.0 else "string" - self.range_queries = CoreTestMixin.range_queries + [ - ("NULL", null_type, type(None), None), - ("array(1, 2, 3)", 'array', array_type, array_val), - ("struct(1 as a, 2 as b)", 'struct', struct_type, struct_val), - ("map(1, 2, 3, 4)", 'map', map_type, map_val), - ] - - self.run_tests_on_queries({}) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_incorrect_query_throws_exception(self): - with self.cursor({}) as cursor: - # Syntax errors should contain the invalid SQL - with self.assertRaises(DatabaseError) as cm: - cursor.execute("^ FOO BAR") - self.assertIn("FOO BAR", str(cm.exception)) - - # Database error should contain the missing database - with self.assertRaises(DatabaseError) as cm: - cursor.execute("USE foo234823498ydfsiusdhf") - self.assertIn("foo234823498ydfsiusdhf", str(cm.exception)) - - # SQL with Extraneous input should send back the extraneous input - with self.assertRaises(DatabaseError) as cm: - cursor.execute("CREATE TABLE IF NOT EXISTS TABLE table_234234234") - self.assertIn("table_234234234", str(cm.exception)) - - def test_create_table_will_return_empty_result_set(self): - with self.cursor({}) as cursor: - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - try: - cursor.execute( - "CREATE TABLE IF NOT EXISTS {} AS (SELECT 1 AS col_1, '2' AS col_2)".format( - table_name)) - self.assertEqual(cursor.fetchall(), []) - finally: - cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) - - def test_get_tables(self): - with self.cursor({}) as cursor: - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - table_names = [table_name + '_1', table_name + '_2'] - - try: - for table in table_names: - cursor.execute( - "CREATE TABLE IF NOT EXISTS {} AS (SELECT 1 AS col_1, '2' AS col_2)".format( - table)) - cursor.tables(schema_name="defa%") - tables = cursor.fetchall() - tables_desc = cursor.description - - for table in table_names: - # Test only schema name and table name. - # From other columns, what is supported depends on DBR version. - self.assertIn(['default', table], [list(table[1:3]) for table in tables]) - self.assertEqual( - tables_desc, - [('TABLE_CAT', 'string', None, None, None, None, None), - ('TABLE_SCHEM', 'string', None, None, None, None, None), - ('TABLE_NAME', 'string', None, None, None, None, None), - ('TABLE_TYPE', 'string', None, None, None, None, None), - ('REMARKS', 'string', None, None, None, None, None), - ('TYPE_CAT', 'string', None, None, None, None, None), - ('TYPE_SCHEM', 'string', None, None, None, None, None), - ('TYPE_NAME', 'string', None, None, None, None, None), - ('SELF_REFERENCING_COL_NAME', 'string', None, None, None, None, None), - ('REF_GENERATION', 'string', None, None, None, None, None)]) - finally: - for table in table_names: - cursor.execute('DROP TABLE IF EXISTS {}'.format(table)) - - def test_get_columns(self): - with self.cursor({}) as cursor: - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - table_names = [table_name + '_1', table_name + '_2'] - - try: - for table in table_names: - cursor.execute("CREATE TABLE IF NOT EXISTS {} AS (SELECT " - "1 AS col_1, " - "'2' AS col_2, " - "named_struct('name', 'alice', 'age', 28) as col_3, " - "map('items', 45, 'cost', 228) as col_4, " - "array('item1', 'item2', 'item3') as col_5)".format(table)) - - cursor.columns(schema_name="defa%", table_name=table_name + '%') - cols = cursor.fetchall() - cols_desc = cursor.description - - # Catalogue name not consistent across DBR versions, so we skip that - cleaned_response = [list(col[1:6]) for col in cols] - # We also replace ` as DBR changes how it represents struct names - for col in cleaned_response: - col[4] = col[4].replace("`", "") - - self.assertEqual(cleaned_response, [ - ['default', table_name + '_1', 'col_1', 4, 'INT'], - ['default', table_name + '_1', 'col_2', 12, 'STRING'], - ['default', table_name + '_1', 'col_3', 2002, 'STRUCT'], - ['default', table_name + '_1', 'col_4', 2000, 'MAP'], - ['default', table_name + '_1', 'col_5', 2003, 'ARRAY'], - ['default', table_name + '_2', 'col_1', 4, 'INT'], - ['default', table_name + '_2', 'col_2', 12, 'STRING'], - ['default', table_name + '_2', 'col_3', 2002, 'STRUCT'], - ['default', table_name + '_2', 'col_4', 2000, 'MAP'], - [ - 'default', - table_name + '_2', - 'col_5', - 2003, - 'ARRAY', - ] - ]) - - self.assertEqual(cols_desc, - [('TABLE_CAT', 'string', None, None, None, None, None), - ('TABLE_SCHEM', 'string', None, None, None, None, None), - ('TABLE_NAME', 'string', None, None, None, None, None), - ('COLUMN_NAME', 'string', None, None, None, None, None), - ('DATA_TYPE', 'int', None, None, None, None, None), - ('TYPE_NAME', 'string', None, None, None, None, None), - ('COLUMN_SIZE', 'int', None, None, None, None, None), - ('BUFFER_LENGTH', 'tinyint', None, None, None, None, None), - ('DECIMAL_DIGITS', 'int', None, None, None, None, None), - ('NUM_PREC_RADIX', 'int', None, None, None, None, None), - ('NULLABLE', 'int', None, None, None, None, None), - ('REMARKS', 'string', None, None, None, None, None), - ('COLUMN_DEF', 'string', None, None, None, None, None), - ('SQL_DATA_TYPE', 'int', None, None, None, None, None), - ('SQL_DATETIME_SUB', 'int', None, None, None, None, None), - ('CHAR_OCTET_LENGTH', 'int', None, None, None, None, None), - ('ORDINAL_POSITION', 'int', None, None, None, None, None), - ('IS_NULLABLE', 'string', None, None, None, None, None), - ('SCOPE_CATALOG', 'string', None, None, None, None, None), - ('SCOPE_SCHEMA', 'string', None, None, None, None, None), - ('SCOPE_TABLE', 'string', None, None, None, None, None), - ('SOURCE_DATA_TYPE', 'smallint', None, None, None, None, None), - ('IS_AUTO_INCREMENT', 'string', None, None, None, None, None)]) - finally: - for table in table_names: - cursor.execute('DROP TABLE IF EXISTS {}'.format(table)) - - def test_escape_single_quotes(self): - with self.cursor({}) as cursor: - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - # Test escape syntax directly - cursor.execute("CREATE TABLE IF NOT EXISTS {} AS (SELECT 'you\\'re' AS col_1)".format(table_name)) - cursor.execute("SELECT * FROM {} WHERE col_1 LIKE 'you\\'re'".format(table_name)) - rows = cursor.fetchall() - assert rows[0]["col_1"] == "you're" - - # Test escape syntax in parameter - cursor.execute("SELECT * FROM {} WHERE {}.col_1 LIKE %(var)s".format(table_name, table_name), parameters={"var": "you're"}) - rows = cursor.fetchall() - assert rows[0]["col_1"] == "you're" - - def test_get_schemas(self): - with self.cursor({}) as cursor: - database_name = 'db_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - try: - cursor.execute('CREATE DATABASE IF NOT EXISTS {}'.format(database_name)) - cursor.schemas() - schemas = cursor.fetchall() - schemas_desc = cursor.description - # Catalogue name not consistent across DBR versions, so we skip that - self.assertIn(database_name, [schema[0] for schema in schemas]) - self.assertEqual(schemas_desc, - [('TABLE_SCHEM', 'string', None, None, None, None, None), - ('TABLE_CATALOG', 'string', None, None, None, None, None)]) - finally: - cursor.execute('DROP DATABASE IF EXISTS {}'.format(database_name)) - - def test_get_catalogs(self): - with self.cursor({}) as cursor: - cursor.catalogs() - cursor.fetchall() - catalogs_desc = cursor.description - self.assertEqual(catalogs_desc, [('TABLE_CAT', 'string', None, None, None, None, None)]) - - @skipUnless(pysql_supports_arrow(), 'arrow test need arrow support') - def test_get_arrow(self): - # These tests are quite light weight as the arrow fetch methods are used internally - # by everything else - with self.cursor({}) as cursor: - cursor.execute("SELECT * FROM range(10)") - table_1 = cursor.fetchmany_arrow(1).to_pydict() - self.assertEqual(table_1, OrderedDict([("id", [0])])) - - table_2 = cursor.fetchall_arrow().to_pydict() - self.assertEqual(table_2, OrderedDict([("id", [1, 2, 3, 4, 5, 6, 7, 8, 9])])) - - def test_unicode(self): - unicode_str = "数据砖" - with self.cursor({}) as cursor: - cursor.execute("SELECT '{}'".format(unicode_str)) - results = cursor.fetchall() - self.assertTrue(len(results) == 1 and len(results[0]) == 1) - self.assertEqual(results[0][0], unicode_str) - - def test_cancel_during_execute(self): - with self.cursor({}) as cursor: - - def execute_really_long_query(): - cursor.execute("SELECT SUM(A.id - B.id) " + - "FROM range(1000000000) A CROSS JOIN range(100000000) B " + - "GROUP BY (A.id - B.id)") - - exec_thread = threading.Thread(target=execute_really_long_query) - - exec_thread.start() - # Make sure the query has started before cancelling - time.sleep(15) - cursor.cancel() - exec_thread.join(5) - self.assertFalse(exec_thread.is_alive()) - - # Fetching results should throw an exception - with self.assertRaises((Error, thrift.Thrift.TException)): - cursor.fetchall() - with self.assertRaises((Error, thrift.Thrift.TException)): - cursor.fetchone() - with self.assertRaises((Error, thrift.Thrift.TException)): - cursor.fetchmany(10) - - # We should be able to execute a new command on the cursor - cursor.execute("SELECT * FROM range(3)") - self.assertEqual(len(cursor.fetchall()), 3) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_can_execute_command_after_failure(self): - with self.cursor({}) as cursor: - with self.assertRaises(DatabaseError): - cursor.execute("this is a sytnax error") - - cursor.execute("SELECT 1;") - - res = cursor.fetchall() - self.assertEqualRowValues(res, [[1]]) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_can_execute_command_after_success(self): - with self.cursor({}) as cursor: - cursor.execute("SELECT 1;") - cursor.execute("SELECT 2;") - - res = cursor.fetchall() - self.assertEqualRowValues(res, [[2]]) - - def generate_multi_row_query(self): - query = "SELECT * FROM range(3);" - return query - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_fetchone(self): - with self.cursor({}) as cursor: - query = self.generate_multi_row_query() - cursor.execute(query) - - self.assertSequenceEqual(cursor.fetchone(), [0]) - self.assertSequenceEqual(cursor.fetchone(), [1]) - self.assertSequenceEqual(cursor.fetchone(), [2]) - - self.assertEqual(cursor.fetchone(), None) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_fetchall(self): - with self.cursor({}) as cursor: - query = self.generate_multi_row_query() - cursor.execute(query) - - self.assertEqualRowValues(cursor.fetchall(), [[0], [1], [2]]) - - self.assertEqual(cursor.fetchone(), None) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_fetchmany_when_stride_fits(self): - with self.cursor({}) as cursor: - query = "SELECT * FROM range(4)" - cursor.execute(query) - - self.assertEqualRowValues(cursor.fetchmany(2), [[0], [1]]) - self.assertEqualRowValues(cursor.fetchmany(2), [[2], [3]]) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_fetchmany_in_excess(self): - with self.cursor({}) as cursor: - query = "SELECT * FROM range(4)" - cursor.execute(query) - - self.assertEqualRowValues(cursor.fetchmany(3), [[0], [1], [2]]) - self.assertEqualRowValues(cursor.fetchmany(3), [[3]]) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_iterator_api(self): - with self.cursor({}) as cursor: - query = "SELECT * FROM range(4)" - cursor.execute(query) - - expected_results = [[0], [1], [2], [3]] - for (i, row) in enumerate(cursor): - self.assertSequenceEqual(row, expected_results[i]) - - def test_temp_view_fetch(self): - with self.cursor({}) as cursor: - query = "create temporary view f as select * from range(10)" - cursor.execute(query) - # TODO assert on a result - # once what is being returned has stabilised - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_socket_timeout(self): - # We we expect to see a BlockingIO error when the socket is opened - # in non-blocking mode, since no poll is done before the read - with self.assertRaises(OperationalError) as cm: - with self.cursor({"_socket_timeout": 0}): - pass - - self.assertIsInstance(cm.exception.args[1], io.BlockingIOError) - - def test_ssp_passthrough(self): - for enable_ansi in (True, False): - with self.cursor({"session_configuration": {"ansi_mode": enable_ansi}}) as cursor: - cursor.execute("SET ansi_mode") - self.assertEqual(list(cursor.fetchone()), ["ansi_mode", str(enable_ansi)]) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_timestamps_arrow(self): - with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: - for (timestamp, expected) in self.timestamp_and_expected_results: - cursor.execute("SELECT TIMESTAMP('{timestamp}')".format(timestamp=timestamp)) - arrow_table = cursor.fetchmany_arrow(1) - if self.should_add_timezone(): - ts_type = pyarrow.timestamp("us", tz="Etc/UTC") - else: - ts_type = pyarrow.timestamp("us") - self.assertEqual(arrow_table.field(0).type, ts_type) - result_value = arrow_table.column(0).combine_chunks()[0].value - # To work consistently across different local timezones, we specify the timezone - # of the expected result to - # be UTC (what it should be by default on the server) - aware_timestamp = expected and expected.replace(tzinfo=datetime.timezone.utc) - self.assertEqual(result_value, aware_timestamp and - aware_timestamp.timestamp() * 1000000, - "timestamp {} did not match {}".format(timestamp, expected)) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_multi_timestamps_arrow(self): - with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: - query, expected = self.multi_query() - expected = [[self.maybe_add_timezone_to_timestamp(ts) for ts in row] - for row in expected] - cursor.execute(query) - table = cursor.fetchall_arrow() - # Transpose columnar result to list of rows - list_of_cols = [c.to_pylist() for c in table] - result = [[col[row_index] for col in list_of_cols] - for row_index in range(table.num_rows)] - self.assertEqual(result, expected) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_timezone_with_timestamp(self): - if self.should_add_timezone(): - with self.cursor() as cursor: - cursor.execute("SET TIME ZONE 'Europe/Amsterdam'") - cursor.execute("select CAST('2022-03-02 12:54:56' as TIMESTAMP)") - amsterdam = pytz.timezone("Europe/Amsterdam") - expected = amsterdam.localize(datetime.datetime(2022, 3, 2, 12, 54, 56)) - result = cursor.fetchone()[0] - self.assertEqual(result, expected) - - cursor.execute("select CAST('2022-03-02 12:54:56' as TIMESTAMP)") - arrow_result_table = cursor.fetchmany_arrow(1) - arrow_result_value = arrow_result_table.column(0).combine_chunks()[0].value - ts_type = pyarrow.timestamp("us", tz="Europe/Amsterdam") - - self.assertEqual(arrow_result_table.field(0).type, ts_type) - self.assertEqual(arrow_result_value, expected.timestamp() * 1000000) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_can_flip_compression(self): - with self.cursor() as cursor: - cursor.execute("SELECT array(1,2,3,4)") - cursor.fetchall() - lz4_compressed = cursor.active_result_set.lz4_compressed - #The endpoint should support compression - self.assertEqual(lz4_compressed, True) - cursor.connection.lz4_compression=False - cursor.execute("SELECT array(1,2,3,4)") - cursor.fetchall() - lz4_compressed = cursor.active_result_set.lz4_compressed - self.assertEqual(lz4_compressed, False) - - def _should_have_native_complex_types(self): - return pysql_has_version(">=", 2) and is_thrift_v5_plus(self.arguments) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_arrays_are_not_returned_as_strings_arrow(self): - if self._should_have_native_complex_types(): - with self.cursor() as cursor: - cursor.execute("SELECT array(1,2,3,4)") - arrow_df = cursor.fetchall_arrow() - - list_type = arrow_df.field(0).type - self.assertTrue(pyarrow.types.is_list(list_type)) - self.assertTrue(pyarrow.types.is_integer(list_type.value_type)) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_structs_are_not_returned_as_strings_arrow(self): - if self._should_have_native_complex_types(): - with self.cursor() as cursor: - cursor.execute("SELECT named_struct('foo', 42, 'bar', 'baz')") - arrow_df = cursor.fetchall_arrow() - - struct_type = arrow_df.field(0).type - self.assertTrue(pyarrow.types.is_struct(struct_type)) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_decimal_not_returned_as_strings_arrow(self): - if self._should_have_native_complex_types(): - with self.cursor() as cursor: - cursor.execute("SELECT 5E3BD") - arrow_df = cursor.fetchall_arrow() - - decimal_type = arrow_df.field(0).type - self.assertTrue(pyarrow.types.is_decimal(decimal_type)) - - def test_close_connection_closes_cursors(self): - - from databricks.sql.thrift_api.TCLIService import ttypes - - with self.connection() as conn: - cursor = conn.cursor() - cursor.execute('SELECT id, id `id2`, id `id3` FROM RANGE(1000000) order by RANDOM()') - ars = cursor.active_result_set - - # We must manually run this check because thrift_backend always forces `has_been_closed_server_side` to True - - # Cursor op state should be open before connection is closed - status_request = ttypes.TGetOperationStatusReq(operationHandle=ars.command_id, getProgressUpdate=False) - op_status_at_server = ars.thrift_backend._client.GetOperationStatus(status_request) - assert op_status_at_server.operationState != ttypes.TOperationState.CLOSED_STATE - - conn.close() - - # When connection closes, any cursor operations should no longer exist at the server - with self.assertRaises(thrift.Thrift.TApplicationException) as cm: - op_status_at_server = ars.thrift_backend._client.GetOperationStatus(status_request) - if hasattr(cm, "exception"): - assert "RESOURCE_DOES_NOT_EXIST" in cm.exception.message - - - - -# use a RetrySuite to encapsulate these tests which we'll typically want to run together; however keep -# the 429/503 subsuites separate since they execute under different circumstances. -class PySQLRetryTestSuite: - class HTTP429Suite(Client429ResponseMixin, PySQLTestCase): - pass # Mixin covers all - - class HTTP503Suite(Client503ResponseMixin, PySQLTestCase): - # 503Response suite gets custom error here vs PyODBC - def test_retry_disabled(self): - self._test_retry_disabled_with_message("TEMPORARILY_UNAVAILABLE", OperationalError) - - -class PySQLUnityCatalogTestSuite(PySQLTestCase): - """Simple namespace tests that should be run against a unity-catalog-enabled cluster""" - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_initial_namespace(self): - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - with self.cursor() as cursor: - cursor.execute("USE CATALOG {}".format(self.arguments["catA"])) - cursor.execute("CREATE TABLE table_{} (col1 int)".format(table_name)) - with self.connection({ - "catalog": self.arguments["catA"], - "schema": table_name - }) as connection: - cursor = connection.cursor() - cursor.execute("select current_catalog()") - self.assertEqual(cursor.fetchone()[0], self.arguments["catA"]) - cursor.execute("select current_database()") - self.assertEqual(cursor.fetchone()[0], table_name) - -class PySQLStagingIngestionTestSuite(PySQLTestCase): - """Simple namespace for ingestion tests. These should be run against DBR >12.x - - In addition to connection credentials (host, path, token) this suite requires an env var - named staging_ingestion_user""" - - staging_ingestion_user = os.getenv("staging_ingestion_user") - - if staging_ingestion_user is None: - raise ValueError( - "To run these tests you must designate a `staging_ingestion_user` environment variable. This will the user associated with the personal access token." - ) - - def test_staging_ingestion_life_cycle(self): - """PUT a file into the staging location - GET the file from the staging location - REMOVE the file from the staging location - Try to GET the file again expecting to raise an exception - """ - - # PUT should succeed - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: - - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - # GET should succeed - - new_fh, new_temp_path = tempfile.mkstemp() - - with self.connection(extra_params={"staging_allowed_local_path": new_temp_path}) as conn: - cursor = conn.cursor() - query = f"GET 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' TO '{new_temp_path}'" - cursor.execute(query) - - with open(new_fh, "rb") as fp: - fetched_text = fp.read() - - assert fetched_text == original_text - - # REMOVE should succeed - - remove_query = ( - f"REMOVE 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv'" - ) - - with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: - cursor = conn.cursor() - cursor.execute(remove_query) - - # GET after REMOVE should fail - - with pytest.raises(Error): - cursor = conn.cursor() - query = f"GET 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' TO '{new_temp_path}'" - cursor.execute(query) - - os.remove(temp_path) - os.remove(new_temp_path) - - - def test_staging_ingestion_put_fails_without_staging_allowed_local_path(self): - """PUT operations are not supported unless the connection was built with - a parameter called staging_allowed_local_path - """ - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - with pytest.raises(Error): - with self.connection() as conn: - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_put_fails_if_localFile_not_in_staging_allowed_local_path(self): - - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - base_path, filename = os.path.split(temp_path) - - # Add junk to base_path - base_path = os.path.join(base_path, "temp") - - with pytest.raises(Error): - with self.connection(extra_params={"staging_allowed_local_path": base_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_put_fails_if_file_exists_and_overwrite_not_set(self): - """PUT a file into the staging location twice. First command should succeed. Second should fail. - """ - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - def perform_put(): - with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/12/15/file1.csv'" - cursor.execute(query) - - def perform_remove(): - remove_query = ( - f"REMOVE 'stage://tmp/{self.staging_ingestion_user}/tmp/12/15/file1.csv'" - ) - - with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: - cursor = conn.cursor() - cursor.execute(remove_query) - - - # Make sure file does not exist - perform_remove() - - # Put the file - perform_put() - - # Try to put it again - with pytest.raises(sql.exc.ServerOperationError, match="FILE_IN_STAGING_PATH_ALREADY_EXISTS"): - perform_put() - - # Clean up after ourselves - perform_remove() - - def test_staging_ingestion_fails_to_modify_another_staging_user(self): - """The server should only allow modification of the staging_ingestion_user's files - """ - - some_other_user = "mary.poppins@databricks.com" - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - def perform_put(): - with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{some_other_user}/tmp/12/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def perform_remove(): - remove_query = ( - f"REMOVE 'stage://tmp/{some_other_user}/tmp/12/15/file1.csv'" - ) - - with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: - cursor = conn.cursor() - cursor.execute(remove_query) - - def perform_get(): - with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: - cursor = conn.cursor() - query = f"GET 'stage://tmp/{some_other_user}/tmp/11/15/file1.csv' TO '{temp_path}'" - cursor.execute(query) - - # PUT should fail with permissions error - with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): - perform_put() - - # REMOVE should fail with permissions error - with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): - perform_remove() - - # GET should fail with permissions error - with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): - perform_get() - - def test_staging_ingestion_put_fails_if_absolute_localFile_not_in_staging_allowed_local_path(self): - """ - This test confirms that staging_allowed_local_path and target_file are resolved into absolute paths. - """ - - # If these two paths are not resolved absolutely, they appear to share a common path of /var/www/html - # after resolution their common path is only /var/www which should raise an exception - # Because the common path must always be equal to staging_allowed_local_path - staging_allowed_local_path = "/var/www/html" - target_file = "/var/www/html/../html1/not_allowed.html" - - with pytest.raises(Error): - with self.connection(extra_params={"staging_allowed_local_path": staging_allowed_local_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{target_file}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_empty_local_path_fails_to_parse_at_server(self): - staging_allowed_local_path = "/var/www/html" - target_file = "" - - with pytest.raises(Error, match="EMPTY_LOCAL_FILE_IN_STAGING_ACCESS_QUERY"): - with self.connection(extra_params={"staging_allowed_local_path": staging_allowed_local_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{target_file}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_invalid_staging_path_fails_at_server(self): - staging_allowed_local_path = "/var/www/html" - target_file = "index.html" - - with pytest.raises(Error, match="INVALID_STAGING_PATH_IN_STAGING_ACCESS_QUERY"): - with self.connection(extra_params={"staging_allowed_local_path": staging_allowed_local_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{target_file}' INTO 'stageRANDOMSTRINGOFCHARACTERS://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_supports_multiple_staging_allowed_local_path_values(self): - """staging_allowed_local_path may be either a path-like object or a list of path-like objects. - - This test confirms that two configured base paths: - 1 - doesn't raise an exception - 2 - allows uploads from both paths - 3 - doesn't allow uploads from a third path - """ - - def generate_file_and_path_and_queries(): - """ - 1. Makes a temp file with some contents. - 2. Write a query to PUT it into a staging location - 3. Write a query to REMOVE it from that location (for cleanup) - """ - fh, temp_path = tempfile.mkstemp() - with open(fh, "wb") as fp: - original_text = "hello world!".encode("utf-8") - fp.write(original_text) - put_query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/{id(temp_path)}.csv' OVERWRITE" - remove_query = f"REMOVE 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/{id(temp_path)}.csv'" - return fh, temp_path, put_query, remove_query - - fh1, temp_path1, put_query1, remove_query1 = generate_file_and_path_and_queries() - fh2, temp_path2, put_query2, remove_query2 = generate_file_and_path_and_queries() - fh3, temp_path3, put_query3, remove_query3 = generate_file_and_path_and_queries() - - with self.connection(extra_params={"staging_allowed_local_path": [temp_path1, temp_path2]}) as conn: - cursor = conn.cursor() - - cursor.execute(put_query1) - cursor.execute(put_query2) - - with pytest.raises(Error, match="Local file operations are restricted to paths within the configured staging_allowed_local_path"): - cursor.execute(put_query3) - - # Then clean up the files we made - cursor.execute(remove_query1) - cursor.execute(remove_query2) - - -def main(cli_args): - global get_args_from_env - get_args_from_env = True - print(f"Running tests with version: {sql.__version__}") - logging.getLogger("databricks.sql").setLevel(logging.INFO) - unittest.main(module=__file__, argv=sys.argv[0:1] + cli_args) - - -if __name__ == "__main__": - main(sys.argv[1:]) \ No newline at end of file diff --git a/tests/e2e/sqlalchemy/test_basic.py b/tests/e2e/sqlalchemy/test_basic.py deleted file mode 100644 index 4f4df91b..00000000 --- a/tests/e2e/sqlalchemy/test_basic.py +++ /dev/null @@ -1,254 +0,0 @@ -import os, datetime, decimal -import pytest -from unittest import skipIf -from sqlalchemy import create_engine, select, insert, Column, MetaData, Table -from sqlalchemy.orm import declarative_base, Session -from sqlalchemy.types import SMALLINT, Integer, BOOLEAN, String, DECIMAL, Date - - -USER_AGENT_TOKEN = "PySQL e2e Tests" - - -@pytest.fixture -def db_engine(): - - HOST = os.environ.get("host") - HTTP_PATH = os.environ.get("http_path") - ACCESS_TOKEN = os.environ.get("access_token") - CATALOG = os.environ.get("catalog") - SCHEMA = os.environ.get("schema") - - connect_args = {"_user_agent_entry": USER_AGENT_TOKEN} - - engine = create_engine( - f"databricks://token:{ACCESS_TOKEN}@{HOST}?http_path={HTTP_PATH}&catalog={CATALOG}&schema={SCHEMA}", - connect_args=connect_args, - ) - return engine - - -@pytest.fixture() -def base(db_engine): - return declarative_base(bind=db_engine) - - -@pytest.fixture() -def session(db_engine): - return Session(bind=db_engine) - - -@pytest.fixture() -def metadata_obj(db_engine): - return MetaData(bind=db_engine) - - -def test_can_connect(db_engine): - simple_query = "SELECT 1" - result = db_engine.execute(simple_query).fetchall() - assert len(result) == 1 - - -def test_connect_args(db_engine): - """Verify that extra connect args passed to sqlalchemy.create_engine are passed to DBAPI - - This will most commonly happen when partners supply a user agent entry - """ - - conn = db_engine.connect() - connection_headers = conn.connection.thrift_backend._transport._headers - user_agent = connection_headers["User-Agent"] - - expected = f"(sqlalchemy + {USER_AGENT_TOKEN})" - assert expected in user_agent - - -def test_pandas_upload(db_engine, metadata_obj): - - import pandas as pd - - SCHEMA = os.environ.get("schema") - try: - df = pd.read_excel("tests/sqlalchemy/demo_data/MOCK_DATA.xlsx") - df.to_sql( - "mock_data", - db_engine, - schema=SCHEMA, - index=False, - method="multi", - if_exists="replace", - ) - - df_after = pd.read_sql_table("mock_data", db_engine, schema=SCHEMA) - assert len(df) == len(df_after) - except Exception as e: - raise e - finally: - db_engine.execute("DROP TABLE mock_data") - - -def test_create_table_not_null(db_engine, metadata_obj): - - table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) - - SampleTable = Table( - table_name, - metadata_obj, - Column("name", String(255)), - Column("episodes", Integer), - Column("some_bool", BOOLEAN, nullable=False), - ) - - metadata_obj.create_all() - - columns = db_engine.dialect.get_columns( - connection=db_engine.connect(), table_name=table_name - ) - - name_column_description = columns[0] - some_bool_column_description = columns[2] - - assert name_column_description.get("nullable") is True - assert some_bool_column_description.get("nullable") is False - - metadata_obj.drop_all() - - -def test_bulk_insert_with_core(db_engine, metadata_obj, session): - - import random - - num_to_insert = random.choice(range(10_000, 20_000)) - - table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) - - names = ["Bim", "Miki", "Sarah", "Ira"] - - SampleTable = Table( - table_name, metadata_obj, Column("name", String(255)), Column("number", Integer) - ) - - rows = [ - {"name": names[i % 3], "number": random.choice(range(10000))} - for i in range(num_to_insert) - ] - - metadata_obj.create_all() - db_engine.execute(insert(SampleTable).values(rows)) - - rows = db_engine.execute(select(SampleTable)).fetchall() - - assert len(rows) == num_to_insert - - -def test_create_insert_drop_table_core(base, db_engine, metadata_obj: MetaData): - """ """ - - SampleTable = Table( - "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")), - metadata_obj, - Column("name", String(255)), - Column("episodes", Integer), - Column("some_bool", BOOLEAN), - Column("dollars", DECIMAL(10, 2)), - ) - - metadata_obj.create_all() - - insert_stmt = insert(SampleTable).values( - name="Bim Adewunmi", episodes=6, some_bool=True, dollars=decimal.Decimal(125) - ) - - with db_engine.connect() as conn: - conn.execute(insert_stmt) - - select_stmt = select(SampleTable) - resp = db_engine.execute(select_stmt) - - result = resp.fetchall() - - assert len(result) == 1 - - metadata_obj.drop_all() - - -# ORM tests are made following this tutorial -# https://docs.sqlalchemy.org/en/14/orm/quickstart.html - - -@skipIf(False, "Unity catalog must be supported") -def test_create_insert_drop_table_orm(base, session: Session): - """ORM classes built on the declarative base class must have a primary key. - This is restricted to Unity Catalog. - """ - - class SampleObject(base): - - __tablename__ = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) - - name = Column(String(255), primary_key=True) - episodes = Column(Integer) - some_bool = Column(BOOLEAN) - - base.metadata.create_all() - - sample_object_1 = SampleObject(name="Bim Adewunmi", episodes=6, some_bool=True) - sample_object_2 = SampleObject(name="Miki Meek", episodes=12, some_bool=False) - session.add(sample_object_1) - session.add(sample_object_2) - session.commit() - - stmt = select(SampleObject).where( - SampleObject.name.in_(["Bim Adewunmi", "Miki Meek"]) - ) - - output = [i for i in session.scalars(stmt)] - assert len(output) == 2 - - base.metadata.drop_all() - - -def test_dialect_type_mappings(base, db_engine, metadata_obj: MetaData): - """Confirms that we get back the same time we declared in a model and inserted using Core""" - - SampleTable = Table( - "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")), - metadata_obj, - Column("string_example", String(255)), - Column("integer_example", Integer), - Column("boolean_example", BOOLEAN), - Column("decimal_example", DECIMAL(10, 2)), - Column("date_example", Date), - ) - - string_example = "" - integer_example = 100 - boolean_example = True - decimal_example = decimal.Decimal(125) - date_example = datetime.date(2013, 1, 1) - - metadata_obj.create_all() - - insert_stmt = insert(SampleTable).values( - string_example=string_example, - integer_example=integer_example, - boolean_example=boolean_example, - decimal_example=decimal_example, - date_example=date_example, - ) - - with db_engine.connect() as conn: - conn.execute(insert_stmt) - - select_stmt = select(SampleTable) - resp = db_engine.execute(select_stmt) - - result = resp.fetchall() - this_row = result[0] - - assert this_row["string_example"] == string_example - assert this_row["integer_example"] == integer_example - assert this_row["boolean_example"] == boolean_example - assert this_row["decimal_example"] == decimal_example - assert this_row["date_example"] == date_example - - metadata_obj.drop_all() diff --git a/tests/e2e/test_complex_types.py b/tests/e2e/test_complex_types.py new file mode 100644 index 00000000..446a6b50 --- /dev/null +++ b/tests/e2e/test_complex_types.py @@ -0,0 +1,63 @@ +import pytest +from numpy import ndarray + +from tests.e2e.test_driver import PySQLPytestTestCase + + +class TestComplexTypes(PySQLPytestTestCase): + @pytest.fixture(scope="class") + def table_fixture(self, connection_details): + self.arguments = connection_details.copy() + """A pytest fixture that creates a table with a complex type, inserts a record, yields, and then drops the table""" + + with self.cursor() as cursor: + # Create the table + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS pysql_test_complex_types_table ( + array_col ARRAY, + map_col MAP, + struct_col STRUCT + ) + """ + ) + # Insert a record + cursor.execute( + """ + INSERT INTO pysql_test_complex_types_table + VALUES ( + ARRAY('a', 'b', 'c'), + MAP('a', 1, 'b', 2, 'c', 3), + NAMED_STRUCT('field1', 'a', 'field2', 1) + ) + """ + ) + yield + # Clean up the table after the test + cursor.execute("DROP TABLE IF EXISTS pysql_test_complex_types_table") + + @pytest.mark.parametrize( + "field,expected_type", + [("array_col", ndarray), ("map_col", list), ("struct_col", dict)], + ) + def test_read_complex_types_as_arrow(self, field, expected_type, table_fixture): + """Confirms the return types of a complex type field when reading as arrow""" + + with self.cursor() as cursor: + result = cursor.execute( + "SELECT * FROM pysql_test_complex_types_table LIMIT 1" + ).fetchone() + + assert isinstance(result[field], expected_type) + + @pytest.mark.parametrize("field", [("array_col"), ("map_col"), ("struct_col")]) + def test_read_complex_types_as_string(self, field, table_fixture): + """Confirms the return type of a complex type that is returned as a string""" + with self.cursor( + extra_params={"_use_arrow_native_complex_types": False} + ) as cursor: + result = cursor.execute( + "SELECT * FROM pysql_test_complex_types_table LIMIT 1" + ).fetchone() + + assert isinstance(result[field], str) diff --git a/tests/e2e/test_driver.py b/tests/e2e/test_driver.py new file mode 100644 index 00000000..2f0881cd --- /dev/null +++ b/tests/e2e/test_driver.py @@ -0,0 +1,831 @@ +import itertools +from contextlib import contextmanager +from collections import OrderedDict +import datetime +import io +import logging +import os +import sys +import threading +import time +from unittest import loader, skipIf, skipUnless, TestCase +from uuid import uuid4 + +import numpy as np +import pyarrow +import pytz +import thrift +import pytest +from urllib3.connectionpool import ReadTimeoutError + +import databricks.sql as sql +from databricks.sql import ( + STRING, + BINARY, + NUMBER, + DATETIME, + DATE, + DatabaseError, + Error, + OperationalError, + RequestError, +) +from tests.e2e.common.predicates import ( + pysql_has_version, + pysql_supports_arrow, + compare_dbr_versions, + is_thrift_v5_plus, +) +from databricks.sql.thrift_api.TCLIService import ttypes +from tests.e2e.common.core_tests import CoreTestMixin, SmokeTestMixin +from tests.e2e.common.large_queries_mixin import LargeQueriesMixin +from tests.e2e.common.timestamp_tests import TimestampTestsMixin +from tests.e2e.common.decimal_tests import DecimalTestsMixin +from tests.e2e.common.retry_test_mixins import ( + Client429ResponseMixin, + Client503ResponseMixin, +) +from tests.e2e.common.staging_ingestion_tests import PySQLStagingIngestionTestSuiteMixin +from tests.e2e.common.retry_test_mixins import PySQLRetryTestsMixin + +from tests.e2e.common.uc_volume_tests import PySQLUCVolumeTestSuiteMixin + +from databricks.sql.exc import SessionAlreadyClosedError + +log = logging.getLogger(__name__) + +unsafe_logger = logging.getLogger("databricks.sql.unsafe") +unsafe_logger.setLevel(logging.DEBUG) +unsafe_logger.addHandler(logging.FileHandler("./tests-unsafe.log")) + +# manually decorate DecimalTestsMixin to need arrow support +for name in loader.getTestCaseNames(DecimalTestsMixin, "test_"): + fn = getattr(DecimalTestsMixin, name) + decorated = skipUnless(pysql_supports_arrow(), "Decimal tests need arrow support")( + fn + ) + setattr(DecimalTestsMixin, name, decorated) + + +class PySQLPytestTestCase: + """A mirror of PySQLTest case that doesn't inherit from unittest.TestCase + so that we can use pytest.mark.parameterize + """ + + error_type = Error + conf_to_disable_rate_limit_retries = {"_retry_stop_after_attempts_count": 1} + conf_to_disable_temporarily_unavailable_retries = { + "_retry_stop_after_attempts_count": 1 + } + arraysize = 1000 + buffer_size_bytes = 104857600 + POLLING_INTERVAL = 2 + + @pytest.fixture(autouse=True) + def get_details(self, connection_details): + self.arguments = connection_details.copy() + + def connection_params(self): + params = { + "server_hostname": self.arguments["host"], + "http_path": self.arguments["http_path"], + **self.auth_params(), + } + + return params + + def auth_params(self): + return { + "access_token": self.arguments.get("access_token"), + } + + @contextmanager + def connection(self, extra_params=()): + connection_params = dict(self.connection_params(), **dict(extra_params)) + + log.info("Connecting with args: {}".format(connection_params)) + conn = sql.connect(**connection_params) + + try: + yield conn + finally: + conn.close() + + @contextmanager + def cursor(self, extra_params=()): + with self.connection(extra_params) as conn: + cursor = conn.cursor( + arraysize=self.arraysize, buffer_size_bytes=self.buffer_size_bytes + ) + try: + yield cursor + finally: + cursor.close() + + def assertEqualRowValues(self, actual, expected): + len_actual = len(actual) if actual else 0 + len_expected = len(expected) if expected else 0 + assert len_actual == len_expected + for act, exp in zip(actual, expected): + assert len(act) == len(exp) + for i in range(len(act)): + assert act[i] == exp[i] + + +class TestPySQLLargeQueriesSuite(PySQLPytestTestCase, LargeQueriesMixin): + def get_some_rows(self, cursor, fetchmany_size): + row = cursor.fetchone() + if row: + return [row] + else: + return None + + @skipUnless(pysql_supports_arrow(), "needs arrow support") + @pytest.mark.skip("This test requires a previously uploaded data set") + def test_cloud_fetch(self): + # This test can take several minutes to run + limits = [100000, 300000] + threads = [10, 25] + self.arraysize = 100000 + # This test requires a large table with many rows to properly initiate cloud fetch. + # e2-dogfood host > hive_metastore catalog > main schema has such a table called store_sales. + # If this table is deleted or this test is run on a different host, a different table may need to be used. + base_query = "SELECT * FROM store_sales WHERE ss_sold_date_sk = 2452234 " + for num_limit, num_threads, lz4_compression in itertools.product( + limits, threads, [True, False] + ): + with self.subTest( + num_limit=num_limit, + num_threads=num_threads, + lz4_compression=lz4_compression, + ): + cf_result, noop_result = None, None + query = base_query + "LIMIT " + str(num_limit) + with self.cursor( + { + "use_cloud_fetch": True, + "max_download_threads": num_threads, + "catalog": "hive_metastore", + }, + ) as cursor: + cursor.execute(query) + cf_result = cursor.fetchall() + with self.cursor({"catalog": "hive_metastore"}) as cursor: + cursor.execute(query) + noop_result = cursor.fetchall() + assert len(cf_result) == len(noop_result) + for i in range(len(cf_result)): + assert cf_result[i] == noop_result[i] + + def test_execute_async(self): + def isExecuting(operation_state): + return not operation_state or operation_state in [ + ttypes.TOperationState.RUNNING_STATE, + ttypes.TOperationState.PENDING_STATE, + ] + + long_running_query = "SELECT COUNT(*) FROM RANGE(10000 * 16) x JOIN RANGE(10000) y ON FROM_UNIXTIME(x.id * y.id, 'yyyy-MM-dd') LIKE '%not%a%date%'" + with self.cursor() as cursor: + cursor.execute_async(long_running_query) + + ## Polling after every POLLING_INTERVAL seconds + while isExecuting(cursor.get_query_state()): + time.sleep(self.POLLING_INTERVAL) + log.info("Polling the status in test_execute_async") + + cursor.get_async_execution_result() + result = cursor.fetchall() + + assert result[0].asDict() == {"count(1)": 0} + + +# Exclude Retry tests because they require specific setups, and LargeQueries too slow for core +# tests +class TestPySQLCoreSuite( + SmokeTestMixin, + CoreTestMixin, + DecimalTestsMixin, + TimestampTestsMixin, + PySQLPytestTestCase, + PySQLStagingIngestionTestSuiteMixin, + PySQLRetryTestsMixin, + PySQLUCVolumeTestSuiteMixin, +): + validate_row_value_type = True + validate_result = True + + # An output column in description evaluates to equal to multiple types + # - type code returned by the client as string. + # - also potentially a PEP-249 object like NUMBER, DATETIME etc. + def expected_column_types(self, type_): + type_mappings = { + "boolean": ["boolean", NUMBER], + "byte": ["tinyint", NUMBER], + "short": ["smallint", NUMBER], + "integer": ["int", NUMBER], + "long": ["bigint", NUMBER], + "decimal": ["decimal", NUMBER], + "timestamp": ["timestamp", DATETIME], + "date": ["date", DATE], + "binary": ["binary", BINARY], + "string": ["string", STRING], + "array": ["array"], + "struct": ["struct"], + "map": ["map"], + "double": ["double", NUMBER], + "null": ["null"], + } + return type_mappings[type_] + + def test_queries(self): + if not self._should_have_native_complex_types(): + array_type = str + array_val = "[1,2,3]" + struct_type = str + struct_val = '{"a":1,"b":2}' + map_type = str + map_val = "{1:2,3:4}" + else: + array_type = np.ndarray + array_val = np.array([1, 2, 3]) + struct_type = dict + struct_val = {"a": 1, "b": 2} + map_type = list + map_val = [(1, 2), (3, 4)] + + null_type = "null" if float(sql.__version__[0:2]) < 2.0 else "string" + self.range_queries = CoreTestMixin.range_queries + [ + ("NULL", null_type, type(None), None), + ("array(1, 2, 3)", "array", array_type, array_val), + ("struct(1 as a, 2 as b)", "struct", struct_type, struct_val), + ("map(1, 2, 3, 4)", "map", map_type, map_val), + ] + + self.run_tests_on_queries({}) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_incorrect_query_throws_exception(self): + with self.cursor({}) as cursor: + # Syntax errors should contain the invalid SQL + with pytest.raises(DatabaseError) as cm: + cursor.execute("^ FOO BAR") + assert "FOO BAR" in str(cm.value) + + # Database error should contain the missing database + with pytest.raises(DatabaseError) as cm: + cursor.execute("USE foo234823498ydfsiusdhf") + assert "foo234823498ydfsiusdhf" in str(cm.value) + + # SQL with Extraneous input should send back the extraneous input + with pytest.raises(DatabaseError) as cm: + cursor.execute("CREATE TABLE IF NOT EXISTS TABLE table_234234234") + assert "table_234234234" in str(cm.value) + + def test_create_table_will_return_empty_result_set(self): + with self.cursor({}) as cursor: + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + try: + cursor.execute( + "CREATE TABLE IF NOT EXISTS {} AS (SELECT 1 AS col_1, '2' AS col_2)".format( + table_name + ) + ) + assert cursor.fetchall() == [] + finally: + cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) + + def test_get_tables(self): + with self.cursor({}) as cursor: + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + table_names = [table_name + "_1", table_name + "_2"] + + try: + for table in table_names: + cursor.execute( + "CREATE TABLE IF NOT EXISTS {} AS (SELECT 1 AS col_1, '2' AS col_2)".format( + table + ) + ) + cursor.tables(schema_name="defa%") + tables = cursor.fetchall() + tables_desc = cursor.description + + for table in table_names: + # Test only schema name and table name. + # From other columns, what is supported depends on DBR version. + assert ["default", table] in [list(table[1:3]) for table in tables] + expected = [ + ("TABLE_CAT", "string", None, None, None, None, None), + ("TABLE_SCHEM", "string", None, None, None, None, None), + ("TABLE_NAME", "string", None, None, None, None, None), + ("TABLE_TYPE", "string", None, None, None, None, None), + ("REMARKS", "string", None, None, None, None, None), + ("TYPE_CAT", "string", None, None, None, None, None), + ("TYPE_SCHEM", "string", None, None, None, None, None), + ("TYPE_NAME", "string", None, None, None, None, None), + ( + "SELF_REFERENCING_COL_NAME", + "string", + None, + None, + None, + None, + None, + ), + ("REF_GENERATION", "string", None, None, None, None, None), + ] + assert tables_desc == expected + + finally: + for table in table_names: + cursor.execute("DROP TABLE IF EXISTS {}".format(table)) + + def test_get_columns(self): + with self.cursor({}) as cursor: + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + table_names = [table_name + "_1", table_name + "_2"] + + try: + for table in table_names: + cursor.execute( + "CREATE TABLE IF NOT EXISTS {} AS (SELECT " + "1 AS col_1, " + "'2' AS col_2, " + "named_struct('name', 'alice', 'age', 28) as col_3, " + "map('items', 45, 'cost', 228) as col_4, " + "array('item1', 'item2', 'item3') as col_5)".format(table) + ) + + cursor.columns(schema_name="defa%", table_name=table_name + "%") + cols = cursor.fetchall() + cols_desc = cursor.description + + # Catalogue name not consistent across DBR versions, so we skip that + cleaned_response = [list(col[1:6]) for col in cols] + # We also replace ` as DBR changes how it represents struct names + for col in cleaned_response: + col[4] = col[4].replace("`", "") + + expected = [ + ["default", table_name + "_1", "col_1", 4, "INT"], + ["default", table_name + "_1", "col_2", 12, "STRING"], + [ + "default", + table_name + "_1", + "col_3", + 2002, + "STRUCT", + ], + ["default", table_name + "_1", "col_4", 2000, "MAP"], + ["default", table_name + "_1", "col_5", 2003, "ARRAY"], + ["default", table_name + "_2", "col_1", 4, "INT"], + ["default", table_name + "_2", "col_2", 12, "STRING"], + [ + "default", + table_name + "_2", + "col_3", + 2002, + "STRUCT", + ], + ["default", table_name + "_2", "col_4", 2000, "MAP"], + [ + "default", + table_name + "_2", + "col_5", + 2003, + "ARRAY", + ], + ] + assert cleaned_response == expected + expected = [ + ("TABLE_CAT", "string", None, None, None, None, None), + ("TABLE_SCHEM", "string", None, None, None, None, None), + ("TABLE_NAME", "string", None, None, None, None, None), + ("COLUMN_NAME", "string", None, None, None, None, None), + ("DATA_TYPE", "int", None, None, None, None, None), + ("TYPE_NAME", "string", None, None, None, None, None), + ("COLUMN_SIZE", "int", None, None, None, None, None), + ("BUFFER_LENGTH", "tinyint", None, None, None, None, None), + ("DECIMAL_DIGITS", "int", None, None, None, None, None), + ("NUM_PREC_RADIX", "int", None, None, None, None, None), + ("NULLABLE", "int", None, None, None, None, None), + ("REMARKS", "string", None, None, None, None, None), + ("COLUMN_DEF", "string", None, None, None, None, None), + ("SQL_DATA_TYPE", "int", None, None, None, None, None), + ("SQL_DATETIME_SUB", "int", None, None, None, None, None), + ("CHAR_OCTET_LENGTH", "int", None, None, None, None, None), + ("ORDINAL_POSITION", "int", None, None, None, None, None), + ("IS_NULLABLE", "string", None, None, None, None, None), + ("SCOPE_CATALOG", "string", None, None, None, None, None), + ("SCOPE_SCHEMA", "string", None, None, None, None, None), + ("SCOPE_TABLE", "string", None, None, None, None, None), + ("SOURCE_DATA_TYPE", "smallint", None, None, None, None, None), + ("IS_AUTO_INCREMENT", "string", None, None, None, None, None), + ] + assert cols_desc == expected + finally: + for table in table_names: + cursor.execute("DROP TABLE IF EXISTS {}".format(table)) + + def test_escape_single_quotes(self): + with self.cursor({}) as cursor: + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + # Test escape syntax directly + cursor.execute( + "CREATE TABLE IF NOT EXISTS {} AS (SELECT 'you\\'re' AS col_1)".format( + table_name + ) + ) + cursor.execute( + "SELECT * FROM {} WHERE col_1 LIKE 'you\\'re'".format(table_name) + ) + rows = cursor.fetchall() + assert rows[0]["col_1"] == "you're" + + # Test escape syntax in parameter + cursor.execute( + "SELECT * FROM {} WHERE {}.col_1 LIKE %(var)s".format( + table_name, table_name + ), + parameters={"var": "you're"}, + ) + rows = cursor.fetchall() + assert rows[0]["col_1"] == "you're" + + def test_get_schemas(self): + with self.cursor({}) as cursor: + database_name = "db_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + try: + cursor.execute("CREATE DATABASE IF NOT EXISTS {}".format(database_name)) + cursor.schemas() + schemas = cursor.fetchall() + schemas_desc = cursor.description + # Catalogue name not consistent across DBR versions, so we skip that + assert database_name in [schema[0] for schema in schemas] + assert schemas_desc == [ + ("TABLE_SCHEM", "string", None, None, None, None, None), + ("TABLE_CATALOG", "string", None, None, None, None, None), + ] + + finally: + cursor.execute("DROP DATABASE IF EXISTS {}".format(database_name)) + + def test_get_catalogs(self): + with self.cursor({}) as cursor: + cursor.catalogs() + cursor.fetchall() + catalogs_desc = cursor.description + assert catalogs_desc == [ + ("TABLE_CAT", "string", None, None, None, None, None) + ] + + @skipUnless(pysql_supports_arrow(), "arrow test need arrow support") + def test_get_arrow(self): + # These tests are quite light weight as the arrow fetch methods are used internally + # by everything else + with self.cursor({}) as cursor: + cursor.execute("SELECT * FROM range(10)") + table_1 = cursor.fetchmany_arrow(1).to_pydict() + assert table_1 == OrderedDict([("id", [0])]) + + table_2 = cursor.fetchall_arrow().to_pydict() + assert table_2 == OrderedDict([("id", [1, 2, 3, 4, 5, 6, 7, 8, 9])]) + + def test_unicode(self): + unicode_str = "数据砖" + with self.cursor({}) as cursor: + cursor.execute("SELECT '{}'".format(unicode_str)) + results = cursor.fetchall() + assert len(results) == 1 and len(results[0]) == 1 + assert results[0][0] == unicode_str + + def test_cancel_during_execute(self): + with self.cursor({}) as cursor: + + def execute_really_long_query(): + cursor.execute( + "SELECT SUM(A.id - B.id) " + + "FROM range(1000000000) A CROSS JOIN range(100000000) B " + + "GROUP BY (A.id - B.id)" + ) + + exec_thread = threading.Thread(target=execute_really_long_query) + + exec_thread.start() + # Make sure the query has started before cancelling + time.sleep(15) + cursor.cancel() + exec_thread.join(5) + assert not exec_thread.is_alive() + + # Fetching results should throw an exception + with pytest.raises((Error, thrift.Thrift.TException)): + cursor.fetchall() + with pytest.raises((Error, thrift.Thrift.TException)): + cursor.fetchone() + with pytest.raises((Error, thrift.Thrift.TException)): + cursor.fetchmany(10) + + # We should be able to execute a new command on the cursor + cursor.execute("SELECT * FROM range(3)") + assert len(cursor.fetchall()) == 3 + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_can_execute_command_after_failure(self): + with self.cursor({}) as cursor: + with pytest.raises(DatabaseError): + cursor.execute("this is a sytnax error") + + cursor.execute("SELECT 1;") + + res = cursor.fetchall() + self.assertEqualRowValues(res, [[1]]) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_can_execute_command_after_success(self): + with self.cursor({}) as cursor: + cursor.execute("SELECT 1;") + cursor.execute("SELECT 2;") + + res = cursor.fetchall() + self.assertEqualRowValues(res, [[2]]) + + def generate_multi_row_query(self): + query = "SELECT * FROM range(3);" + return query + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_fetchone(self): + with self.cursor({}) as cursor: + query = self.generate_multi_row_query() + cursor.execute(query) + + assert cursor.fetchone()[0] == 0 + assert cursor.fetchone()[0] == 1 + assert cursor.fetchone()[0] == 2 + + assert cursor.fetchone() == None + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_fetchall(self): + with self.cursor({}) as cursor: + query = self.generate_multi_row_query() + cursor.execute(query) + + self.assertEqualRowValues(cursor.fetchall(), [[0], [1], [2]]) + + assert cursor.fetchone() == None + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_fetchmany_when_stride_fits(self): + with self.cursor({}) as cursor: + query = "SELECT * FROM range(4)" + cursor.execute(query) + + self.assertEqualRowValues(cursor.fetchmany(2), [[0], [1]]) + self.assertEqualRowValues(cursor.fetchmany(2), [[2], [3]]) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_fetchmany_in_excess(self): + with self.cursor({}) as cursor: + query = "SELECT * FROM range(4)" + cursor.execute(query) + + self.assertEqualRowValues(cursor.fetchmany(3), [[0], [1], [2]]) + self.assertEqualRowValues(cursor.fetchmany(3), [[3]]) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_iterator_api(self): + with self.cursor({}) as cursor: + query = "SELECT * FROM range(4)" + cursor.execute(query) + + expected_results = [[0], [1], [2], [3]] + for i, row in enumerate(cursor): + for j in range(len(row)): + assert row[j] == expected_results[i][j] + + def test_temp_view_fetch(self): + with self.cursor({}) as cursor: + query = "create temporary view f as select * from range(10)" + cursor.execute(query) + # TODO assert on a result + # once what is being returned has stabilised + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + @skipIf( + True, + "Unclear the purpose of this test since urllib3 does not complain when timeout == 0", + ) + def test_socket_timeout(self): + # We expect to see a BlockingIO error when the socket is opened + # in non-blocking mode, since no poll is done before the read + with pytest.raises(OperationalError) as cm: + with self.cursor({"_socket_timeout": 0}): + pass + + self.assertIsInstance(cm.exception.args[1], io.BlockingIOError) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + @skipIf(pysql_has_version(">", "2.8"), "This test has been broken for a while") + def test_socket_timeout_user_defined(self): + # We expect to see a TimeoutError when the socket timeout is only + # 1 sec for a query that takes longer than that to process + with pytest.raises(ReadTimeoutError) as cm: + with self.cursor({"_socket_timeout": 1}) as cursor: + query = "select * from range(1000000000)" + cursor.execute(query) + + def test_ssp_passthrough(self): + for enable_ansi in (True, False): + with self.cursor( + {"session_configuration": {"ansi_mode": enable_ansi}} + ) as cursor: + cursor.execute("SET ansi_mode") + assert list(cursor.fetchone()) == ["ansi_mode", str(enable_ansi)] + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_timestamps_arrow(self): + with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: + for timestamp, expected in self.timestamp_and_expected_results: + cursor.execute( + "SELECT TIMESTAMP('{timestamp}')".format(timestamp=timestamp) + ) + arrow_table = cursor.fetchmany_arrow(1) + if self.should_add_timezone(): + ts_type = pyarrow.timestamp("us", tz="Etc/UTC") + else: + ts_type = pyarrow.timestamp("us") + assert arrow_table.field(0).type == ts_type + result_value = arrow_table.column(0).combine_chunks()[0].value + # To work consistently across different local timezones, we specify the timezone + # of the expected result to + # be UTC (what it should be by default on the server) + aware_timestamp = expected and expected.replace( + tzinfo=datetime.timezone.utc + ) + assert result_value == ( + aware_timestamp and aware_timestamp.timestamp() * 1000000 + ), "timestamp {} did not match {}".format(timestamp, expected) + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_multi_timestamps_arrow(self): + with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: + query, expected = self.multi_query() + expected = [ + [self.maybe_add_timezone_to_timestamp(ts) for ts in row] + for row in expected + ] + cursor.execute(query) + table = cursor.fetchall_arrow() + # Transpose columnar result to list of rows + list_of_cols = [c.to_pylist() for c in table] + result = [ + [col[row_index] for col in list_of_cols] + for row_index in range(table.num_rows) + ] + assert result == expected + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_timezone_with_timestamp(self): + if self.should_add_timezone(): + with self.cursor() as cursor: + cursor.execute("SET TIME ZONE 'Europe/Amsterdam'") + cursor.execute("select CAST('2022-03-02 12:54:56' as TIMESTAMP)") + amsterdam = pytz.timezone("Europe/Amsterdam") + expected = amsterdam.localize(datetime.datetime(2022, 3, 2, 12, 54, 56)) + result = cursor.fetchone()[0] + assert result == expected + + cursor.execute("select CAST('2022-03-02 12:54:56' as TIMESTAMP)") + arrow_result_table = cursor.fetchmany_arrow(1) + arrow_result_value = ( + arrow_result_table.column(0).combine_chunks()[0].value + ) + ts_type = pyarrow.timestamp("us", tz="Europe/Amsterdam") + + assert arrow_result_table.field(0).type == ts_type + assert arrow_result_value == expected.timestamp() * 1000000 + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_can_flip_compression(self): + with self.cursor() as cursor: + cursor.execute("SELECT array(1,2,3,4)") + cursor.fetchall() + lz4_compressed = cursor.active_result_set.lz4_compressed + # The endpoint should support compression + assert lz4_compressed + cursor.connection.lz4_compression = False + cursor.execute("SELECT array(1,2,3,4)") + cursor.fetchall() + lz4_compressed = cursor.active_result_set.lz4_compressed + assert not lz4_compressed + + def _should_have_native_complex_types(self): + return pysql_has_version(">=", 2) and is_thrift_v5_plus(self.arguments) + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_arrays_are_not_returned_as_strings_arrow(self): + if self._should_have_native_complex_types(): + with self.cursor() as cursor: + cursor.execute("SELECT array(1,2,3,4)") + arrow_df = cursor.fetchall_arrow() + + list_type = arrow_df.field(0).type + assert pyarrow.types.is_list(list_type) + assert pyarrow.types.is_integer(list_type.value_type) + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_structs_are_not_returned_as_strings_arrow(self): + if self._should_have_native_complex_types(): + with self.cursor() as cursor: + cursor.execute("SELECT named_struct('foo', 42, 'bar', 'baz')") + arrow_df = cursor.fetchall_arrow() + + struct_type = arrow_df.field(0).type + assert pyarrow.types.is_struct(struct_type) + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_decimal_not_returned_as_strings_arrow(self): + if self._should_have_native_complex_types(): + with self.cursor() as cursor: + cursor.execute("SELECT 5E3BD") + arrow_df = cursor.fetchall_arrow() + + decimal_type = arrow_df.field(0).type + assert pyarrow.types.is_decimal(decimal_type) + + def test_close_connection_closes_cursors(self): + + from databricks.sql.thrift_api.TCLIService import ttypes + + with self.connection() as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT id, id `id2`, id `id3` FROM RANGE(1000000) order by RANDOM()" + ) + ars = cursor.active_result_set + + # We must manually run this check because thrift_backend always forces `has_been_closed_server_side` to True + + # Cursor op state should be open before connection is closed + status_request = ttypes.TGetOperationStatusReq( + operationHandle=ars.command_id, getProgressUpdate=False + ) + op_status_at_server = ars.thrift_backend._client.GetOperationStatus( + status_request + ) + assert ( + op_status_at_server.operationState + != ttypes.TOperationState.CLOSED_STATE + ) + + conn.close() + + # When connection closes, any cursor operations should no longer exist at the server + with pytest.raises(SessionAlreadyClosedError) as cm: + op_status_at_server = ars.thrift_backend._client.GetOperationStatus( + status_request + ) + + def test_closing_a_closed_connection_doesnt_fail(self, caplog): + caplog.set_level(logging.DEBUG) + # Second .close() call is when this context manager exits + with self.connection() as conn: + # First .close() call is explicit here + conn.close() + + assert "Session appears to have been closed already" in caplog.text + + +# use a RetrySuite to encapsulate these tests which we'll typically want to run together; however keep +# the 429/503 subsuites separate since they execute under different circumstances. +class TestPySQLRetrySuite: + class HTTP429Suite(Client429ResponseMixin, PySQLPytestTestCase): + pass # Mixin covers all + + class HTTP503Suite(Client503ResponseMixin, PySQLPytestTestCase): + # 503Response suite gets custom error here vs PyODBC + def test_retry_disabled(self): + self._test_retry_disabled_with_message( + "TEMPORARILY_UNAVAILABLE", OperationalError + ) + + +class TestPySQLUnityCatalogSuite(PySQLPytestTestCase): + """Simple namespace tests that should be run against a unity-catalog-enabled cluster""" + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_initial_namespace(self): + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + with self.cursor() as cursor: + cursor.execute("USE CATALOG {}".format(self.arguments["catalog"])) + cursor.execute("CREATE TABLE table_{} (col1 int)".format(table_name)) + with self.connection( + {"catalog": self.arguments["catalog"], "schema": table_name} + ) as connection: + cursor = connection.cursor() + cursor.execute("select current_catalog()") + assert cursor.fetchone()[0] == self.arguments["catalog"] + cursor.execute("select current_database()") + assert cursor.fetchone()[0] == table_name diff --git a/tests/e2e/test_parameterized_queries.py b/tests/e2e/test_parameterized_queries.py new file mode 100644 index 00000000..d346ad5c --- /dev/null +++ b/tests/e2e/test_parameterized_queries.py @@ -0,0 +1,455 @@ +import datetime +from contextlib import contextmanager +from decimal import Decimal +from enum import Enum +from typing import Dict, List, Type, Union +from unittest.mock import patch + +import pytest +import pytz + +from databricks.sql.parameters.native import ( + BigIntegerParameter, + BooleanParameter, + DateParameter, + DbsqlParameterBase, + DecimalParameter, + DoubleParameter, + FloatParameter, + IntegerParameter, + ParameterApproach, + ParameterStructure, + SmallIntParameter, + StringParameter, + TDbsqlParameter, + TimestampNTZParameter, + TimestampParameter, + TinyIntParameter, + VoidParameter, +) +from tests.e2e.test_driver import PySQLPytestTestCase + + +class ParamStyle(Enum): + NAMED = 1 + PYFORMAT = 2 + NONE = 3 + + +class Primitive(Enum): + """These are the inferrable types. This Enum is used for parametrized tests.""" + + NONE = None + BOOL = True + INT = 50 + BIGINT = 2147483648 + STRING = "Hello" + DECIMAL = Decimal("1234.56") + DATE = datetime.date(2023, 9, 6) + TIMESTAMP = datetime.datetime(2023, 9, 6, 3, 14, 27, 843, tzinfo=pytz.UTC) + DOUBLE = 3.14 + FLOAT = 3.15 + SMALLINT = 51 + + +class PrimitiveExtra(Enum): + """These are not inferrable types. This Enum is used for parametrized tests.""" + + TIMESTAMP_NTZ = datetime.datetime(2023, 9, 6, 3, 14, 27, 843) + TINYINT = 20 + + +# We don't test inline approach with named paramstyle because it's never supported +# We don't test inline approach with positional parameters because it's never supported +# Paramstyle doesn't apply when ParameterStructure.POSITIONAL because question marks are used. +approach_paramstyle_combinations = [ + (ParameterApproach.INLINE, ParamStyle.PYFORMAT, ParameterStructure.NAMED), + (ParameterApproach.NATIVE, ParamStyle.NONE, ParameterStructure.POSITIONAL), + (ParameterApproach.NATIVE, ParamStyle.PYFORMAT, ParameterStructure.NAMED), + (ParameterApproach.NATIVE, ParamStyle.NONE, ParameterStructure.POSITIONAL), + (ParameterApproach.NATIVE, ParamStyle.NAMED, ParameterStructure.NAMED), +] + + +class TestParameterizedQueries(PySQLPytestTestCase): + """Namespace for tests of this connector's parameterisation behaviour. + + databricks-sql-connector can approach parameterisation in two ways: + + NATIVE: the connector will use server-side bound parameters implemented by DBR 14.1 and above. + INLINE: the connector will render parameter values as strings and interpolate them into the query. + + Prior to connector version 3.0.0, the connector would always use the INLINE approach. This approach + is still the default but this will be changed in a subsequent release. + + The INLINE and NATIVE approaches use different query syntax, which these tests verify. + + There is not 1-to-1 feature parity between these approaches. Where possible, we run the same test + for @both_approaches. + """ + + NAMED_PARAMSTYLE_QUERY = "SELECT :p AS col" + PYFORMAT_PARAMSTYLE_QUERY = "SELECT %(p)s AS col" + POSITIONAL_PARAMSTYLE_QUERY = "SELECT ? AS col" + + inline_type_map = { + Primitive.INT: "int_col", + Primitive.BIGINT: "bigint_col", + Primitive.SMALLINT: "small_int_col", + Primitive.FLOAT: "float_col", + Primitive.DOUBLE: "double_col", + Primitive.DECIMAL: "decimal_col", + Primitive.STRING: "string_col", + Primitive.BOOL: "boolean_col", + Primitive.DATE: "date_col", + Primitive.TIMESTAMP: "timestamp_col", + Primitive.NONE: "null_col", + } + + def _get_inline_table_column(self, value): + return self.inline_type_map[Primitive(value)] + + @pytest.fixture(scope="class") + def inline_table(self, connection_details): + self.arguments = connection_details.copy() + """This table is necessary to verify that a parameter sent with INLINE + approach can actually write to its analogous data type. + + For example, a Python Decimal(), when rendered inline, should be able + to read/write into a DECIMAL column in Databricks + + Note that this fixture doesn't clean itself up. So the table will remain + in the schema for use by subsequent test runs. + """ + + query = """ + CREATE TABLE IF NOT EXISTS pysql_e2e_inline_param_test_table ( + null_col INT, + int_col INT, + bigint_col BIGINT, + small_int_col SMALLINT, + float_col FLOAT, + double_col DOUBLE, + decimal_col DECIMAL(10, 2), + string_col STRING, + boolean_col BOOLEAN, + date_col DATE, + timestamp_col TIMESTAMP + ) USING DELTA + """ + + with self.connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + + @contextmanager + def patch_server_supports_native_params(self, supports_native_params: bool = True): + """Applies a patch so we can test the connector's behaviour under different SPARK_CLI_SERVICE_PROTOCOL_VERSION conditions.""" + + with patch( + "databricks.sql.client.Connection.server_parameterized_queries_enabled", + return_value=supports_native_params, + ) as mock_parameterized_queries_enabled: + try: + yield mock_parameterized_queries_enabled + finally: + pass + + def _inline_roundtrip(self, params: dict, paramstyle: ParamStyle): + """This INSERT, SELECT, DELETE dance is necessary because simply selecting + ``` + "SELECT %(param)s" + ``` + in INLINE mode would always return a str and the nature of the test is to + confirm that types are maintained. + + :paramstyle: + This is a no-op but is included to make the test-code easier to read. + """ + target_column = self._get_inline_table_column(params.get("p")) + INSERT_QUERY = f"INSERT INTO pysql_e2e_inline_param_test_table (`{target_column}`) VALUES (%(p)s)" + SELECT_QUERY = f"SELECT {target_column} `col` FROM pysql_e2e_inline_param_test_table LIMIT 1" + DELETE_QUERY = "DELETE FROM pysql_e2e_inline_param_test_table" + + with self.connection(extra_params={"use_inline_params": True}) as conn: + with conn.cursor() as cursor: + cursor.execute(INSERT_QUERY, parameters=params) + with conn.cursor() as cursor: + to_return = cursor.execute(SELECT_QUERY).fetchone() + with conn.cursor() as cursor: + cursor.execute(DELETE_QUERY) + + return to_return + + def _native_roundtrip( + self, + parameters: Union[Dict, List[Dict]], + paramstyle: ParamStyle, + parameter_structure: ParameterStructure, + ): + if parameter_structure == ParameterStructure.POSITIONAL: + _query = self.POSITIONAL_PARAMSTYLE_QUERY + elif paramstyle == ParamStyle.NAMED: + _query = self.NAMED_PARAMSTYLE_QUERY + elif paramstyle == ParamStyle.PYFORMAT: + _query = self.PYFORMAT_PARAMSTYLE_QUERY + with self.connection(extra_params={"use_inline_params": False}) as conn: + with conn.cursor() as cursor: + cursor.execute(_query, parameters=parameters) + return cursor.fetchone() + + def _get_one_result( + self, + params, + approach: ParameterApproach = ParameterApproach.NONE, + paramstyle: ParamStyle = ParamStyle.NONE, + parameter_structure: ParameterStructure = ParameterStructure.NONE, + ): + """When approach is INLINE then we use %(param)s paramstyle and a connection with use_inline_params=True + When approach is NATIVE then we use :param paramstyle and a connection with use_inline_params=False + """ + + if approach == ParameterApproach.INLINE: + # inline mode always uses ParamStyle.PYFORMAT + # inline mode doesn't support positional parameters + return self._inline_roundtrip(params, paramstyle=ParamStyle.PYFORMAT) + elif approach == ParameterApproach.NATIVE: + # native mode can use either ParamStyle.NAMED or ParamStyle.PYFORMAT + # native mode can use either ParameterStructure.NAMED or ParameterStructure.POSITIONAL + return self._native_roundtrip( + params, paramstyle=paramstyle, parameter_structure=parameter_structure + ) + + def _quantize(self, input: Union[float, int], place_value=2) -> Decimal: + return Decimal(str(input)).quantize(Decimal("0." + "0" * place_value)) + + def _eq(self, actual, expected: Primitive): + """This is a helper function to make the test code more readable. + + If primitive is Primitive.DOUBLE than an extra quantize step is performed before + making the assertion. + """ + if expected in (Primitive.DOUBLE, Primitive.FLOAT): + return self._quantize(actual) == self._quantize(expected.value) + + return actual == expected.value + + @pytest.mark.parametrize("primitive", Primitive) + @pytest.mark.parametrize( + "approach,paramstyle,parameter_structure", approach_paramstyle_combinations + ) + def test_primitive_single( + self, + approach, + paramstyle, + parameter_structure, + primitive: Primitive, + inline_table, + ): + """When ParameterApproach.INLINE is passed, inferrence will not be used. + When ParameterApproach.NATIVE is passed, primitive inputs will be inferred. + """ + + if parameter_structure == ParameterStructure.NAMED: + params = {"p": primitive.value} + elif parameter_structure == ParameterStructure.POSITIONAL: + params = [primitive.value] + + result = self._get_one_result(params, approach, paramstyle, parameter_structure) + + assert self._eq(result.col, primitive) + + @pytest.mark.parametrize( + "parameter_structure", (ParameterStructure.NAMED, ParameterStructure.POSITIONAL) + ) + @pytest.mark.parametrize( + "primitive,dbsql_parameter_cls", + [ + (Primitive.NONE, VoidParameter), + (Primitive.BOOL, BooleanParameter), + (Primitive.INT, IntegerParameter), + (Primitive.BIGINT, BigIntegerParameter), + (Primitive.STRING, StringParameter), + (Primitive.DECIMAL, DecimalParameter), + (Primitive.DATE, DateParameter), + (Primitive.TIMESTAMP, TimestampParameter), + (Primitive.DOUBLE, DoubleParameter), + (Primitive.FLOAT, FloatParameter), + (Primitive.SMALLINT, SmallIntParameter), + (PrimitiveExtra.TIMESTAMP_NTZ, TimestampNTZParameter), + (PrimitiveExtra.TINYINT, TinyIntParameter), + ], + ) + def test_dbsqlparameter_single( + self, + primitive: Primitive, + dbsql_parameter_cls: Type[TDbsqlParameter], + parameter_structure: ParameterStructure, + ): + dbsql_param = dbsql_parameter_cls( + value=primitive.value, # type: ignore + name="p" if parameter_structure == ParameterStructure.NAMED else None, + ) + + params = [dbsql_param] + result = self._get_one_result( + params, ParameterApproach.NATIVE, ParamStyle.NAMED, parameter_structure + ) + assert self._eq(result.col, primitive) + + @pytest.mark.parametrize("use_inline_params", (True, False, "silent")) + def test_use_inline_off_by_default_with_warning(self, use_inline_params, caplog): + """ + use_inline_params should be False by default. + If a user explicitly sets use_inline_params, don't warn them about it. + """ + + extra_args = ( + {"use_inline_params": use_inline_params} if use_inline_params else {} + ) + + with self.connection(extra_params=extra_args) as conn: + with conn.cursor() as cursor: + with self.patch_server_supports_native_params( + supports_native_params=True + ): + cursor.execute("SELECT %(p)s", parameters={"p": 1}) + if use_inline_params is True: + assert ( + "Consider using native parameters." in caplog.text + ), "Log message should be suppressed" + elif use_inline_params == "silent": + assert ( + "Consider using native parameters." not in caplog.text + ), "Log message should not be supressed" + + def test_positional_native_params_with_defaults(self): + query = "SELECT ? col" + with self.cursor() as cursor: + result = cursor.execute(query, parameters=[1]).fetchone() + + assert result.col == 1 + + @pytest.mark.parametrize( + "params", + ( + [ + StringParameter(value="foo"), + StringParameter(value="bar"), + StringParameter(value="baz"), + ], + ["foo", "bar", "baz"], + ), + ) + def test_positional_native_multiple(self, params): + query = "SELECT ? `foo`, ? `bar`, ? `baz`" + + with self.cursor(extra_params={"use_inline_params": False}) as cursor: + result = cursor.execute(query, params).fetchone() + + expected = [i.value if isinstance(i, DbsqlParameterBase) else i for i in params] + outcome = [result.foo, result.bar, result.baz] + + assert set(outcome) == set(expected) + + def test_readme_example(self): + with self.cursor() as cursor: + result = cursor.execute( + "SELECT :param `p`, * FROM RANGE(10)", {"param": "foo"} + ).fetchall() + + assert len(result) == 10 + assert result[0].p == "foo" + + +class TestInlineParameterSyntax(PySQLPytestTestCase): + """The inline parameter approach uses pyformat markers""" + + def test_params_as_dict(self): + query = "SELECT %(foo)s foo, %(bar)s bar, %(baz)s baz" + params = {"foo": 1, "bar": 2, "baz": 3} + + with self.connection(extra_params={"use_inline_params": True}) as conn: + with conn.cursor() as cursor: + result = cursor.execute(query, parameters=params).fetchone() + + assert result.foo == 1 + assert result.bar == 2 + assert result.baz == 3 + + def test_params_as_sequence(self): + """One side-effect of ParamEscaper using Python string interpolation to inline the values + is that it can work with "ordinal" parameters, but only if a user writes parameter markers + that are not defined with PEP-249. This test exists to prove that it works in the ideal case. + """ + + # `%s` is not a valid paramstyle per PEP-249 + query = "SELECT %s foo, %s bar, %s baz" + params = (1, 2, 3) + + with self.connection(extra_params={"use_inline_params": True}) as conn: + with conn.cursor() as cursor: + result = cursor.execute(query, parameters=params).fetchone() + assert result.foo == 1 + assert result.bar == 2 + assert result.baz == 3 + + def test_inline_ordinals_can_break_sql(self): + """With inline mode, ordinal parameters can break the SQL syntax + because `%` symbols are used to wildcard match within LIKE statements. This test + just proves that's the case. + """ + query = "SELECT 'samsonite', %s WHERE 'samsonite' LIKE '%sonite'" + params = ["luggage"] + with self.cursor(extra_params={"use_inline_params": True}) as cursor: + with pytest.raises( + TypeError, match="not enough arguments for format string" + ): + cursor.execute(query, parameters=params) + + def test_inline_named_dont_break_sql(self): + """With inline mode, ordinal parameters can break the SQL syntax + because `%` symbols are used to wildcard match within LIKE statements. This test + just proves that's the case. + """ + query = """ + with base as (SELECT 'x(one)sonite' as `col_1`) + SELECT col_1 FROM base WHERE col_1 LIKE CONCAT(%(one)s, 'onite') + """ + params = {"one": "%(one)s"} + with self.cursor(extra_params={"use_inline_params": True}) as cursor: + result = cursor.execute(query, parameters=params).fetchone() + print("hello") + + def test_native_ordinals_dont_break_sql(self): + """This test accompanies test_inline_ordinals_can_break_sql to prove that ordinal + parameters work in native mode for the exact same query, if we use the right marker `?` + """ + query = "SELECT 'samsonite', ? WHERE 'samsonite' LIKE '%sonite'" + params = ["luggage"] + with self.cursor(extra_params={"use_inline_params": False}) as cursor: + result = cursor.execute(query, parameters=params).fetchone() + + assert result.samsonite == "samsonite" + assert result.luggage == "luggage" + + def test_inline_like_wildcard_breaks(self): + """One flaw with the ParameterEscaper is that it fails if a query contains + a SQL LIKE wildcard %. This test proves that's the case. + """ + query = "SELECT 1 `col` WHERE 'foo' LIKE '%'" + params = {"param": "bar"} + with self.cursor(extra_params={"use_inline_params": True}) as cursor: + with pytest.raises(ValueError, match="unsupported format character"): + result = cursor.execute(query, parameters=params).fetchone() + + def test_native_like_wildcard_works(self): + """This is a mirror of test_inline_like_wildcard_breaks that proves that LIKE + wildcards work under the native approach. + """ + query = "SELECT 1 `col` WHERE 'foo' LIKE '%'" + params = {"param": "bar"} + with self.cursor(extra_params={"use_inline_params": False}) as cursor: + result = cursor.execute(query, parameters=params).fetchone() + + assert result.col == 1 diff --git a/tests/sqlalchemy/demo_data/MOCK_DATA.xlsx b/tests/sqlalchemy/demo_data/MOCK_DATA.xlsx deleted file mode 100644 index e080689a..00000000 Binary files a/tests/sqlalchemy/demo_data/MOCK_DATA.xlsx and /dev/null differ diff --git a/tests/unit/test_arrow_queue.py b/tests/unit/test_arrow_queue.py index 6834cc9c..6c195bf1 100644 --- a/tests/unit/test_arrow_queue.py +++ b/tests/unit/test_arrow_queue.py @@ -1,10 +1,14 @@ import unittest +import pytest -import pyarrow as pa - +try: + import pyarrow as pa +except ImportError: + pa = None from databricks.sql.utils import ArrowQueue +@pytest.mark.skipif(pa is None, reason="PyArrow is not installed") class ArrowQueueSuite(unittest.TestCase): @staticmethod def make_arrow_table(batch): @@ -14,13 +18,21 @@ def make_arrow_table(batch): return pa.Table.from_pydict(dict(zip(schema.names, cols)), schema=schema) def test_fetchmany_respects_n_rows(self): - arrow_table = self.make_arrow_table([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]) + arrow_table = self.make_arrow_table( + [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + ) aq = ArrowQueue(arrow_table, 3) - self.assertEqual(aq.next_n_rows(2), self.make_arrow_table([[0, 1, 2], [3, 4, 5]])) + self.assertEqual( + aq.next_n_rows(2), self.make_arrow_table([[0, 1, 2], [3, 4, 5]]) + ) self.assertEqual(aq.next_n_rows(2), self.make_arrow_table([[6, 7, 8]])) def test_fetch_remaining_rows_respects_n_rows(self): - arrow_table = self.make_arrow_table([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]) + arrow_table = self.make_arrow_table( + [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + ) aq = ArrowQueue(arrow_table, 3) self.assertEqual(aq.next_n_rows(1), self.make_arrow_table([[0, 1, 2]])) - self.assertEqual(aq.remaining_rows(), self.make_arrow_table([[3, 4, 5], [6, 7, 8]])) + self.assertEqual( + aq.remaining_rows(), self.make_arrow_table([[3, 4, 5], [6, 7, 8]]) + ) diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py index c52f9790..d5b06bbf 100644 --- a/tests/unit/test_auth.py +++ b/tests/unit/test_auth.py @@ -1,103 +1,192 @@ import unittest - -from databricks.sql.auth.auth import AccessTokenAuthProvider, BasicAuthProvider, AuthProvider, ExternalAuthProvider -from databricks.sql.auth.auth import get_python_sql_connector_auth_provider +import pytest +from typing import Optional +from unittest.mock import patch + +from databricks.sql.auth.auth import ( + AccessTokenAuthProvider, + AuthProvider, + ExternalAuthProvider, + AuthType, +) +from databricks.sql.auth.auth import ( + get_python_sql_connector_auth_provider, + PYSQL_OAUTH_CLIENT_ID, +) +from databricks.sql.auth.oauth import OAuthManager +from databricks.sql.auth.authenticators import DatabricksOAuthProvider +from databricks.sql.auth.endpoint import ( + CloudType, + InHouseOAuthEndpointCollection, + AzureOAuthEndpointCollection, +) from databricks.sql.auth.authenticators import CredentialsProvider, HeaderFactory +from databricks.sql.experimental.oauth_persistence import OAuthPersistenceCache class Auth(unittest.TestCase): - def test_access_token_provider(self): access_token = "aBc2" auth = AccessTokenAuthProvider(access_token=access_token) - http_request = {'myKey': 'myVal'} - auth.add_headers(http_request) - self.assertEqual(http_request['Authorization'], 'Bearer aBc2') - self.assertEqual(len(http_request.keys()), 2) - self.assertEqual(http_request['myKey'], 'myVal') - - def test_basic_auth_provider(self): - username = "moderakh" - password = "Elevate Databricks 123!!!" - auth = BasicAuthProvider(username=username, password=password) - - http_request = {'myKey': 'myVal'} + http_request = {"myKey": "myVal"} auth.add_headers(http_request) - - self.assertEqual(http_request['Authorization'], 'Basic bW9kZXJha2g6RWxldmF0ZSBEYXRhYnJpY2tzIDEyMyEhIQ==') + self.assertEqual(http_request["Authorization"], "Bearer aBc2") self.assertEqual(len(http_request.keys()), 2) - self.assertEqual(http_request['myKey'], 'myVal') + self.assertEqual(http_request["myKey"], "myVal") def test_noop_auth_provider(self): auth = AuthProvider() - http_request = {'myKey': 'myVal'} + http_request = {"myKey": "myVal"} auth.add_headers(http_request) self.assertEqual(len(http_request.keys()), 1) - self.assertEqual(http_request['myKey'], 'myVal') + self.assertEqual(http_request["myKey"], "myVal") + + @patch.object(OAuthManager, "check_and_refresh_access_token") + @patch.object(OAuthManager, "get_tokens") + def test_oauth_auth_provider(self, mock_get_tokens, mock_check_and_refresh): + client_id = "mock-id" + scopes = ["offline_access", "sql"] + access_token = "mock_token" + refresh_token = "mock_refresh_token" + mock_get_tokens.return_value = (access_token, refresh_token) + mock_check_and_refresh.return_value = (access_token, refresh_token, False) + + params = [ + ( + CloudType.AWS, + "foo.cloud.databricks.com", + False, + InHouseOAuthEndpointCollection, + "offline_access sql", + ), + ( + CloudType.AZURE, + "foo.1.azuredatabricks.net", + True, + AzureOAuthEndpointCollection, + f"{AzureOAuthEndpointCollection.DATATRICKS_AZURE_APP}/user_impersonation offline_access", + ), + ( + CloudType.AZURE, + "foo.1.azuredatabricks.net", + False, + InHouseOAuthEndpointCollection, + "offline_access sql", + ), + ( + CloudType.GCP, + "foo.gcp.databricks.com", + False, + InHouseOAuthEndpointCollection, + "offline_access sql", + ), + ] + + for ( + cloud_type, + host, + use_azure_auth, + expected_endpoint_type, + expected_scopes, + ) in params: + with self.subTest(cloud_type.value): + oauth_persistence = OAuthPersistenceCache() + auth_provider = DatabricksOAuthProvider( + hostname=host, + oauth_persistence=oauth_persistence, + redirect_port_range=[8020], + client_id=client_id, + scopes=scopes, + auth_type=AuthType.AZURE_OAUTH.value + if use_azure_auth + else AuthType.DATABRICKS_OAUTH.value, + ) + + self.assertIsInstance( + auth_provider.oauth_manager.idp_endpoint, expected_endpoint_type + ) + self.assertEqual(auth_provider.oauth_manager.port_range, [8020]) + self.assertEqual(auth_provider.oauth_manager.client_id, client_id) + self.assertEqual( + oauth_persistence.read(host).refresh_token, refresh_token + ) + mock_get_tokens.assert_called_with(hostname=host, scope=expected_scopes) + + headers = {} + auth_provider.add_headers(headers) + self.assertEqual(headers["Authorization"], f"Bearer {access_token}") def test_external_provider(self): class MyProvider(CredentialsProvider): - def auth_type(self) -> str: - return "mine" + def auth_type(self) -> str: + return "mine" - def __call__(self, *args, **kwargs) -> HeaderFactory: - return lambda: {"foo": "bar"} + def __call__(self, *args, **kwargs) -> HeaderFactory: + return lambda: {"foo": "bar"} auth = ExternalAuthProvider(MyProvider()) - http_request = {'myKey': 'myVal'} + http_request = {"myKey": "myVal"} auth.add_headers(http_request) - self.assertEqual(http_request['foo'], 'bar') + self.assertEqual(http_request["foo"], "bar") self.assertEqual(len(http_request.keys()), 2) - self.assertEqual(http_request['myKey'], 'myVal') + self.assertEqual(http_request["myKey"], "myVal") def test_get_python_sql_connector_auth_provider_access_token(self): hostname = "moderakh-test.cloud.databricks.com" - kwargs = {'access_token': 'dpi123'} + kwargs = {"access_token": "dpi123"} auth_provider = get_python_sql_connector_auth_provider(hostname, **kwargs) self.assertTrue(type(auth_provider).__name__, "AccessTokenAuthProvider") headers = {} auth_provider.add_headers(headers) - self.assertEqual(headers['Authorization'], 'Bearer dpi123') + self.assertEqual(headers["Authorization"], "Bearer dpi123") def test_get_python_sql_connector_auth_provider_external(self): - class MyProvider(CredentialsProvider): - def auth_type(self) -> str: - return "mine" + def auth_type(self) -> str: + return "mine" - def __call__(self, *args, **kwargs) -> HeaderFactory: - return lambda: {"foo": "bar"} + def __call__(self, *args, **kwargs) -> HeaderFactory: + return lambda: {"foo": "bar"} hostname = "moderakh-test.cloud.databricks.com" - kwargs = {'credentials_provider': MyProvider()} + kwargs = {"credentials_provider": MyProvider()} auth_provider = get_python_sql_connector_auth_provider(hostname, **kwargs) self.assertTrue(type(auth_provider).__name__, "ExternalAuthProvider") headers = {} auth_provider.add_headers(headers) - self.assertEqual(headers['foo'], 'bar') - - def test_get_python_sql_connector_auth_provider_username_password(self): - username = "moderakh" - password = "Elevate Databricks 123!!!" - hostname = "moderakh-test.cloud.databricks.com" - kwargs = {'_username': username, '_password': password} - auth_provider = get_python_sql_connector_auth_provider(hostname, **kwargs) - self.assertTrue(type(auth_provider).__name__, "BasicAuthProvider") - - headers = {} - auth_provider.add_headers(headers) - self.assertEqual(headers['Authorization'], 'Basic bW9kZXJha2g6RWxldmF0ZSBEYXRhYnJpY2tzIDEyMyEhIQ==') + self.assertEqual(headers["foo"], "bar") def test_get_python_sql_connector_auth_provider_noop(self): tls_client_cert_file = "fake.cert" use_cert_as_auth = "abc" hostname = "moderakh-test.cloud.databricks.com" - kwargs = {'_tls_client_cert_file': tls_client_cert_file, '_use_cert_as_auth': use_cert_as_auth} + kwargs = { + "_tls_client_cert_file": tls_client_cert_file, + "_use_cert_as_auth": use_cert_as_auth, + } auth_provider = get_python_sql_connector_auth_provider(hostname, **kwargs) self.assertTrue(type(auth_provider).__name__, "CredentialProvider") + + def test_get_python_sql_connector_basic_auth(self): + kwargs = { + "username": "username", + "password": "password", + } + with self.assertRaises(ValueError) as e: + get_python_sql_connector_auth_provider("foo.cloud.databricks.com", **kwargs) + self.assertIn( + "Username/password authentication is no longer supported", str(e.exception) + ) + + @patch.object(DatabricksOAuthProvider, "_initial_get_token") + def test_get_python_sql_connector_default_auth(self, mock__initial_get_token): + hostname = "foo.cloud.databricks.com" + auth_provider = get_python_sql_connector_auth_provider(hostname) + self.assertTrue(type(auth_provider).__name__, "DatabricksOAuthProvider") + self.assertTrue(auth_provider._client_id, PYSQL_OAUTH_CLIENT_ID) diff --git a/tests/unit/tests.py b/tests/unit/test_client.py similarity index 68% rename from tests/unit/tests.py rename to tests/unit/test_client.py index 74274373..0ff660d5 100644 --- a/tests/unit/tests.py +++ b/tests/unit/test_client.py @@ -2,10 +2,20 @@ import re import sys import unittest -from unittest.mock import patch, MagicMock, Mock +from unittest.mock import patch, MagicMock, Mock, PropertyMock import itertools from decimal import Decimal from datetime import datetime, date +from uuid import UUID + +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TOpenSessionResp, + TExecuteStatementResp, + TOperationHandle, + THandleIdentifier, + TOperationType, +) +from databricks.sql.thrift_backend import ThriftBackend import databricks.sql import databricks.sql.client as client @@ -17,6 +27,47 @@ from tests.unit.test_arrow_queue import ArrowQueueSuite +class ThriftBackendMockFactory: + @classmethod + def new(cls): + ThriftBackendMock = Mock(spec=ThriftBackend) + ThriftBackendMock.return_value = ThriftBackendMock + + cls.apply_property_to_mock(ThriftBackendMock, staging_allowed_local_path=None) + MockTExecuteStatementResp = MagicMock(spec=TExecuteStatementResp()) + + cls.apply_property_to_mock( + MockTExecuteStatementResp, + description=None, + arrow_queue=None, + is_staging_operation=False, + command_handle=b"\x22", + has_been_closed_server_side=True, + has_more_rows=True, + lz4_compressed=True, + arrow_schema_bytes=b"schema", + ) + + ThriftBackendMock.execute_command.return_value = MockTExecuteStatementResp + + return ThriftBackendMock + + @classmethod + def apply_property_to_mock(self, mock_obj, **kwargs): + """ + Apply a property to a mock object. + """ + + for key, value in kwargs.items(): + if value is not None: + kwargs = {"return_value": value} + else: + kwargs = {} + + prop = PropertyMock(**kwargs) + setattr(type(mock_obj), key, prop) + + class ClientTestSuite(unittest.TestCase): """ Unit tests for isolated client behaviour. @@ -32,20 +83,22 @@ class ClientTestSuite(unittest.TestCase): @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_close_uses_the_correct_session_id(self, mock_client_class): instance = mock_client_class.return_value - instance.open_session.return_value = b'\x22' + + mock_open_session_resp = MagicMock(spec=TOpenSessionResp)() + mock_open_session_resp.sessionHandle.sessionId = b"\x22" + instance.open_session.return_value = mock_open_session_resp connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) connection.close() # Check the close session request has an id of x22 - close_session_id = instance.close_session.call_args[0][0] - self.assertEqual(close_session_id, b'\x22') + close_session_id = instance.close_session.call_args[0][0].sessionId + self.assertEqual(close_session_id, b"\x22") @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_auth_args(self, mock_client_class): # Test that the following auth args work: # token = foo, - # token = None, _username = foo, _password = bar # token = None, _tls_client_cert_file = something, _use_cert_as_auth = True connection_args = [ { @@ -53,13 +106,6 @@ def test_auth_args(self, mock_client_class): "http_path": None, "access_token": "tok", }, - { - "server_hostname": "foo", - "http_path": None, - "_username": "foo", - "_password": "bar", - "access_token": None, - }, { "server_hostname": "foo", "http_path": None, @@ -71,7 +117,7 @@ def test_auth_args(self, mock_client_class): for args in connection_args: connection = databricks.sql.connect(**args) - host, port, http_path, _ = mock_client_class.call_args[0] + host, port, http_path, *_ = mock_client_class.call_args[0] self.assertEqual(args["server_hostname"], host) self.assertEqual(args["http_path"], http_path) connection.close() @@ -84,14 +130,6 @@ def test_http_header_passthrough(self, mock_client_class): call_args = mock_client_class.call_args[0][3] self.assertIn(("foo", "bar"), call_args) - @patch("%s.client.ThriftBackend" % PACKAGE_NAME) - def test_authtoken_passthrough(self, mock_client_class): - databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) - - headers = mock_client_class.call_args[0][3] - - self.assertIn(("Authorization", "Bearer tok"), headers) - @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_tls_arg_passthrough(self, mock_client_class): databricks.sql.connect( @@ -113,19 +151,25 @@ def test_useragent_header(self, mock_client_class): databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) http_headers = mock_client_class.call_args[0][3] - user_agent_header = ("User-Agent", "{}/{}".format(databricks.sql.USER_AGENT_NAME, - databricks.sql.__version__)) + user_agent_header = ( + "User-Agent", + "{}/{}".format(databricks.sql.USER_AGENT_NAME, databricks.sql.__version__), + ) self.assertIn(user_agent_header, http_headers) databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS, _user_agent_entry="foobar") - user_agent_header_with_entry = ("User-Agent", "{}/{} ({})".format( - databricks.sql.USER_AGENT_NAME, databricks.sql.__version__, "foobar")) + user_agent_header_with_entry = ( + "User-Agent", + "{}/{} ({})".format( + databricks.sql.USER_AGENT_NAME, databricks.sql.__version__, "foobar" + ), + ) http_headers = mock_client_class.call_args[0][3] self.assertIn(user_agent_header_with_entry, http_headers) - @patch("%s.client.ThriftBackend" % PACKAGE_NAME) + @patch("%s.client.ThriftBackend" % PACKAGE_NAME, ThriftBackendMockFactory.new()) @patch("%s.client.ResultSet" % PACKAGE_NAME) - def test_closing_connection_closes_commands(self, mock_result_set_class, mock_client_class): + def test_closing_connection_closes_commands(self, mock_result_set_class): # Test once with has_been_closed_server side, once without for closed in (True, False): with self.subTest(closed=closed): @@ -135,7 +179,9 @@ def test_closing_connection_closes_commands(self, mock_result_set_class, mock_cl cursor.execute("SELECT 1;") connection.close() - self.assertTrue(mock_result_set_class.return_value.has_been_closed_server_side) + self.assertTrue( + mock_result_set_class.return_value.has_been_closed_server_side + ) mock_result_set_class.return_value.close.assert_called_once_with() @patch("%s.client.ThriftBackend" % PACKAGE_NAME) @@ -150,7 +196,9 @@ def test_cant_open_cursor_on_closed_connection(self, mock_client_class): @patch("%s.client.ThriftBackend" % PACKAGE_NAME) @patch("%s.client.Cursor" % PACKAGE_NAME) - def test_arraysize_buffer_size_passthrough(self, mock_cursor_class, mock_client_class): + def test_arraysize_buffer_size_passthrough( + self, mock_cursor_class, mock_client_class + ): connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) connection.cursor(arraysize=999, buffer_size_bytes=1234) kwargs = mock_cursor_class.call_args[1] @@ -162,7 +210,10 @@ def test_closing_result_set_with_closed_connection_soft_closes_commands(self): mock_connection = Mock() mock_backend = Mock() result_set = client.ResultSet( - connection=mock_connection, thrift_backend=mock_backend, execute_response=Mock()) + connection=mock_connection, + thrift_backend=mock_backend, + execute_response=Mock(), + ) mock_connection.open = False result_set.close() @@ -176,19 +227,27 @@ def test_closing_result_set_hard_closes_commands(self): mock_connection = Mock() mock_thrift_backend = Mock() mock_connection.open = True - result_set = client.ResultSet(mock_connection, mock_results_response, mock_thrift_backend) + result_set = client.ResultSet( + mock_connection, mock_results_response, mock_thrift_backend + ) result_set.close() mock_thrift_backend.close_command.assert_called_once_with( - mock_results_response.command_handle) + mock_results_response.command_handle + ) @patch("%s.client.ResultSet" % PACKAGE_NAME) - def test_executing_multiple_commands_uses_the_most_recent_command(self, mock_result_set_class): + def test_executing_multiple_commands_uses_the_most_recent_command( + self, mock_result_set_class + ): + mock_result_sets = [Mock(), Mock()] mock_result_set_class.side_effect = mock_result_sets - cursor = client.Cursor(Mock(), Mock()) + cursor = client.Cursor( + connection=Mock(), thrift_backend=ThriftBackendMockFactory.new() + ) cursor.execute("SELECT 1;") cursor.execute("SELECT 1;") @@ -227,14 +286,17 @@ def test_context_manager_closes_cursor(self): @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_context_manager_closes_connection(self, mock_client_class): instance = mock_client_class.return_value - instance.open_session.return_value = b'\x22' + + mock_open_session_resp = MagicMock(spec=TOpenSessionResp)() + mock_open_session_resp.sessionHandle.sessionId = b"\x22" + instance.open_session.return_value = mock_open_session_resp with databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) as connection: pass # Check the close session request has an id of x22 - close_session_id = instance.close_session.call_args[0][0] - self.assertEqual(close_session_id, b'\x22') + close_session_id = instance.close_session.call_args[0][0].sessionId + self.assertEqual(close_session_id, b"\x22") def dict_product(self, dicts): """ @@ -253,7 +315,9 @@ def test_get_schemas_parameters_passed_to_thrift_backend(self, mock_thrift_backe req_args_combinations = self.dict_product( dict( catalog_name=["NOT_SET", None, "catalog_pattern"], - schema_name=["NOT_SET", None, "schema_pattern"])) + schema_name=["NOT_SET", None, "schema_pattern"], + ) + ) for req_args in req_args_combinations: req_args = {k: v for k, v in req_args.items() if v != "NOT_SET"} @@ -274,7 +338,9 @@ def test_get_tables_parameters_passed_to_thrift_backend(self, mock_thrift_backen catalog_name=["NOT_SET", None, "catalog_pattern"], schema_name=["NOT_SET", None, "schema_pattern"], table_name=["NOT_SET", None, "table_pattern"], - table_types=["NOT_SET", [], ["type1", "type2"]])) + table_types=["NOT_SET", [], ["type1", "type2"]], + ) + ) for req_args in req_args_combinations: req_args = {k: v for k, v in req_args.items() if v != "NOT_SET"} @@ -295,7 +361,9 @@ def test_get_columns_parameters_passed_to_thrift_backend(self, mock_thrift_backe catalog_name=["NOT_SET", None, "catalog_pattern"], schema_name=["NOT_SET", None, "schema_pattern"], table_name=["NOT_SET", None, "table_pattern"], - column_name=["NOT_SET", None, "column_pattern"])) + column_name=["NOT_SET", None, "column_pattern"], + ) + ) for req_args in req_args_combinations: req_args = {k: v for k, v in req_args.items() if v != "NOT_SET"} @@ -315,11 +383,12 @@ def test_cancel_command_calls_the_backend(self): mock_op_handle = Mock() cursor.active_op_handle = mock_op_handle cursor.cancel() - self.assertTrue(mock_thrift_backend.cancel_command.called_with(mock_op_handle)) + mock_thrift_backend.cancel_command.assert_called_with(mock_op_handle) @patch("databricks.sql.client.logger") def test_cancel_command_will_issue_warning_for_cancel_with_no_executing_command( - self, logger_instance): + self, logger_instance + ): mock_thrift_backend = Mock() cursor = client.Cursor(Mock(), mock_thrift_backend) cursor.cancel() @@ -329,9 +398,13 @@ def test_cancel_command_will_issue_warning_for_cancel_with_no_executing_command( @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_max_number_of_retries_passthrough(self, mock_client_class): - databricks.sql.connect(_retry_stop_after_attempts_count=54, **self.DUMMY_CONNECTION_ARGS) + databricks.sql.connect( + _retry_stop_after_attempts_count=54, **self.DUMMY_CONNECTION_ARGS + ) - self.assertEqual(mock_client_class.call_args[1]["_retry_stop_after_attempts_count"], 54) + self.assertEqual( + mock_client_class.call_args[1]["_retry_stop_after_attempts_count"], 54 + ) @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_socket_timeout_passthrough(self, mock_client_class): @@ -340,62 +413,74 @@ def test_socket_timeout_passthrough(self, mock_client_class): def test_version_is_canonical(self): version = databricks.sql.__version__ - canonical_version_re = r'^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)' \ - r'(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$' + canonical_version_re = ( + r"^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)" + r"(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$" + ) self.assertIsNotNone(re.match(canonical_version_re, version)) @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_configuration_passthrough(self, mock_client_class): mock_session_config = Mock() databricks.sql.connect( - session_configuration=mock_session_config, **self.DUMMY_CONNECTION_ARGS) + session_configuration=mock_session_config, **self.DUMMY_CONNECTION_ARGS + ) - self.assertEqual(mock_client_class.return_value.open_session.call_args[0][0], - mock_session_config) + self.assertEqual( + mock_client_class.return_value.open_session.call_args[0][0], + mock_session_config, + ) @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_initial_namespace_passthrough(self, mock_client_class): mock_cat = Mock() mock_schem = Mock() - databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS, catalog=mock_cat, schema=mock_schem) - self.assertEqual(mock_client_class.return_value.open_session.call_args[0][1], mock_cat) - self.assertEqual(mock_client_class.return_value.open_session.call_args[0][2], mock_schem) + databricks.sql.connect( + **self.DUMMY_CONNECTION_ARGS, catalog=mock_cat, schema=mock_schem + ) + self.assertEqual( + mock_client_class.return_value.open_session.call_args[0][1], mock_cat + ) + self.assertEqual( + mock_client_class.return_value.open_session.call_args[0][2], mock_schem + ) def test_execute_parameter_passthrough(self): - mock_thrift_backend = Mock() + mock_thrift_backend = ThriftBackendMockFactory.new() cursor = client.Cursor(Mock(), mock_thrift_backend) - tests = [("SELECT %(string_v)s", "SELECT 'foo_12345'", { - "string_v": "foo_12345" - }), ("SELECT %(x)s", "SELECT NULL", { - "x": None - }), ("SELECT %(int_value)d", "SELECT 48", { - "int_value": 48 - }), ("SELECT %(float_value).2f", "SELECT 48.20", { - "float_value": 48.2 - }), ("SELECT %(iter)s", "SELECT (1,2,3,4,5)", { - "iter": [1, 2, 3, 4, 5] - }), - ("SELECT %(datetime)s", "SELECT '2022-02-01 10:23:00.000000'", { - "datetime": datetime(2022, 2, 1, 10, 23) - }), ("SELECT %(date)s", "SELECT '2022-02-01'", { - "date": date(2022, 2, 1) - })] + tests = [ + ("SELECT %(string_v)s", "SELECT 'foo_12345'", {"string_v": "foo_12345"}), + ("SELECT %(x)s", "SELECT NULL", {"x": None}), + ("SELECT %(int_value)d", "SELECT 48", {"int_value": 48}), + ("SELECT %(float_value).2f", "SELECT 48.20", {"float_value": 48.2}), + ("SELECT %(iter)s", "SELECT (1,2,3,4,5)", {"iter": [1, 2, 3, 4, 5]}), + ( + "SELECT %(datetime)s", + "SELECT '2022-02-01 10:23:00.000000'", + {"datetime": datetime(2022, 2, 1, 10, 23)}, + ), + ("SELECT %(date)s", "SELECT '2022-02-01'", {"date": date(2022, 2, 1)}), + ] for query, expected_query, params in tests: cursor.execute(query, parameters=params) - self.assertEqual(mock_thrift_backend.execute_command.call_args[1]["operation"], - expected_query) + self.assertEqual( + mock_thrift_backend.execute_command.call_args[1]["operation"], + expected_query, + ) + @patch("%s.client.ThriftBackend" % PACKAGE_NAME) @patch("%s.client.ResultSet" % PACKAGE_NAME) def test_executemany_parameter_passhthrough_and_uses_last_result_set( - self, mock_result_set_class): + self, mock_result_set_class, mock_thrift_backend + ): # Create a new mock result set each time the class is instantiated mock_result_set_instances = [Mock(), Mock(), Mock()] mock_result_set_class.side_effect = mock_result_set_instances - mock_thrift_backend = Mock() - cursor = client.Cursor(Mock(), mock_thrift_backend) + mock_thrift_backend = ThriftBackendMockFactory.new() + cursor = client.Cursor(Mock(), mock_thrift_backend()) params = [{"x": None}, {"x": "foo1"}, {"x": "bar2"}] expected_queries = ["SELECT NULL", "SELECT 'foo1'", "SELECT 'bar2'"] @@ -403,17 +488,22 @@ def test_executemany_parameter_passhthrough_and_uses_last_result_set( cursor.executemany("SELECT %(x)s", seq_of_parameters=params) self.assertEqual( - len(mock_thrift_backend.execute_command.call_args_list), len(expected_queries), - "Expected execute_command to be called the same number of times as params were passed") + len(mock_thrift_backend.execute_command.call_args_list), + len(expected_queries), + "Expected execute_command to be called the same number of times as params were passed", + ) - for expected_query, call_args in zip(expected_queries, - mock_thrift_backend.execute_command.call_args_list): + for expected_query, call_args in zip( + expected_queries, mock_thrift_backend.execute_command.call_args_list + ): self.assertEqual(call_args[1]["operation"], expected_query) self.assertEqual( - cursor.active_result_set, mock_result_set_instances[2], + cursor.active_result_set, + mock_result_set_instances[2], "Expected the active result set to be the result set corresponding to the" - "last operation") + "last operation", + ) @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_commit_a_noop(self, mock_thrift_backend_class): @@ -434,6 +524,7 @@ def test_rollback_not_supported(self, mock_thrift_backend_class): with self.assertRaises(NotSupportedError): c.rollback() + @unittest.skip("JDW: skipping winter 2024 as we're about to rewrite this interface") @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_row_number_respected(self, mock_thrift_backend_class): def make_fake_row_slice(n_rows): @@ -448,7 +539,7 @@ def make_fake_row_slice(n_rows): mock_thrift_backend.fetch_results.return_value = (mock_aq, True) cursor = client.Cursor(Mock(), mock_thrift_backend) - cursor.execute('foo') + cursor.execute("foo") self.assertEqual(cursor.rownumber, 0) cursor.fetchmany_arrow(10) @@ -458,6 +549,7 @@ def make_fake_row_slice(n_rows): cursor.fetchmany_arrow(6) self.assertEqual(cursor.rownumber, 29) + @unittest.skip("JDW: skipping winter 2024 as we're about to rewrite this interface") @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_disable_pandas_respected(self, mock_thrift_backend_class): mock_thrift_backend = mock_thrift_backend_class.return_value @@ -468,12 +560,14 @@ def test_disable_pandas_respected(self, mock_thrift_backend_class): mock_aq = Mock() mock_aq.remaining_rows.return_value = mock_table mock_thrift_backend.execute_command.return_value.arrow_queue = mock_aq - mock_thrift_backend.execute_command.return_value.has_been_closed_server_side = True + mock_thrift_backend.execute_command.return_value.has_been_closed_server_side = ( + True + ) mock_con = Mock() mock_con.disable_pandas = True cursor = client.Cursor(mock_con, mock_thrift_backend) - cursor.execute('foo') + cursor.execute("foo") cursor.fetchall() mock_table.itercolumns.assert_called_once_with() @@ -500,16 +594,22 @@ def test_column_name_api(self): self.assertEqual(row[1], expected[1]) self.assertEqual(row[2], expected[2]) - self.assertEqual(row.asDict(), { - "first_col": expected[0], - "second_col": expected[1], - "third_col": expected[2] - }) + self.assertEqual( + row.asDict(), + { + "first_col": expected[0], + "second_col": expected[1], + "third_col": expected[2], + }, + ) @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_finalizer_closes_abandoned_connection(self, mock_client_class): instance = mock_client_class.return_value - instance.open_session.return_value = b'\x22' + + mock_open_session_resp = MagicMock(spec=TOpenSessionResp)() + mock_open_session_resp.sessionHandle.sessionId = b"\x22" + instance.open_session.return_value = mock_open_session_resp databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) @@ -517,13 +617,16 @@ def test_finalizer_closes_abandoned_connection(self, mock_client_class): gc.collect() # Check the close session request has an id of x22 - close_session_id = instance.close_session.call_args[0][0] - self.assertEqual(close_session_id, b'\x22') + close_session_id = instance.close_session.call_args[0][0].sessionId + self.assertEqual(close_session_id, b"\x22") @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_cursor_keeps_connection_alive(self, mock_client_class): instance = mock_client_class.return_value - instance.open_session.return_value = b'\x22' + + mock_open_session_resp = MagicMock(spec=TOpenSessionResp)() + mock_open_session_resp.sessionHandle.sessionId = b"\x22" + instance.open_session.return_value = mock_open_session_resp connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) cursor = connection.cursor() @@ -534,26 +637,55 @@ def test_cursor_keeps_connection_alive(self, mock_client_class): self.assertEqual(instance.close_session.call_count, 0) cursor.close() - @patch("%s.client.ThriftBackend" % PACKAGE_NAME) + @patch("%s.utils.ExecuteResponse" % PACKAGE_NAME, autospec=True) @patch("%s.client.Cursor._handle_staging_operation" % PACKAGE_NAME) - @patch("%s.utils.ExecuteResponse" % PACKAGE_NAME) - def test_staging_operation_response_is_handled(self, mock_client_class, mock_handle_staging_operation, mock_execute_response): + @patch("%s.client.ThriftBackend" % PACKAGE_NAME) + def test_staging_operation_response_is_handled( + self, mock_client_class, mock_handle_staging_operation, mock_execute_response + ): # If server sets ExecuteResponse.is_staging_operation True then _handle_staging_operation should be called - mock_execute_response.is_staging_operation = True - + ThriftBackendMockFactory.apply_property_to_mock( + mock_execute_response, is_staging_operation=True + ) + mock_client_class.execute_command.return_value = mock_execute_response + mock_client_class.return_value = mock_client_class + connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) cursor = connection.cursor() cursor.execute("Text of some staging operation command;") connection.close() - mock_handle_staging_operation.assert_called_once_with() + mock_handle_staging_operation.call_count == 1 + + @patch("%s.client.ThriftBackend" % PACKAGE_NAME, ThriftBackendMockFactory.new()) + def test_access_current_query_id(self): + operation_id = "EE6A8778-21FC-438B-92D8-96AC51EE3821" + + connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) + cursor = connection.cursor() + + self.assertIsNone(cursor.query_id) + + cursor.active_op_handle = TOperationHandle( + operationId=THandleIdentifier(guid=UUID(operation_id).bytes, secret=0x00), + operationType=TOperationType.EXECUTE_STATEMENT, + ) + self.assertEqual(cursor.query_id.upper(), operation_id.upper()) + + cursor.close() + self.assertIsNone(cursor.query_id) -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) loader = unittest.TestLoader() - test_classes = [ClientTestSuite, FetchTests, ThriftBackendTestSuite, ArrowQueueSuite] + test_classes = [ + ClientTestSuite, + FetchTests, + ThriftBackendTestSuite, + ArrowQueueSuite, + ] suites_list = [] for test_class in test_classes: suite = loader.loadTestsFromTestCase(test_class) diff --git a/tests/unit/test_cloud_fetch_queue.py b/tests/unit/test_cloud_fetch_queue.py new file mode 100644 index 00000000..7dec4e68 --- /dev/null +++ b/tests/unit/test_cloud_fetch_queue.py @@ -0,0 +1,335 @@ +try: + import pyarrow +except ImportError: + pyarrow = None +import unittest +import pytest +from unittest.mock import MagicMock, patch + +from databricks.sql.thrift_api.TCLIService.ttypes import TSparkArrowResultLink +import databricks.sql.utils as utils +from databricks.sql.types import SSLOptions + + +@pytest.mark.skipif(pyarrow is None, reason="PyArrow is not installed") +class CloudFetchQueueSuite(unittest.TestCase): + def create_result_link( + self, + file_link: str = "fileLink", + start_row_offset: int = 0, + row_count: int = 8000, + bytes_num: int = 20971520, + ): + return TSparkArrowResultLink( + file_link, None, start_row_offset, row_count, bytes_num + ) + + def create_result_links(self, num_files: int, start_row_offset: int = 0): + result_links = [] + for i in range(num_files): + file_link = "fileLink_" + str(i) + result_link = self.create_result_link( + file_link=file_link, start_row_offset=start_row_offset + ) + result_links.append(result_link) + start_row_offset += result_link.rowCount + return result_links + + @staticmethod + def make_arrow_table(): + batch = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + n_cols = len(batch[0]) if batch else 0 + schema = pyarrow.schema({"col%s" % i: pyarrow.uint32() for i in range(n_cols)}) + cols = [[batch[row][col] for row in range(len(batch))] for col in range(n_cols)] + return pyarrow.Table.from_pydict(dict(zip(schema.names, cols)), schema=schema) + + @staticmethod + def get_schema_bytes(): + schema = pyarrow.schema({"col%s" % i: pyarrow.uint32() for i in range(4)}) + sink = pyarrow.BufferOutputStream() + writer = pyarrow.ipc.RecordBatchStreamWriter(sink, schema) + writer.close() + return sink.getvalue().to_pybytes() + + @patch( + "databricks.sql.utils.CloudFetchQueue._create_next_table", + return_value=[None, None], + ) + def test_initializer_adds_links(self, mock_create_next_table): + schema_bytes = MagicMock() + result_links = self.create_result_links(10) + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=result_links, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + + assert len(queue.download_manager._pending_links) == 10 + assert len(queue.download_manager._download_tasks) == 0 + mock_create_next_table.assert_called() + + def test_initializer_no_links_to_add(self): + schema_bytes = MagicMock() + result_links = [] + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=result_links, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + + assert len(queue.download_manager._pending_links) == 0 + assert len(queue.download_manager._download_tasks) == 0 + assert queue.table is None + + @patch( + "databricks.sql.cloudfetch.download_manager.ResultFileDownloadManager.get_next_downloaded_file", + return_value=None, + ) + def test_create_next_table_no_download(self, mock_get_next_downloaded_file): + queue = utils.CloudFetchQueue( + MagicMock(), + result_links=[], + max_download_threads=10, + ssl_options=SSLOptions(), + ) + + assert queue._create_next_table() is None + mock_get_next_downloaded_file.assert_called_with(0) + + @patch("databricks.sql.utils.create_arrow_table_from_arrow_file") + @patch( + "databricks.sql.cloudfetch.download_manager.ResultFileDownloadManager.get_next_downloaded_file", + return_value=MagicMock(file_bytes=b"1234567890", row_count=4), + ) + def test_initializer_create_next_table_success( + self, mock_get_next_downloaded_file, mock_create_arrow_table + ): + mock_create_arrow_table.return_value = self.make_arrow_table() + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + expected_result = self.make_arrow_table() + + mock_get_next_downloaded_file.assert_called_with(0) + mock_create_arrow_table.assert_called_with(b"1234567890", description) + assert queue.table == expected_result + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + assert queue.start_row_index == 4 + + table = queue._create_next_table() + assert table == expected_result + assert table.num_rows == 4 + assert queue.start_row_index == 8 + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_next_n_rows_0_rows(self, mock_create_next_table): + mock_create_next_table.return_value = self.make_arrow_table() + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.next_n_rows(0) + assert result.num_rows == 0 + assert queue.table_row_index == 0 + assert result == self.make_arrow_table()[0:0] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_next_n_rows_partial_table(self, mock_create_next_table): + mock_create_next_table.return_value = self.make_arrow_table() + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.next_n_rows(3) + assert result.num_rows == 3 + assert queue.table_row_index == 3 + assert result == self.make_arrow_table()[:3] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_next_n_rows_more_than_one_table(self, mock_create_next_table): + mock_create_next_table.return_value = self.make_arrow_table() + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.next_n_rows(7) + assert result.num_rows == 7 + assert queue.table_row_index == 3 + assert ( + result + == pyarrow.concat_tables( + [self.make_arrow_table(), self.make_arrow_table()] + )[:7] + ) + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_next_n_rows_only_one_table_returned(self, mock_create_next_table): + mock_create_next_table.side_effect = [self.make_arrow_table(), None] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.next_n_rows(7) + assert result.num_rows == 4 + assert result == self.make_arrow_table() + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table", return_value=None) + def test_next_n_rows_empty_table(self, mock_create_next_table): + schema_bytes = self.get_schema_bytes() + description = MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table is None + + result = queue.next_n_rows(100) + mock_create_next_table.assert_called() + assert result == pyarrow.ipc.open_stream(bytearray(schema_bytes)).read_all() + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_remaining_rows_empty_table_fully_returned(self, mock_create_next_table): + mock_create_next_table.side_effect = [self.make_arrow_table(), None, 0] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + queue.table_row_index = 4 + + result = queue.remaining_rows() + assert result.num_rows == 0 + assert result == self.make_arrow_table()[0:0] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_remaining_rows_partial_table_fully_returned(self, mock_create_next_table): + mock_create_next_table.side_effect = [self.make_arrow_table(), None] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + queue.table_row_index = 2 + + result = queue.remaining_rows() + assert result.num_rows == 2 + assert result == self.make_arrow_table()[2:] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_remaining_rows_one_table_fully_returned(self, mock_create_next_table): + mock_create_next_table.side_effect = [self.make_arrow_table(), None] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.remaining_rows() + assert result.num_rows == 4 + assert result == self.make_arrow_table() + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_remaining_rows_multiple_tables_fully_returned( + self, mock_create_next_table + ): + mock_create_next_table.side_effect = [ + self.make_arrow_table(), + self.make_arrow_table(), + None, + ] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + queue.table_row_index = 3 + + result = queue.remaining_rows() + assert mock_create_next_table.call_count == 3 + assert result.num_rows == 5 + assert ( + result + == pyarrow.concat_tables( + [self.make_arrow_table(), self.make_arrow_table()] + )[3:] + ) + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table", return_value=None) + def test_remaining_rows_empty_table(self, mock_create_next_table): + schema_bytes = self.get_schema_bytes() + description = MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table is None + + result = queue.remaining_rows() + assert result == pyarrow.ipc.open_stream(bytearray(schema_bytes)).read_all() diff --git a/tests/unit/test_column_queue.py b/tests/unit/test_column_queue.py new file mode 100644 index 00000000..234af88e --- /dev/null +++ b/tests/unit/test_column_queue.py @@ -0,0 +1,26 @@ +from databricks.sql.utils import ColumnQueue, ColumnTable + + +class TestColumnQueueSuite: + @staticmethod + def make_column_table(table): + n_cols = len(table) if table else 0 + return ColumnTable(table, [f"col_{i}" for i in range(n_cols)]) + + def test_fetchmany_respects_n_rows(self): + column_table = self.make_column_table( + [[0, 3, 6, 9], [1, 4, 7, 10], [2, 5, 8, 11]] + ) + column_queue = ColumnQueue(column_table) + + assert column_queue.next_n_rows(2) == column_table.slice(0, 2) + assert column_queue.next_n_rows(2) == column_table.slice(2, 2) + + def test_fetch_remaining_rows_respects_n_rows(self): + column_table = self.make_column_table( + [[0, 3, 6, 9], [1, 4, 7, 10], [2, 5, 8, 11]] + ) + column_queue = ColumnQueue(column_table) + + assert column_queue.next_n_rows(2) == column_table.slice(0, 2) + assert column_queue.remaining_rows() == column_table.slice(2, 2) diff --git a/tests/unit/test_download_manager.py b/tests/unit/test_download_manager.py new file mode 100644 index 00000000..64edbdeb --- /dev/null +++ b/tests/unit/test_download_manager.py @@ -0,0 +1,73 @@ +import unittest +from unittest.mock import patch, MagicMock + +import databricks.sql.cloudfetch.download_manager as download_manager +from databricks.sql.types import SSLOptions +from databricks.sql.thrift_api.TCLIService.ttypes import TSparkArrowResultLink + + +class DownloadManagerTests(unittest.TestCase): + """ + Unit tests for checking download manager logic. + """ + + def create_download_manager( + self, links, max_download_threads=10, lz4_compressed=True + ): + return download_manager.ResultFileDownloadManager( + links, + max_download_threads, + lz4_compressed, + ssl_options=SSLOptions(), + ) + + def create_result_link( + self, + file_link: str = "fileLink", + start_row_offset: int = 0, + row_count: int = 8000, + bytes_num: int = 20971520, + ): + return TSparkArrowResultLink( + file_link, None, start_row_offset, row_count, bytes_num + ) + + def create_result_links(self, num_files: int, start_row_offset: int = 0): + result_links = [] + for i in range(num_files): + file_link = "fileLink_" + str(i) + result_link = self.create_result_link( + file_link=file_link, start_row_offset=start_row_offset + ) + result_links.append(result_link) + start_row_offset += result_link.rowCount + return result_links + + def test_add_file_links_zero_row_count(self): + links = [self.create_result_link(row_count=0, bytes_num=0)] + manager = self.create_download_manager(links) + + assert ( + len(manager._pending_links) == 0 + ) # the only link supplied contains no data, so should be skipped + assert len(manager._download_tasks) == 0 + + def test_add_file_links_success(self): + links = self.create_result_links(num_files=10) + manager = self.create_download_manager(links) + + assert len(manager._pending_links) == len(links) + assert len(manager._download_tasks) == 0 + + @patch("concurrent.futures.ThreadPoolExecutor.submit") + def test_schedule_downloads(self, mock_submit): + max_download_threads = 4 + links = self.create_result_links(num_files=10) + manager = self.create_download_manager( + links, max_download_threads=max_download_threads + ) + + manager._schedule_downloads() + assert mock_submit.call_count == max_download_threads + assert len(manager._pending_links) == len(links) - max_download_threads + assert len(manager._download_tasks) == max_download_threads diff --git a/tests/unit/test_downloader.py b/tests/unit/test_downloader.py new file mode 100644 index 00000000..2a3b715b --- /dev/null +++ b/tests/unit/test_downloader.py @@ -0,0 +1,142 @@ +import unittest +from unittest.mock import Mock, patch, MagicMock + +import requests + +import databricks.sql.cloudfetch.downloader as downloader +from databricks.sql.exc import Error +from databricks.sql.types import SSLOptions + + +def create_response(**kwargs) -> requests.Response: + result = requests.Response() + for k, v in kwargs.items(): + setattr(result, k, v) + return result + + +class DownloaderTests(unittest.TestCase): + """ + Unit tests for checking downloader logic. + """ + + @patch("time.time", return_value=1000) + def test_run_link_expired(self, mock_time): + settings = Mock() + result_link = Mock() + # Already expired + result_link.expiryTime = 999 + d = downloader.ResultSetDownloadHandler( + settings, result_link, ssl_options=SSLOptions() + ) + + with self.assertRaises(Error) as context: + d.run() + self.assertTrue("link has expired" in context.exception.message) + + mock_time.assert_called_once() + + @patch("time.time", return_value=1000) + def test_run_link_past_expiry_buffer(self, mock_time): + settings = Mock(link_expiry_buffer_secs=5) + result_link = Mock() + # Within the expiry buffer time + result_link.expiryTime = 1004 + d = downloader.ResultSetDownloadHandler( + settings, result_link, ssl_options=SSLOptions() + ) + + with self.assertRaises(Error) as context: + d.run() + self.assertTrue("link has expired" in context.exception.message) + + mock_time.assert_called_once() + + @patch("requests.Session", return_value=MagicMock(get=MagicMock(return_value=None))) + @patch("time.time", return_value=1000) + def test_run_get_response_not_ok(self, mock_time, mock_session): + mock_session.return_value.get.return_value = create_response(status_code=404) + + settings = Mock(link_expiry_buffer_secs=0, download_timeout=0) + settings.download_timeout = 0 + settings.use_proxy = False + result_link = Mock(expiryTime=1001) + + d = downloader.ResultSetDownloadHandler( + settings, result_link, ssl_options=SSLOptions() + ) + with self.assertRaises(requests.exceptions.HTTPError) as context: + d.run() + self.assertTrue("404" in str(context.exception)) + + @patch("requests.Session", return_value=MagicMock(get=MagicMock(return_value=None))) + @patch("time.time", return_value=1000) + def test_run_uncompressed_successful(self, mock_time, mock_session): + file_bytes = b"1234567890" * 10 + mock_session.return_value.get.return_value = create_response( + status_code=200, _content=file_bytes + ) + + settings = Mock(link_expiry_buffer_secs=0, download_timeout=0, use_proxy=False) + settings.is_lz4_compressed = False + result_link = Mock(bytesNum=100, expiryTime=1001) + + d = downloader.ResultSetDownloadHandler( + settings, result_link, ssl_options=SSLOptions() + ) + file = d.run() + + assert file.file_bytes == b"1234567890" * 10 + + @patch( + "requests.Session", + return_value=MagicMock(get=MagicMock(return_value=MagicMock(ok=True))), + ) + @patch("time.time", return_value=1000) + def test_run_compressed_successful(self, mock_time, mock_session): + file_bytes = b"1234567890" * 10 + compressed_bytes = b'\x04"M\x18h@d\x00\x00\x00\x00\x00\x00\x00#\x14\x00\x00\x00\xaf1234567890\n\x00BP67890\x00\x00\x00\x00' + mock_session.return_value.get.return_value = create_response( + status_code=200, _content=compressed_bytes + ) + + settings = Mock(link_expiry_buffer_secs=0, download_timeout=0, use_proxy=False) + settings.is_lz4_compressed = True + result_link = Mock(bytesNum=100, expiryTime=1001) + + d = downloader.ResultSetDownloadHandler( + settings, result_link, ssl_options=SSLOptions() + ) + file = d.run() + + assert file.file_bytes == b"1234567890" * 10 + + @patch("requests.Session.get", side_effect=ConnectionError("foo")) + @patch("time.time", return_value=1000) + def test_download_connection_error(self, mock_time, mock_session): + settings = Mock( + link_expiry_buffer_secs=0, use_proxy=False, is_lz4_compressed=True + ) + result_link = Mock(bytesNum=100, expiryTime=1001) + mock_session.return_value.get.return_value.content = b'\x04"M\x18h@d\x00\x00\x00\x00\x00\x00\x00#\x14\x00\x00\x00\xaf1234567890\n\x00BP67890\x00\x00\x00\x00' + + d = downloader.ResultSetDownloadHandler( + settings, result_link, ssl_options=SSLOptions() + ) + with self.assertRaises(ConnectionError): + d.run() + + @patch("requests.Session.get", side_effect=TimeoutError("foo")) + @patch("time.time", return_value=1000) + def test_download_timeout(self, mock_time, mock_session): + settings = Mock( + link_expiry_buffer_secs=0, use_proxy=False, is_lz4_compressed=True + ) + result_link = Mock(bytesNum=100, expiryTime=1001) + mock_session.return_value.get.return_value.content = b'\x04"M\x18h@d\x00\x00\x00\x00\x00\x00\x00#\x14\x00\x00\x00\xaf1234567890\n\x00BP67890\x00\x00\x00\x00' + + d = downloader.ResultSetDownloadHandler( + settings, result_link, ssl_options=SSLOptions() + ) + with self.assertRaises(TimeoutError): + d.run() diff --git a/tests/unit/test_endpoint.py b/tests/unit/test_endpoint.py new file mode 100644 index 00000000..1f7d7cdd --- /dev/null +++ b/tests/unit/test_endpoint.py @@ -0,0 +1,124 @@ +import unittest +import os +import pytest + +from unittest.mock import patch + +from databricks.sql.auth.auth import AuthType +from databricks.sql.auth.endpoint import ( + infer_cloud_from_host, + CloudType, + get_oauth_endpoints, + AzureOAuthEndpointCollection, +) + +aws_host = "foo-bar.cloud.databricks.com" +azure_host = "foo-bar.1.azuredatabricks.net" +azure_cn_host = "foo-bar2.databricks.azure.cn" +gcp_host = "foo.1.gcp.databricks.com" + + +class EndpointTest(unittest.TestCase): + def test_infer_cloud_from_host(self): + param_list = [ + (CloudType.AWS, aws_host), + (CloudType.AZURE, azure_host), + (None, "foo.example.com"), + ] + + for expected_type, host in param_list: + with self.subTest(expected_type or "None", expected_type=expected_type): + self.assertEqual(infer_cloud_from_host(host), expected_type) + self.assertEqual( + infer_cloud_from_host(f"https://{host}/to/path"), expected_type + ) + + def test_oauth_endpoint(self): + scopes = ["offline_access", "sql", "admin"] + scopes2 = ["sql", "admin"] + azure_scope = ( + f"{AzureOAuthEndpointCollection.DATATRICKS_AZURE_APP}/user_impersonation" + ) + + param_list = [ + ( + CloudType.AWS, + aws_host, + False, + f"https://{aws_host}/oidc/oauth2/v2.0/authorize", + f"https://{aws_host}/oidc/.well-known/oauth-authorization-server", + scopes, + scopes2, + ), + ( + CloudType.AZURE, + azure_cn_host, + False, + f"https://{azure_cn_host}/oidc/oauth2/v2.0/authorize", + "https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration", + [azure_scope, "offline_access"], + [azure_scope], + ), + ( + CloudType.AZURE, + azure_host, + True, + f"https://{azure_host}/oidc/oauth2/v2.0/authorize", + "https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration", + [azure_scope, "offline_access"], + [azure_scope], + ), + ( + CloudType.AZURE, + azure_host, + False, + f"https://{azure_host}/oidc/oauth2/v2.0/authorize", + f"https://{azure_host}/oidc/.well-known/oauth-authorization-server", + scopes, + scopes2, + ), + ( + CloudType.GCP, + gcp_host, + False, + f"https://{gcp_host}/oidc/oauth2/v2.0/authorize", + f"https://{gcp_host}/oidc/.well-known/oauth-authorization-server", + scopes, + scopes2, + ), + ] + + for ( + cloud_type, + host, + use_azure_auth, + expected_auth_url, + expected_config_url, + expected_scopes, + expected_scope2, + ) in param_list: + with self.subTest(cloud_type): + endpoint = get_oauth_endpoints(host, use_azure_auth) + self.assertEqual( + endpoint.get_authorization_url(host), expected_auth_url + ) + self.assertEqual( + endpoint.get_openid_config_url(host), expected_config_url + ) + self.assertEqual(endpoint.get_scopes_mapping(scopes), expected_scopes) + self.assertEqual(endpoint.get_scopes_mapping(scopes2), expected_scope2) + + @patch.dict( + os.environ, + {"DATABRICKS_AZURE_TENANT_ID": "052ee82f-b79d-443c-8682-3ec1749e56b0"}, + ) + def test_azure_oauth_scope_mappings_from_different_tenant_id(self): + scopes = ["offline_access", "sql", "all"] + endpoint = get_oauth_endpoints(azure_host, True) + self.assertEqual( + endpoint.get_scopes_mapping(scopes), + [ + "052ee82f-b79d-443c-8682-3ec1749e56b0/user_impersonation", + "offline_access", + ], + ) diff --git a/tests/unit/test_fetches.py b/tests/unit/test_fetches.py index 7d5686f8..71766f2c 100644 --- a/tests/unit/test_fetches.py +++ b/tests/unit/test_fetches.py @@ -1,12 +1,17 @@ import unittest +import pytest from unittest.mock import Mock -import pyarrow as pa +try: + import pyarrow as pa +except ImportError: + pa = None import databricks.sql.client as client from databricks.sql.utils import ExecuteResponse, ArrowQueue +@pytest.mark.skipif(pa is None, reason="PyArrow is not installed") class FetchTests(unittest.TestCase): """ Unit tests for checking the fetch logic. @@ -17,7 +22,9 @@ def make_arrow_table(batch): n_cols = len(batch[0]) if batch else 0 schema = pa.schema({"col%s" % i: pa.uint32() for i in range(n_cols)}) cols = [[batch[row][col] for row in range(len(batch))] for col in range(n_cols)] - return schema, pa.Table.from_pydict(dict(zip(schema.names, cols)), schema=schema) + return schema, pa.Table.from_pydict( + dict(zip(schema.names, cols)), schema=schema + ) @staticmethod def make_arrow_queue(batch): @@ -42,18 +49,30 @@ def make_dummy_result_set_from_initial_results(initial_results): command_handle=None, arrow_queue=arrow_queue, arrow_schema_bytes=schema.serialize().to_pybytes(), - is_staging_operation=False)) + is_staging_operation=False, + ), + ) num_cols = len(initial_results[0]) if initial_results else 0 - rs.description = [(f'col{col_id}', 'integer', None, None, None, None, None) - for col_id in range(num_cols)] + rs.description = [ + (f"col{col_id}", "integer", None, None, None, None, None) + for col_id in range(num_cols) + ] return rs @staticmethod def make_dummy_result_set_from_batch_list(batch_list): batch_index = 0 - def fetch_results(op_handle, max_rows, max_bytes, expected_row_start_offset, lz4_compressed, - arrow_schema_bytes, description): + def fetch_results( + op_handle, + max_rows, + max_bytes, + expected_row_start_offset, + lz4_compressed, + arrow_schema_bytes, + description, + use_cloud_fetch=True, + ): nonlocal batch_index results = FetchTests.make_arrow_queue(batch_list[batch_index]) batch_index += 1 @@ -71,13 +90,17 @@ def fetch_results(op_handle, max_rows, max_bytes, expected_row_start_offset, lz4 status=None, has_been_closed_server_side=False, has_more_rows=True, - description=[(f'col{col_id}', 'integer', None, None, None, None, None) - for col_id in range(num_cols)], + description=[ + (f"col{col_id}", "integer", None, None, None, None, None) + for col_id in range(num_cols) + ], lz4_compressed=Mock(), command_handle=None, arrow_queue=None, arrow_schema_bytes=None, - is_staging_operation=False)) + is_staging_operation=False, + ), + ) return rs def assertEqualRowValues(self, actual, expected): @@ -87,30 +110,44 @@ def assertEqualRowValues(self, actual, expected): def test_fetchmany_with_initial_results(self): # Fetch all in one go - initial_results_1 = [[1], [2], [3]] # This is a list of rows, each row with 1 col - dummy_result_set = self.make_dummy_result_set_from_initial_results(initial_results_1) + initial_results_1 = [ + [1], + [2], + [3], + ] # This is a list of rows, each row with 1 col + dummy_result_set = self.make_dummy_result_set_from_initial_results( + initial_results_1 + ) self.assertEqualRowValues(dummy_result_set.fetchmany(3), [[1], [2], [3]]) # Fetch in small amounts initial_results_2 = [[1], [2], [3], [4]] - dummy_result_set = self.make_dummy_result_set_from_initial_results(initial_results_2) + dummy_result_set = self.make_dummy_result_set_from_initial_results( + initial_results_2 + ) self.assertEqualRowValues(dummy_result_set.fetchmany(1), [[1]]) self.assertEqualRowValues(dummy_result_set.fetchmany(2), [[2], [3]]) self.assertEqualRowValues(dummy_result_set.fetchmany(1), [[4]]) # Fetch too many initial_results_3 = [[2], [3]] - dummy_result_set = self.make_dummy_result_set_from_initial_results(initial_results_3) + dummy_result_set = self.make_dummy_result_set_from_initial_results( + initial_results_3 + ) self.assertEqualRowValues(dummy_result_set.fetchmany(5), [[2], [3]]) # Empty results initial_results_4 = [[]] - dummy_result_set = self.make_dummy_result_set_from_initial_results(initial_results_4) + dummy_result_set = self.make_dummy_result_set_from_initial_results( + initial_results_4 + ) self.assertEqualRowValues(dummy_result_set.fetchmany(0), []) def test_fetch_many_without_initial_results(self): # Fetch all in one go; single batch - batch_list_1 = [[[1], [2], [3]]] # This is a list of one batch of rows, each row with 1 col + batch_list_1 = [ + [[1], [2], [3]] + ] # This is a list of one batch of rows, each row with 1 col dummy_result_set = self.make_dummy_result_set_from_batch_list(batch_list_1) self.assertEqualRowValues(dummy_result_set.fetchmany(3), [[1], [2], [3]]) @@ -140,7 +177,9 @@ def test_fetch_many_without_initial_results(self): # Fetch too many; multiple batches batch_list_6 = [[[1]], [[2], [3], [4]], [[5], [6]]] dummy_result_set = self.make_dummy_result_set_from_batch_list(batch_list_6) - self.assertEqualRowValues(dummy_result_set.fetchmany(100), [[1], [2], [3], [4], [5], [6]]) + self.assertEqualRowValues( + dummy_result_set.fetchmany(100), [[1], [2], [3], [4], [5], [6]] + ) # Fetch 0; 1 empty batch batch_list_7 = [[]] @@ -154,19 +193,25 @@ def test_fetch_many_without_initial_results(self): def test_fetchall_with_initial_results(self): initial_results_1 = [[1], [2], [3]] - dummy_result_set = self.make_dummy_result_set_from_initial_results(initial_results_1) + dummy_result_set = self.make_dummy_result_set_from_initial_results( + initial_results_1 + ) self.assertEqualRowValues(dummy_result_set.fetchall(), [[1], [2], [3]]) def test_fetchall_without_initial_results(self): # Fetch all, single batch - batch_list_1 = [[[1], [2], [3]]] # This is a list of one batch of rows, each row with 1 col + batch_list_1 = [ + [[1], [2], [3]] + ] # This is a list of one batch of rows, each row with 1 col dummy_result_set = self.make_dummy_result_set_from_batch_list(batch_list_1) self.assertEqualRowValues(dummy_result_set.fetchall(), [[1], [2], [3]]) # Fetch all, multiple batches batch_list_2 = [[[1], [2]], [[3]], [[4], [5], [6]]] dummy_result_set = self.make_dummy_result_set_from_batch_list(batch_list_2) - self.assertEqualRowValues(dummy_result_set.fetchall(), [[1], [2], [3], [4], [5], [6]]) + self.assertEqualRowValues( + dummy_result_set.fetchall(), [[1], [2], [3], [4], [5], [6]] + ) batch_list_3 = [[]] dummy_result_set = self.make_dummy_result_set_from_batch_list(batch_list_3) @@ -174,12 +219,16 @@ def test_fetchall_without_initial_results(self): def test_fetchmany_fetchall_with_initial_results(self): initial_results_1 = [[1], [2], [3]] - dummy_result_set = self.make_dummy_result_set_from_initial_results(initial_results_1) + dummy_result_set = self.make_dummy_result_set_from_initial_results( + initial_results_1 + ) self.assertEqualRowValues(dummy_result_set.fetchmany(2), [[1], [2]]) self.assertEqualRowValues(dummy_result_set.fetchall(), [[3]]) def test_fetchmany_fetchall_without_initial_results(self): - batch_list_1 = [[[1], [2], [3]]] # This is a list of one batch of rows, each row with 1 col + batch_list_1 = [ + [[1], [2], [3]] + ] # This is a list of one batch of rows, each row with 1 col dummy_result_set = self.make_dummy_result_set_from_batch_list(batch_list_1) self.assertEqualRowValues(dummy_result_set.fetchmany(2), [[1], [2]]) self.assertEqualRowValues(dummy_result_set.fetchall(), [[3]]) @@ -191,7 +240,9 @@ def test_fetchmany_fetchall_without_initial_results(self): def test_fetchone_with_initial_results(self): initial_results_1 = [[1], [2], [3]] - dummy_result_set = self.make_dummy_result_set_from_initial_results(initial_results_1) + dummy_result_set = self.make_dummy_result_set_from_initial_results( + initial_results_1 + ) self.assertSequenceEqual(dummy_result_set.fetchone(), [1]) self.assertSequenceEqual(dummy_result_set.fetchone(), [2]) self.assertSequenceEqual(dummy_result_set.fetchone(), [3]) @@ -210,5 +261,5 @@ def test_fetchone_without_initial_results(self): self.assertEqual(dummy_result_set.fetchone(), None) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_fetches_bench.py b/tests/unit/test_fetches_bench.py index e322b44a..55287222 100644 --- a/tests/unit/test_fetches_bench.py +++ b/tests/unit/test_fetches_bench.py @@ -1,7 +1,10 @@ import unittest from unittest.mock import Mock -import pyarrow as pa +try: + import pyarrow as pa +except ImportError: + pa = None import uuid import time import pytest @@ -10,6 +13,7 @@ from databricks.sql.utils import ExecuteResponse, ArrowQueue +@pytest.mark.skipif(pa is None, reason="PyArrow is not installed") class FetchBenchmarkTests(unittest.TestCase): """ Micro benchmark test for Arrow result handling. @@ -35,12 +39,18 @@ def make_dummy_result_set_from_initial_results(arrow_table): description=Mock(), command_handle=None, arrow_queue=arrow_queue, - arrow_schema=arrow_table.schema)) - rs.description = [(f'col{col_id}', 'string', None, None, None, None, None) - for col_id in range(arrow_table.num_columns)] + arrow_schema=arrow_table.schema, + ), + ) + rs.description = [ + (f"col{col_id}", "string", None, None, None, None, None) + for col_id in range(arrow_table.num_columns) + ] return rs - @pytest.mark.skip(reason="Test has not been updated for latest connector API (June 2022)") + @pytest.mark.skip( + reason="Test has not been updated for latest connector API (June 2022)" + ) def test_benchmark_fetchall(self): print("preparing dummy arrow table") arrow_table = FetchBenchmarkTests.make_arrow_table(10, 25000) @@ -50,7 +60,9 @@ def test_benchmark_fetchall(self): start_time = time.time() count = 0 while time.time() < start_time + benchmark_seconds: - dummy_result_set = self.make_dummy_result_set_from_initial_results(arrow_table) + dummy_result_set = self.make_dummy_result_set_from_initial_results( + arrow_table + ) res = dummy_result_set.fetchall() for _ in res: pass @@ -59,5 +71,5 @@ def test_benchmark_fetchall(self): print(f"Executed query {count} times, in {time.time() - start_time} seconds") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_init_file.py b/tests/unit/test_init_file.py new file mode 100644 index 00000000..75b15ac1 --- /dev/null +++ b/tests/unit/test_init_file.py @@ -0,0 +1,19 @@ +import hashlib + + +class TestInitFile: + """ + Micro test to confirm the contents of `databricks/__init__.py` does not change. + + Also see https://github.com/databricks/databricks-sdk-py/issues/343#issuecomment-1866029118. + """ + + def test_init_file_contents(self): + with open("src/databricks/__init__.py") as f: + init_file_contents = f.read() + + # This hash is the expected hash of the contents of `src/databricks/__init__.py`. + # It must not change, or else parallel package installation may lead to clobbered and invalid files. + expected_sha1 = "2772edbf52e517542acf8c039479c4b57b6ca2cd" + actual_sha1 = hashlib.sha1(init_file_contents.encode("utf-8")).hexdigest() + assert expected_sha1 == actual_sha1 diff --git a/tests/unit/test_oauth_persistence.py b/tests/unit/test_oauth_persistence.py index 10677c16..a8ceb14e 100644 --- a/tests/unit/test_oauth_persistence.py +++ b/tests/unit/test_oauth_persistence.py @@ -1,18 +1,17 @@ - import unittest -from databricks.sql.auth.auth import AccessTokenAuthProvider, BasicAuthProvider, AuthProvider -from databricks.sql.auth.auth import get_python_sql_connector_auth_provider -from databricks.sql.experimental.oauth_persistence import DevOnlyFilePersistence, OAuthToken +from databricks.sql.experimental.oauth_persistence import ( + DevOnlyFilePersistence, + OAuthToken, +) import tempfile import os class OAuthPersistenceTests(unittest.TestCase): - def test_DevOnlyFilePersistence_read_my_write(self): with tempfile.TemporaryDirectory() as tempdir: - test_json_file_path = os.path.join(tempdir, 'test.json') + test_json_file_path = os.path.join(tempdir, "test.json") persistence_manager = DevOnlyFilePersistence(test_json_file_path) access_token = "abc#$%%^&^*&*()()_=-/" refresh_token = "#$%%^^&**()+)_gter243]xyz" @@ -25,7 +24,7 @@ def test_DevOnlyFilePersistence_read_my_write(self): def test_DevOnlyFilePersistence_file_does_not_exist(self): with tempfile.TemporaryDirectory() as tempdir: - test_json_file_path = os.path.join(tempdir, 'test.json') + test_json_file_path = os.path.join(tempdir, "test.json") persistence_manager = DevOnlyFilePersistence(test_json_file_path) new_token = persistence_manager.read("https://randomserver") diff --git a/tests/unit/test_param_escaper.py b/tests/unit/test_param_escaper.py index 6c1f1770..925fcea5 100644 --- a/tests/unit/test_param_escaper.py +++ b/tests/unit/test_param_escaper.py @@ -1,104 +1,135 @@ from datetime import date, datetime import unittest, pytest, decimal +from typing import Any, Dict +from databricks.sql.parameters.native import dbsql_parameter_from_primitive -from databricks.sql.utils import ParamEscaper, inject_parameters +from databricks.sql.utils import ( + ParamEscaper, + inject_parameters, + transform_paramstyle, + ParameterStructure, +) pe = ParamEscaper() -class TestIndividualFormatters(object): +class TestIndividualFormatters(object): # Test individual type escapers def test_escape_number_integer(self): - """This behaviour falls back to Python's default string formatting of numbers - """ + """This behaviour falls back to Python's default string formatting of numbers""" assert pe.escape_number(100) == 100 def test_escape_number_float(self): - """This behaviour falls back to Python's default string formatting of numbers - """ + """This behaviour falls back to Python's default string formatting of numbers""" assert pe.escape_number(100.1234) == 100.1234 def test_escape_number_decimal(self): - """This behaviour uses the string representation of a decimal - """ + """This behaviour uses the string representation of a decimal""" assert pe.escape_decimal(decimal.Decimal("124.32")) == "124.32" def test_escape_string_normal(self): - """ - """ + """ """ assert pe.escape_string("golly bob howdy") == "'golly bob howdy'" def test_escape_string_that_includes_special_characters(self): - """Tests for how special characters are treated. - - When passed a string, the `escape_string` method wraps it in single quotes - and escapes any special characters with a back stroke (\) - - Example: - - IN : his name was 'robert palmer' - OUT: 'his name was \'robert palmer\'' - """ - - # Testing for the presence of these characters: '"/\😂 - - assert pe.escape_string("his name was 'robert palmer'") == r"'his name was \'robert palmer\''" + r"""Tests for how special characters are treated. - # These tests represent the same user input in the several ways it can be written in Python - # Each argument to `escape_string` evaluates to the same bytes. But Python lets us write it differently. - assert pe.escape_string("his name was \"robert palmer\"") == "'his name was \"robert palmer\"'" - assert pe.escape_string('his name was "robert palmer"') == "'his name was \"robert palmer\"'" - assert pe.escape_string('his name was {}'.format('"robert palmer"')) == "'his name was \"robert palmer\"'" + When passed a string, the `escape_string` method wraps it in single quotes + and escapes any special characters with a back stroke (\) - assert pe.escape_string("his name was robert / palmer") == r"'his name was robert / palmer'" + Example: - # If you need to include a single backslash, use an r-string to prevent Python from raising a - # DeprecationWarning for an invalid escape sequence - assert pe.escape_string("his name was robert \\/ palmer") == r"'his name was robert \\/ palmer'" - assert pe.escape_string("his name was robert \\ palmer") == r"'his name was robert \\ palmer'" - assert pe.escape_string("his name was robert \\\\ palmer") == r"'his name was robert \\\\ palmer'" - - assert pe.escape_string("his name was robert palmer 😂") == r"'his name was robert palmer 😂'" - - # Adding the test from PR #56 to prove escape behaviour - - assert pe.escape_string("you're") == r"'you\'re'" - - # Adding this test from #51 to prove escape behaviour when the target string involves repeated SQL escape chars - assert pe.escape_string("cat\\'s meow") == r"'cat\\\'s meow'" - - # Tests from the docs: https://docs.databricks.com/sql/language-manual/data-types/string-type.html + IN : his name was 'robert palmer' + OUT: 'his name was \'robert palmer\'' + """ - assert pe.escape_string('Spark') == "'Spark'" - assert pe.escape_string("O'Connell") == r"'O\'Connell'" - assert pe.escape_string("Some\\nText") == r"'Some\\nText'" - assert pe.escape_string("Some\\\\nText") == r"'Some\\\\nText'" - assert pe.escape_string("서울시") == "'서울시'" - assert pe.escape_string("\\\\") == r"'\\\\'" + # Testing for the presence of these characters: '"/\😂 + + assert ( + pe.escape_string("his name was 'robert palmer'") + == r"'his name was \'robert palmer\''" + ) + + # These tests represent the same user input in the several ways it can be written in Python + # Each argument to `escape_string` evaluates to the same bytes. But Python lets us write it differently. + assert ( + pe.escape_string('his name was "robert palmer"') + == "'his name was \"robert palmer\"'" + ) + assert ( + pe.escape_string('his name was "robert palmer"') + == "'his name was \"robert palmer\"'" + ) + assert ( + pe.escape_string("his name was {}".format('"robert palmer"')) + == "'his name was \"robert palmer\"'" + ) + + assert ( + pe.escape_string("his name was robert / palmer") + == r"'his name was robert / palmer'" + ) + + # If you need to include a single backslash, use an r-string to prevent Python from raising a + # DeprecationWarning for an invalid escape sequence + assert ( + pe.escape_string("his name was robert \\/ palmer") + == r"'his name was robert \\/ palmer'" + ) + assert ( + pe.escape_string("his name was robert \\ palmer") + == r"'his name was robert \\ palmer'" + ) + assert ( + pe.escape_string("his name was robert \\\\ palmer") + == r"'his name was robert \\\\ palmer'" + ) + + assert ( + pe.escape_string("his name was robert palmer 😂") + == r"'his name was robert palmer 😂'" + ) + + # Adding the test from PR #56 to prove escape behaviour + + assert pe.escape_string("you're") == r"'you\'re'" + + # Adding this test from #51 to prove escape behaviour when the target string involves repeated SQL escape chars + assert pe.escape_string("cat\\'s meow") == r"'cat\\\'s meow'" + + # Tests from the docs: https://docs.databricks.com/sql/language-manual/data-types/string-type.html + + assert pe.escape_string("Spark") == "'Spark'" + assert pe.escape_string("O'Connell") == r"'O\'Connell'" + assert pe.escape_string("Some\\nText") == r"'Some\\nText'" + assert pe.escape_string("Some\\\\nText") == r"'Some\\\\nText'" + assert pe.escape_string("서울시") == "'서울시'" + assert pe.escape_string("\\\\") == r"'\\\\'" def test_escape_date_time(self): - INPUT = datetime(1991,8,3,21,55) + INPUT = datetime(1991, 8, 3, 21, 55) FORMAT = "%Y-%m-%d %H:%M:%S" OUTPUT = "'1991-08-03 21:55:00'" assert pe.escape_datetime(INPUT, FORMAT) == OUTPUT def test_escape_date(self): - INPUT = date(1991,8,3) + INPUT = date(1991, 8, 3) FORMAT = "%Y-%m-%d" OUTPUT = "'1991-08-03'" assert pe.escape_datetime(INPUT, FORMAT) == OUTPUT def test_escape_sequence_integer(self): - assert pe.escape_sequence([1,2,3,4]) == "(1,2,3,4)" + assert pe.escape_sequence([1, 2, 3, 4]) == "(1,2,3,4)" def test_escape_sequence_float(self): - assert pe.escape_sequence([1.1,2.2,3.3,4.4]) == "(1.1,2.2,3.3,4.4)" + assert pe.escape_sequence([1.1, 2.2, 3.3, 4.4]) == "(1.1,2.2,3.3,4.4)" def test_escape_sequence_string(self): - assert pe.escape_sequence( - ["his", "name", "was", "robert", "palmer"]) == \ - "('his','name','was','robert','palmer')" + assert ( + pe.escape_sequence(["his", "name", "was", "robert", "palmer"]) + == "('his','name','was','robert','palmer')" + ) def test_escape_sequence_sequence_of_strings(self): # This is not valid SQL. @@ -109,9 +140,7 @@ def test_escape_sequence_sequence_of_strings(self): class TestFullQueryEscaping(object): - def test_simple(self): - INPUT = """ SELECT field1, @@ -140,7 +169,6 @@ def test_simple(self): @unittest.skipUnless(False, "Thrift server supports native parameter binding.") def test_only_bind_in_where_clause(self): - INPUT = """ SELECT %(field)s, @@ -153,3 +181,55 @@ def test_only_bind_in_where_clause(self): with pytest.raises(Exception): inject_parameters(INPUT, pe.escape_args(args)) + + +class TestInlineToNativeTransformer(object): + @pytest.mark.parametrize( + ("label", "query", "params", "expected"), + ( + ("no effect", "SELECT 1", {}, "SELECT 1"), + ("one marker", "%(param)s", {"param": ""}, ":param"), + ( + "multiple markers", + "%(foo)s %(bar)s %(baz)s", + {"foo": None, "bar": None, "baz": None}, + ":foo :bar :baz", + ), + ( + "sql query", + "SELECT * FROM table WHERE field = %(param)s AND other_field IN (%(list)s)", + {"param": None, "list": None}, + "SELECT * FROM table WHERE field = :param AND other_field IN (:list)", + ), + ( + "query with like wildcard", + 'select * from table where field like "%"', + {}, + 'select * from table where field like "%"', + ), + ( + "query with named param and like wildcard", + 'select :param from table where field like "%"', + {"param": None}, + 'select :param from table where field like "%"', + ), + ( + "query with doubled wildcards", + "select 1 where " ' like "%%"', + {"param": None}, + "select 1 where " ' like "%%"', + ), + ), + ) + def test_transformer( + self, label: str, query: str, params: Dict[str, Any], expected: str + ): + + _params = [ + dbsql_parameter_from_primitive(value=value, name=name) + for name, value in params.items() + ] + output = transform_paramstyle( + query, _params, param_structure=ParameterStructure.NAMED + ) + assert output == expected diff --git a/tests/unit/test_parameters.py b/tests/unit/test_parameters.py new file mode 100644 index 00000000..eec921e4 --- /dev/null +++ b/tests/unit/test_parameters.py @@ -0,0 +1,204 @@ +import datetime +from decimal import Decimal +from enum import Enum +from typing import Type + +import pytest +import pytz + +from databricks.sql.client import Connection +from databricks.sql.parameters import ( + BigIntegerParameter, + BooleanParameter, + DateParameter, + DecimalParameter, + DoubleParameter, + FloatParameter, + IntegerParameter, + SmallIntParameter, + StringParameter, + TimestampNTZParameter, + TimestampParameter, + TinyIntParameter, + VoidParameter, +) +from databricks.sql.parameters.native import ( + TDbsqlParameter, + TSparkParameterValue, + dbsql_parameter_from_primitive, +) +from databricks.sql.thrift_api.TCLIService import ttypes +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TOpenSessionResp, + TSessionHandle, + TSparkParameterValue, +) + + +class TestSessionHandleChecks(object): + @pytest.mark.parametrize( + "test_input,expected", + [ + ( + TOpenSessionResp( + serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V7, + sessionHandle=TSessionHandle(1, None), + ), + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V7, + ), + # Ensure that protocol version inside sessionhandle takes precedence. + ( + TOpenSessionResp( + serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V7, + sessionHandle=TSessionHandle( + 1, ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V8 + ), + ), + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V8, + ), + ], + ) + def test_get_protocol_version_fallback_behavior(self, test_input, expected): + assert Connection.get_protocol_version(test_input) == expected + + @pytest.mark.parametrize( + "test_input,expected", + [ + ( + None, + False, + ), + ( + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V7, + False, + ), + ( + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V8, + True, + ), + ], + ) + def test_parameters_enabled(self, test_input, expected): + assert Connection.server_parameterized_queries_enabled(test_input) == expected + + +@pytest.mark.parametrize( + "value,expected", + ( + (Decimal("10.00"), "DECIMAL(4,2)"), + (Decimal("123456789123456789.123456789123456789"), "DECIMAL(36,18)"), + (Decimal(".12345678912345678912345678912345678912"), "DECIMAL(38,38)"), + (Decimal("123456789.123456789"), "DECIMAL(18,9)"), + (Decimal("12345678912345678912345678912345678912"), "DECIMAL(38,0)"), + (Decimal("1234.56"), "DECIMAL(6,2)"), + ), +) +def test_calculate_decimal_cast_string(value, expected): + p = DecimalParameter(value) + assert p._cast_expr() == expected + + +class Primitive(Enum): + """These are the inferrable types. This Enum is used for parametrized tests.""" + + NONE = None + BOOL = True + INT = 50 + BIGINT = 2147483648 + STRING = "Hello" + DECIMAL = Decimal("1234.56") + DATE = datetime.date(2023, 9, 6) + TIMESTAMP = datetime.datetime(2023, 9, 6, 3, 14, 27, 843, tzinfo=pytz.UTC) + DOUBLE = 3.14 + FLOAT = 3.15 + SMALLINT = 51 + + +class TestDbsqlParameter: + @pytest.mark.parametrize( + "_type, prim, expect_cast_expr", + ( + (DecimalParameter, Primitive.DECIMAL, "DECIMAL(6,2)"), + (IntegerParameter, Primitive.INT, "INT"), + (StringParameter, Primitive.STRING, "STRING"), + (BigIntegerParameter, Primitive.BIGINT, "BIGINT"), + (BooleanParameter, Primitive.BOOL, "BOOLEAN"), + (DateParameter, Primitive.DATE, "DATE"), + (DoubleParameter, Primitive.DOUBLE, "DOUBLE"), + (FloatParameter, Primitive.FLOAT, "FLOAT"), + (VoidParameter, Primitive.NONE, "VOID"), + (SmallIntParameter, Primitive.INT, "SMALLINT"), + (TimestampParameter, Primitive.TIMESTAMP, "TIMESTAMP"), + (TimestampNTZParameter, Primitive.TIMESTAMP, "TIMESTAMP_NTZ"), + (TinyIntParameter, Primitive.INT, "TINYINT"), + ), + ) + def test_cast_expression( + self, _type: TDbsqlParameter, prim: Primitive, expect_cast_expr: str + ): + p = _type(prim.value) + assert p._cast_expr() == expect_cast_expr + + @pytest.mark.parametrize( + "t, prim", + ( + (DecimalParameter, Primitive.DECIMAL), + (IntegerParameter, Primitive.INT), + (StringParameter, Primitive.STRING), + (BigIntegerParameter, Primitive.BIGINT), + (BooleanParameter, Primitive.BOOL), + (DateParameter, Primitive.DATE), + (DoubleParameter, Primitive.DOUBLE), + (FloatParameter, Primitive.FLOAT), + (VoidParameter, Primitive.NONE), + (SmallIntParameter, Primitive.INT), + (TimestampParameter, Primitive.TIMESTAMP), + (TimestampNTZParameter, Primitive.TIMESTAMP), + (TinyIntParameter, Primitive.INT), + ), + ) + def test_tspark_param_value(self, t: TDbsqlParameter, prim): + p: TDbsqlParameter = t(prim.value) + output = p._tspark_param_value() + + if prim == Primitive.NONE: + assert output == None + else: + assert output == TSparkParameterValue(stringValue=str(prim.value)) + + def test_tspark_param_named(self): + p = dbsql_parameter_from_primitive(Primitive.INT.value, name="p") + tsp = p.as_tspark_param(named=True) + + assert tsp.name == "p" + assert tsp.ordinal is False + + def test_tspark_param_ordinal(self): + p = dbsql_parameter_from_primitive(Primitive.INT.value, name="p") + tsp = p.as_tspark_param(named=False) + + assert tsp.name is None + assert tsp.ordinal is True + + @pytest.mark.parametrize( + "_type, prim", + ( + (DecimalParameter, Primitive.DECIMAL), + (IntegerParameter, Primitive.INT), + (StringParameter, Primitive.STRING), + (BigIntegerParameter, Primitive.BIGINT), + (BooleanParameter, Primitive.BOOL), + (DateParameter, Primitive.DATE), + (FloatParameter, Primitive.FLOAT), + (VoidParameter, Primitive.NONE), + (TimestampParameter, Primitive.TIMESTAMP), + ), + ) + def test_inference(self, _type: TDbsqlParameter, prim: Primitive): + """This method only tests inferrable types. + + Not tested are TinyIntParameter, SmallIntParameter DoubleParameter and TimestampNTZParameter + """ + + inferred_type = dbsql_parameter_from_primitive(prim.value) + assert isinstance(inferred_type, _type) diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py new file mode 100644 index 00000000..1e18e1f4 --- /dev/null +++ b/tests/unit/test_retry.py @@ -0,0 +1,86 @@ +import time +from unittest.mock import patch, call +import pytest +from urllib3 import HTTPResponse +from databricks.sql.auth.retry import DatabricksRetryPolicy, RequestHistory, CommandType +from urllib3.exceptions import MaxRetryError + + +class TestRetry: + @pytest.fixture() + def retry_policy(self) -> DatabricksRetryPolicy: + return DatabricksRetryPolicy( + delay_min=1, + delay_max=30, + stop_after_attempts_count=3, + stop_after_attempts_duration=900, + delay_default=2, + force_dangerous_codes=[], + ) + + @pytest.fixture() + def error_history(self) -> RequestHistory: + return RequestHistory( + method="POST", url=None, error=None, status=503, redirect_location=None + ) + + def calculate_backoff_time(self, attempt, delay_min, delay_max): + exponential_backoff_time = (2**attempt) * delay_min + return min(exponential_backoff_time, delay_max) + + @patch("time.sleep") + def test_sleep__no_retry_after(self, t_mock, retry_policy, error_history): + retry_policy._retry_start_time = time.time() + retry_policy.history = [error_history, error_history] + retry_policy.sleep(HTTPResponse(status=503)) + + expected_backoff_time = self.calculate_backoff_time( + 0, retry_policy.delay_min, retry_policy.delay_max + ) + t_mock.assert_called_with(expected_backoff_time) + + @patch("time.sleep") + def test_sleep__no_retry_after_header__multiple_retries(self, t_mock, retry_policy): + num_attempts = retry_policy.stop_after_attempts_count + + retry_policy._retry_start_time = time.time() + retry_policy.command_type = CommandType.OTHER + + for attempt in range(num_attempts): + retry_policy.sleep(HTTPResponse(status=503)) + # Internally urllib3 calls the increment function generating a new instance for every retry + retry_policy = retry_policy.increment() + + expected_backoff_times = [] + for attempt in range(num_attempts): + expected_backoff_times.append( + self.calculate_backoff_time( + attempt, retry_policy.delay_min, retry_policy.delay_max + ) + ) + + # Asserts if the sleep value was called in the expected order + t_mock.assert_has_calls( + [call(expected_time) for expected_time in expected_backoff_times] + ) + + @patch("time.sleep") + def test_excessive_retry_attempts_error(self, t_mock, retry_policy): + # Attempting more than stop_after_attempt_count + num_attempts = retry_policy.stop_after_attempts_count + 1 + + retry_policy._retry_start_time = time.time() + retry_policy.command_type = CommandType.OTHER + + with pytest.raises(MaxRetryError): + for attempt in range(num_attempts): + retry_policy.sleep(HTTPResponse(status=503)) + # Internally urllib3 calls the increment function generating a new instance for every retry + retry_policy = retry_policy.increment() + + @patch("time.sleep") + def test_sleep__retry_after_present(self, t_mock, retry_policy, error_history): + retry_policy._retry_start_time = time.time() + retry_policy.history = [error_history, error_history, error_history] + retry_policy.sleep(HTTPResponse(status=503, headers={"Retry-After": "3"})) + t_mock.assert_called_with(3) diff --git a/tests/unit/test_thrift_backend.py b/tests/unit/test_thrift_backend.py index 1c2e589b..592fd006 100644 --- a/tests/unit/test_thrift_backend.py +++ b/tests/unit/test_thrift_backend.py @@ -2,12 +2,18 @@ from decimal import Decimal import itertools import unittest +import pytest from unittest.mock import patch, MagicMock, Mock from ssl import CERT_NONE, CERT_REQUIRED +from urllib3 import HTTPSConnectionPool -import pyarrow - +try: + import pyarrow +except ImportError: + pyarrow = None import databricks.sql +from databricks.sql import utils +from databricks.sql.types import SSLOptions from databricks.sql.thrift_api.TCLIService import ttypes from databricks.sql import * from databricks.sql.auth.authenticators import AuthProvider @@ -15,15 +21,16 @@ def retry_policy_factory(): - return { # (type, default, min, max) - "_retry_delay_min": (float, 1, None, None), - "_retry_delay_max": (float, 60, None, None), - "_retry_stop_after_attempts_count": (int, 30, None, None), - "_retry_stop_after_attempts_duration": (float, 900, None, None), - "_retry_delay_default": (float, 5, 1, 60) + return { # (type, default, min, max) + "_retry_delay_min": (float, 1, None, None), + "_retry_delay_max": (float, 60, None, None), + "_retry_stop_after_attempts_count": (int, 30, None, None), + "_retry_stop_after_attempts_duration": (float, 900, None, None), + "_retry_delay_default": (float, 5, 1, 60), } +@pytest.mark.skipif(pyarrow is None, reason="PyArrow is not installed") class ThriftBackendTestSuite(unittest.TestCase): okay_status = ttypes.TStatus(statusCode=ttypes.TStatusCode.SUCCESS_STATUS) @@ -34,14 +41,17 @@ class ThriftBackendTestSuite(unittest.TestCase): operation_handle = ttypes.TOperationHandle( operationId=ttypes.THandleIdentifier(guid=0x33, secret=0x35), - operationType=ttypes.TOperationType.EXECUTE_STATEMENT) + operationType=ttypes.TOperationType.EXECUTE_STATEMENT, + ) session_handle = ttypes.TSessionHandle( - sessionId=ttypes.THandleIdentifier(guid=0x36, secret=0x37)) + sessionId=ttypes.THandleIdentifier(guid=0x36, secret=0x37) + ) open_session_resp = ttypes.TOpenSessionResp( status=okay_status, - serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4) + serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4, + ) metadata_resp = ttypes.TGetResultSetMetadataResp( status=okay_status, @@ -50,8 +60,11 @@ class ThriftBackendTestSuite(unittest.TestCase): ) execute_response_types = [ - ttypes.TExecuteStatementResp, ttypes.TGetCatalogsResp, ttypes.TGetSchemasResp, - ttypes.TGetTablesResp, ttypes.TGetColumnsResp + ttypes.TExecuteStatementResp, + ttypes.TGetCatalogsResp, + ttypes.TGetSchemasResp, + ttypes.TGetTablesResp, + ttypes.TGetColumnsResp, ] def test_make_request_checks_thrift_status_code(self): @@ -60,15 +73,31 @@ def test_make_request_checks_thrift_status_code(self): mock_method = Mock() mock_method.__name__ = "method name" mock_method.return_value = mock_response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(DatabaseError): thrift_backend.make_request(mock_method, Mock()) def _make_type_desc(self, type): - return ttypes.TTypeDesc(types=[ttypes.TTypeEntry(ttypes.TPrimitiveTypeEntry(type=type))]) + return ttypes.TTypeDesc( + types=[ttypes.TTypeEntry(ttypes.TTAllowedParameterValueEntry(type=type))] + ) def _make_fake_thrift_backend(self): - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend._hive_schema_to_arrow_schema = Mock() thrift_backend._hive_schema_to_description = Mock() thrift_backend._create_arrow_table = MagicMock() @@ -78,13 +107,20 @@ def _make_fake_thrift_backend(self): def test_hive_schema_to_arrow_schema_preserves_column_names(self): columns = [ ttypes.TColumnDesc( - columnName="column 1", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 1", + typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE), + ), ttypes.TColumnDesc( - columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 2", + typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE), + ), ttypes.TColumnDesc( - columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 2", + typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE), + ), ttypes.TColumnDesc( - columnName="", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)) + columnName="", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE) + ), ] t_table_schema = ttypes.TTableSchema(columns) @@ -95,7 +131,7 @@ def test_hive_schema_to_arrow_schema_preserves_column_names(self): self.assertEqual(arrow_schema.field(2).name, "column 2") self.assertEqual(arrow_schema.field(3).name, "") - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_bad_protocol_versions_are_rejected(self, tcli_service_client_cass): t_http_client_instance = tcli_service_client_cass.return_value bad_protocol_versions = [ @@ -114,34 +150,47 @@ def test_bad_protocol_versions_are_rejected(self, tcli_service_client_cass): for protocol_version in bad_protocol_versions: t_http_client_instance.OpenSession.return_value = ttypes.TOpenSessionResp( - status=self.okay_status, serverProtocolVersion=protocol_version) + status=self.okay_status, serverProtocolVersion=protocol_version + ) with self.assertRaises(OperationalError) as cm: thrift_backend = self._make_fake_thrift_backend() thrift_backend.open_session({}, None, None) - self.assertIn("expected server to use a protocol version", str(cm.exception)) + self.assertIn( + "expected server to use a protocol version", str(cm.exception) + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_okay_protocol_versions_succeed(self, tcli_service_client_cass): t_http_client_instance = tcli_service_client_cass.return_value good_protocol_versions = [ ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V2, ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V3, - ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4 + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4, ] for protocol_version in good_protocol_versions: t_http_client_instance.OpenSession.return_value = ttypes.TOpenSessionResp( - status=self.okay_status, serverProtocolVersion=protocol_version) + status=self.okay_status, serverProtocolVersion=protocol_version + ) thrift_backend = self._make_fake_thrift_backend() thrift_backend.open_session({}, None, None) @patch("databricks.sql.auth.thrift_http_client.THttpClient") def test_headers_are_set(self, t_http_client_class): - ThriftBackend("foo", 123, "bar", [("header", "value")], auth_provider=AuthProvider()) - t_http_client_class.return_value.setCustomHeaders.assert_called_with({"header": "value"}) + ThriftBackend( + "foo", + 123, + "bar", + [("header", "value")], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) + t_http_client_class.return_value.setCustomHeaders.assert_called_with( + {"header": "value"} + ) def test_proxy_headers_are_set(self): @@ -150,85 +199,268 @@ def test_proxy_headers_are_set(self): fake_proxy_spec = "https://someuser:somepassword@8.8.8.8:12340" parsed_proxy = urlparse(fake_proxy_spec) - + try: - result = THttpClient.basic_proxy_auth_header(parsed_proxy) + result = THttpClient.basic_proxy_auth_headers(parsed_proxy) except TypeError as e: assert False - assert isinstance(result, type(str())) + assert isinstance(result, type(dict())) + assert isinstance(result.get("proxy-authorization"), type(str())) @patch("databricks.sql.auth.thrift_http_client.THttpClient") - @patch("databricks.sql.thrift_backend.create_default_context") - def test_tls_cert_args_are_propagated(self, mock_create_default_context, t_http_client_class): + @patch("databricks.sql.types.create_default_context") + def test_tls_cert_args_are_propagated( + self, mock_create_default_context, t_http_client_class + ): mock_cert_key_file = Mock() mock_cert_key_password = Mock() mock_trusted_ca_file = Mock() mock_cert_file = Mock() + mock_ssl_options = SSLOptions( + tls_client_cert_file=mock_cert_file, + tls_client_cert_key_file=mock_cert_key_file, + tls_client_cert_key_password=mock_cert_key_password, + tls_trusted_ca_file=mock_trusted_ca_file, + ) + mock_ssl_context = mock_ssl_options.create_ssl_context() + mock_create_default_context.assert_called_once_with(cafile=mock_trusted_ca_file) + ThriftBackend( "foo", 123, - "bar", [], + "bar", + [], auth_provider=AuthProvider(), - _tls_client_cert_file=mock_cert_file, - _tls_client_cert_key_file=mock_cert_key_file, - _tls_client_cert_key_password=mock_cert_key_password, - _tls_trusted_ca_file=mock_trusted_ca_file) + ssl_options=mock_ssl_options, + ) - mock_create_default_context.assert_called_once_with(cafile=mock_trusted_ca_file) - mock_ssl_context = mock_create_default_context.return_value mock_ssl_context.load_cert_chain.assert_called_once_with( - certfile=mock_cert_file, keyfile=mock_cert_key_file, password=mock_cert_key_password) + certfile=mock_cert_file, + keyfile=mock_cert_key_file, + password=mock_cert_key_password, + ) self.assertTrue(mock_ssl_context.check_hostname) self.assertEqual(mock_ssl_context.verify_mode, CERT_REQUIRED) - self.assertEqual(t_http_client_class.call_args[1]["ssl_context"], mock_ssl_context) + self.assertEqual( + t_http_client_class.call_args[1]["ssl_options"], mock_ssl_options + ) + + @patch("databricks.sql.types.create_default_context") + def test_tls_cert_args_are_used_by_http_client(self, mock_create_default_context): + from databricks.sql.auth.thrift_http_client import THttpClient + + mock_cert_key_file = Mock() + mock_cert_key_password = Mock() + mock_trusted_ca_file = Mock() + mock_cert_file = Mock() + + mock_ssl_options = SSLOptions( + tls_verify=True, + tls_client_cert_file=mock_cert_file, + tls_client_cert_key_file=mock_cert_key_file, + tls_client_cert_key_password=mock_cert_key_password, + tls_trusted_ca_file=mock_trusted_ca_file, + ) + + http_client = THttpClient( + auth_provider=None, + uri_or_host="https://example.com", + ssl_options=mock_ssl_options, + ) + + self.assertEqual(http_client.scheme, "https") + self.assertEqual(http_client.certfile, mock_ssl_options.tls_client_cert_file) + self.assertEqual(http_client.keyfile, mock_ssl_options.tls_client_cert_key_file) + self.assertIsNotNone(http_client.certfile) + mock_create_default_context.assert_called() + + http_client.open() + + conn_pool = http_client._THttpClient__pool + self.assertIsInstance(conn_pool, HTTPSConnectionPool) + self.assertEqual(conn_pool.cert_reqs, CERT_REQUIRED) + self.assertEqual(conn_pool.ca_certs, mock_ssl_options.tls_trusted_ca_file) + self.assertEqual(conn_pool.cert_file, mock_ssl_options.tls_client_cert_file) + self.assertEqual(conn_pool.key_file, mock_ssl_options.tls_client_cert_key_file) + self.assertEqual( + conn_pool.key_password, mock_ssl_options.tls_client_cert_key_password + ) + + def test_tls_no_verify_is_respected_by_http_client(self): + from databricks.sql.auth.thrift_http_client import THttpClient + + http_client = THttpClient( + auth_provider=None, + uri_or_host="https://example.com", + ssl_options=SSLOptions(tls_verify=False), + ) + self.assertEqual(http_client.scheme, "https") + + http_client.open() + + conn_pool = http_client._THttpClient__pool + self.assertIsInstance(conn_pool, HTTPSConnectionPool) + self.assertEqual(conn_pool.cert_reqs, CERT_NONE) @patch("databricks.sql.auth.thrift_http_client.THttpClient") - @patch("databricks.sql.thrift_backend.create_default_context") - def test_tls_no_verify_is_respected(self, mock_create_default_context, t_http_client_class): - ThriftBackend("foo", 123, "bar", [], auth_provider=AuthProvider(), _tls_no_verify=True) + @patch("databricks.sql.types.create_default_context") + def test_tls_no_verify_is_respected( + self, mock_create_default_context, t_http_client_class + ): + mock_ssl_options = SSLOptions(tls_verify=False) + mock_ssl_context = mock_ssl_options.create_ssl_context() + mock_create_default_context.assert_called() + + ThriftBackend( + "foo", + 123, + "bar", + [], + auth_provider=AuthProvider(), + ssl_options=mock_ssl_options, + ) - mock_ssl_context = mock_create_default_context.return_value self.assertFalse(mock_ssl_context.check_hostname) self.assertEqual(mock_ssl_context.verify_mode, CERT_NONE) - self.assertEqual(t_http_client_class.call_args[1]["ssl_context"], mock_ssl_context) + self.assertEqual( + t_http_client_class.call_args[1]["ssl_options"], mock_ssl_options + ) @patch("databricks.sql.auth.thrift_http_client.THttpClient") - @patch("databricks.sql.thrift_backend.create_default_context") - def test_tls_verify_hostname_is_respected(self, mock_create_default_context, - t_http_client_class): - ThriftBackend("foo", 123, "bar", [], auth_provider=AuthProvider(), _tls_verify_hostname=False) + @patch("databricks.sql.types.create_default_context") + def test_tls_verify_hostname_is_respected( + self, mock_create_default_context, t_http_client_class + ): + mock_ssl_options = SSLOptions(tls_verify_hostname=False) + mock_ssl_context = mock_ssl_options.create_ssl_context() + mock_create_default_context.assert_called() + + ThriftBackend( + "foo", + 123, + "bar", + [], + auth_provider=AuthProvider(), + ssl_options=mock_ssl_options, + ) - mock_ssl_context = mock_create_default_context.return_value self.assertFalse(mock_ssl_context.check_hostname) self.assertEqual(mock_ssl_context.verify_mode, CERT_REQUIRED) - self.assertEqual(t_http_client_class.call_args[1]["ssl_context"], mock_ssl_context) + self.assertEqual( + t_http_client_class.call_args[1]["ssl_options"], mock_ssl_options + ) @patch("databricks.sql.auth.thrift_http_client.THttpClient") def test_port_and_host_are_respected(self, t_http_client_class): - ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider()) - self.assertEqual(t_http_client_class.call_args[1]["uri_or_host"], - "https://hostname:123/path_value") + ThriftBackend( + "hostname", + 123, + "path_value", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) + self.assertEqual( + t_http_client_class.call_args[1]["uri_or_host"], + "https://hostname:123/path_value", + ) + + @patch("databricks.sql.auth.thrift_http_client.THttpClient") + def test_host_with_https_does_not_duplicate(self, t_http_client_class): + ThriftBackend( + "https://hostname", + 123, + "path_value", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) + self.assertEqual( + t_http_client_class.call_args[1]["uri_or_host"], + "https://hostname:123/path_value", + ) + + @patch("databricks.sql.auth.thrift_http_client.THttpClient") + def test_host_with_trailing_backslash_does_not_duplicate(self, t_http_client_class): + ThriftBackend( + "https://hostname/", + 123, + "path_value", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) + self.assertEqual( + t_http_client_class.call_args[1]["uri_or_host"], + "https://hostname:123/path_value", + ) @patch("databricks.sql.auth.thrift_http_client.THttpClient") def test_socket_timeout_is_propagated(self, t_http_client_class): - ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider(), _socket_timeout=129) - self.assertEqual(t_http_client_class.return_value.setTimeout.call_args[0][0], 129 * 1000) - ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider(), _socket_timeout=0) + ThriftBackend( + "hostname", + 123, + "path_value", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + _socket_timeout=129, + ) + self.assertEqual( + t_http_client_class.return_value.setTimeout.call_args[0][0], 129 * 1000 + ) + ThriftBackend( + "hostname", + 123, + "path_value", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + _socket_timeout=0, + ) self.assertEqual(t_http_client_class.return_value.setTimeout.call_args[0][0], 0) - ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider(), _socket_timeout=None) - self.assertEqual(t_http_client_class.return_value.setTimeout.call_args[0][0], None) + ThriftBackend( + "hostname", + 123, + "path_value", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) + self.assertEqual( + t_http_client_class.return_value.setTimeout.call_args[0][0], 900 * 1000 + ) + ThriftBackend( + "hostname", + 123, + "path_value", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + _socket_timeout=None, + ) + self.assertEqual( + t_http_client_class.return_value.setTimeout.call_args[0][0], None + ) def test_non_primitive_types_raise_error(self): columns = [ ttypes.TColumnDesc( - columnName="column 1", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 1", + typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE), + ), ttypes.TColumnDesc( columnName="column 2", - typeDesc=ttypes.TTypeDesc(types=[ - ttypes.TTypeEntry(userDefinedTypeEntry=ttypes.TUserDefinedTypeEntry("foo")) - ])) + typeDesc=ttypes.TTypeDesc( + types=[ + ttypes.TTypeEntry( + userDefinedTypeEntry=ttypes.TUserDefinedTypeEntry("foo") + ) + ] + ), + ), ] t_table_schema = ttypes.TTableSchema(columns) @@ -242,50 +474,83 @@ def test_hive_schema_to_description_preserves_column_names_and_types(self): # canary test columns = [ ttypes.TColumnDesc( - columnName="column 1", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 1", + typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE), + ), ttypes.TColumnDesc( - columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.BOOLEAN_TYPE)), + columnName="column 2", + typeDesc=self._make_type_desc(ttypes.TTypeId.BOOLEAN_TYPE), + ), ttypes.TColumnDesc( - columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.MAP_TYPE)), + columnName="column 2", + typeDesc=self._make_type_desc(ttypes.TTypeId.MAP_TYPE), + ), ttypes.TColumnDesc( - columnName="", typeDesc=self._make_type_desc(ttypes.TTypeId.STRUCT_TYPE)) + columnName="", typeDesc=self._make_type_desc(ttypes.TTypeId.STRUCT_TYPE) + ), ] t_table_schema = ttypes.TTableSchema(columns) description = ThriftBackend._hive_schema_to_description(t_table_schema) - self.assertEqual(description, [ - ("column 1", "int", None, None, None, None, None), - ("column 2", "boolean", None, None, None, None, None), - ("column 2", "map", None, None, None, None, None), - ("", "struct", None, None, None, None, None), - ]) + self.assertEqual( + description, + [ + ("column 1", "int", None, None, None, None, None), + ("column 2", "boolean", None, None, None, None, None), + ("column 2", "map", None, None, None, None, None), + ("", "struct", None, None, None, None, None), + ], + ) def test_hive_schema_to_description_preserves_scale_and_precision(self): columns = [ ttypes.TColumnDesc( columnName="column 1", - typeDesc=ttypes.TTypeDesc(types=[ - ttypes.TTypeEntry( - ttypes.TPrimitiveTypeEntry( - type=ttypes.TTypeId.DECIMAL_TYPE, - typeQualifiers=ttypes.TTypeQualifiers( - qualifiers={ - "precision": ttypes.TTypeQualifierValue(i32Value=10), - "scale": ttypes.TTypeQualifierValue(i32Value=100), - }))) - ])), + typeDesc=ttypes.TTypeDesc( + types=[ + ttypes.TTypeEntry( + ttypes.TTAllowedParameterValueEntry( + type=ttypes.TTypeId.DECIMAL_TYPE, + typeQualifiers=ttypes.TTypeQualifiers( + qualifiers={ + "precision": ttypes.TTypeQualifierValue( + i32Value=10 + ), + "scale": ttypes.TTypeQualifierValue( + i32Value=100 + ), + } + ), + ) + ) + ] + ), + ), ] t_table_schema = ttypes.TTableSchema(columns) description = ThriftBackend._hive_schema_to_description(t_table_schema) - self.assertEqual(description, [ - ("column 1", "decimal", None, None, 10, 100, None), - ]) + self.assertEqual( + description, + [ + ("column 1", "decimal", None, None, 10, 100, None), + ], + ) def test_make_request_checks_status_code(self): - error_codes = [ttypes.TStatusCode.ERROR_STATUS, ttypes.TStatusCode.INVALID_HANDLE_STATUS] - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + error_codes = [ + ttypes.TStatusCode.ERROR_STATUS, + ttypes.TStatusCode.INVALID_HANDLE_STATUS, + ] + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) for code in error_codes: mock_error_response = Mock() @@ -296,8 +561,9 @@ def test_make_request_checks_status_code(self): self.assertIn("a detailed error message", str(cm.exception)) success_codes = [ - ttypes.TStatusCode.SUCCESS_STATUS, ttypes.TStatusCode.SUCCESS_WITH_INFO_STATUS, - ttypes.TStatusCode.STILL_EXECUTING_STATUS + ttypes.TStatusCode.SUCCESS_STATUS, + ttypes.TStatusCode.SUCCESS_WITH_INFO_STATUS, + ttypes.TStatusCode.STILL_EXECUTING_STATUS, ] for code in success_codes: @@ -314,69 +580,111 @@ def test_handle_execute_response_checks_operation_state_in_direct_results(self): operationStatus=ttypes.TGetOperationStatusResp( status=self.okay_status, operationState=ttypes.TOperationState.ERROR_STATE, - errorMessage="some information about the error"), + errorMessage="some information about the error", + ), resultSetMetadata=None, resultSet=None, - closeOperation=None)) - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + closeOperation=None, + ), + ) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(DatabaseError) as cm: thrift_backend._handle_execute_response(t_execute_resp, Mock()) self.assertIn("some information about the error", str(cm.exception)) - def test_handle_execute_response_sets_compression_in_direct_results(self): + @patch( + "databricks.sql.utils.ResultSetQueueFactory.build_queue", return_value=Mock() + ) + def test_handle_execute_response_sets_compression_in_direct_results( + self, build_queue + ): for resp_type in self.execute_response_types: - lz4Compressed=Mock() - resultSet=MagicMock() + lz4Compressed = Mock() + resultSet = MagicMock() resultSet.results.startRowOffset = 0 t_execute_resp = resp_type( status=Mock(), operationHandle=Mock(), directResults=ttypes.TSparkDirectResults( - operationStatus= Mock(), + operationStatus=Mock(), resultSetMetadata=ttypes.TGetResultSetMetadataResp( status=self.okay_status, resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET, schema=MagicMock(), arrowSchema=MagicMock(), - lz4Compressed=lz4Compressed), + lz4Compressed=lz4Compressed, + ), resultSet=resultSet, - closeOperation=None)) - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + closeOperation=None, + ), + ) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) - execute_response = thrift_backend._handle_execute_response(t_execute_resp, Mock()) + execute_response = thrift_backend._handle_execute_response( + t_execute_resp, Mock() + ) self.assertEqual(execute_response.lz4_compressed, lz4Compressed) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_handle_execute_response_checks_operation_state_in_polls(self, tcli_service_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_handle_execute_response_checks_operation_state_in_polls( + self, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value error_resp = ttypes.TGetOperationStatusResp( status=self.okay_status, operationState=ttypes.TOperationState.ERROR_STATE, - errorMessage="some information about the error") + errorMessage="some information about the error", + ) closed_resp = ttypes.TGetOperationStatusResp( - status=self.okay_status, operationState=ttypes.TOperationState.CLOSED_STATE) + status=self.okay_status, operationState=ttypes.TOperationState.CLOSED_STATE + ) - for op_state_resp, exec_resp_type in itertools.product([error_resp, closed_resp], - self.execute_response_types): - with self.subTest(op_state_resp=op_state_resp, exec_resp_type=exec_resp_type): + for op_state_resp, exec_resp_type in itertools.product( + [error_resp, closed_resp], self.execute_response_types + ): + with self.subTest( + op_state_resp=op_state_resp, exec_resp_type=exec_resp_type + ): tcli_service_instance = tcli_service_class.return_value t_execute_resp = exec_resp_type( status=self.okay_status, directResults=None, - operationHandle=self.operation_handle) + operationHandle=self.operation_handle, + ) tcli_service_instance.GetOperationStatus.return_value = op_state_resp - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(DatabaseError) as cm: thrift_backend._handle_execute_response(t_execute_resp, Mock()) if op_state_resp.errorMessage: self.assertIn(op_state_resp.errorMessage, str(cm.exception)) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_get_status_uses_display_message_if_available(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value @@ -387,21 +695,34 @@ def test_get_status_uses_display_message_if_available(self, tcli_service_class): operationState=ttypes.TOperationState.ERROR_STATE, errorMessage="foo", displayMessage=display_message, - diagnosticInfo=diagnostic_info) + diagnosticInfo=diagnostic_info, + ) t_execute_resp = ttypes.TExecuteStatementResp( - status=self.okay_status, directResults=None, operationHandle=self.operation_handle) - tcli_service_instance.GetOperationStatus.return_value = t_get_operation_status_resp + status=self.okay_status, + directResults=None, + operationHandle=self.operation_handle, + ) + tcli_service_instance.GetOperationStatus.return_value = ( + t_get_operation_status_resp + ) tcli_service_instance.ExecuteStatement.return_value = t_execute_resp - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(DatabaseError) as cm: thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock()) self.assertEqual(display_message, str(cm.exception)) self.assertIn(diagnostic_info, str(cm.exception.message_with_context())) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_direct_results_uses_display_message_if_available(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value @@ -412,7 +733,8 @@ def test_direct_results_uses_display_message_if_available(self, tcli_service_cla operationState=ttypes.TOperationState.ERROR_STATE, errorMessage="foo", displayMessage=display_message, - diagnosticInfo=diagnostic_info) + diagnosticInfo=diagnostic_info, + ) t_execute_resp = ttypes.TExecuteStatementResp( status=self.okay_status, @@ -420,11 +742,20 @@ def test_direct_results_uses_display_message_if_available(self, tcli_service_cla operationStatus=t_get_operation_status_resp, resultSetMetadata=None, resultSet=None, - closeOperation=None)) + closeOperation=None, + ), + ) tcli_service_instance.ExecuteStatement.return_value = t_execute_resp - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(DatabaseError) as cm: thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock()) @@ -436,18 +767,26 @@ def test_handle_execute_response_checks_direct_results_for_error_statuses(self): resp_1 = resp_type( status=self.okay_status, directResults=ttypes.TSparkDirectResults( - operationStatus=ttypes.TGetOperationStatusResp(status=self.bad_status), + operationStatus=ttypes.TGetOperationStatusResp( + status=self.bad_status + ), resultSetMetadata=None, resultSet=None, - closeOperation=None)) + closeOperation=None, + ), + ) resp_2 = resp_type( status=self.okay_status, directResults=ttypes.TSparkDirectResults( operationStatus=None, - resultSetMetadata=ttypes.TGetResultSetMetadataResp(status=self.bad_status), + resultSetMetadata=ttypes.TGetResultSetMetadataResp( + status=self.bad_status + ), resultSet=None, - closeOperation=None)) + closeOperation=None, + ), + ) resp_3 = resp_type( status=self.okay_status, @@ -455,7 +794,9 @@ def test_handle_execute_response_checks_direct_results_for_error_statuses(self): operationStatus=None, resultSetMetadata=None, resultSet=ttypes.TFetchResultsResp(status=self.bad_status), - closeOperation=None)) + closeOperation=None, + ), + ) resp_4 = resp_type( status=self.okay_status, @@ -463,18 +804,29 @@ def test_handle_execute_response_checks_direct_results_for_error_statuses(self): operationStatus=None, resultSetMetadata=None, resultSet=None, - closeOperation=ttypes.TCloseOperationResp(status=self.bad_status))) + closeOperation=ttypes.TCloseOperationResp(status=self.bad_status), + ), + ) for error_resp in [resp_1, resp_2, resp_3, resp_4]: with self.subTest(error_resp=error_resp): - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(DatabaseError) as cm: thrift_backend._handle_execute_response(error_resp, Mock()) self.assertIn("this is a bad error", str(cm.exception)) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_handle_execute_response_can_handle_without_direct_results(self, tcli_service_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_handle_execute_response_can_handle_without_direct_results( + self, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value for resp_type in self.execute_response_types: @@ -497,17 +849,33 @@ def test_handle_execute_response_can_handle_without_direct_results(self, tcli_se ) op_state_3 = ttypes.TGetOperationStatusResp( - status=self.okay_status, operationState=ttypes.TOperationState.FINISHED_STATE) + status=self.okay_status, + operationState=ttypes.TOperationState.FINISHED_STATE, + ) - tcli_service_instance.GetResultSetMetadata.return_value = self.metadata_resp + tcli_service_instance.GetResultSetMetadata.return_value = ( + self.metadata_resp + ) tcli_service_instance.GetOperationStatus.side_effect = [ - op_state_1, op_state_2, op_state_3 + op_state_1, + op_state_2, + op_state_3, ] - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) results_message_response = thrift_backend._handle_execute_response( - execute_resp, Mock()) - self.assertEqual(results_message_response.status, - ttypes.TOperationState.FINISHED_STATE) + execute_resp, Mock() + ) + self.assertEqual( + results_message_response.status, + ttypes.TOperationState.FINISHED_STATE, + ) def test_handle_execute_response_can_handle_with_direct_results(self): result_set_metadata_mock = Mock() @@ -519,16 +887,25 @@ def test_handle_execute_response_can_handle_with_direct_results(self): ), resultSetMetadata=result_set_metadata_mock, resultSet=Mock(), - closeOperation=Mock()) + closeOperation=Mock(), + ) for resp_type in self.execute_response_types: with self.subTest(resp_type=resp_type): execute_resp = resp_type( status=self.okay_status, directResults=direct_results_message, - operationHandle=self.operation_handle) + operationHandle=self.operation_handle, + ) - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend._results_message_to_execute_response = Mock() thrift_backend._handle_execute_response(execute_resp, Mock()) @@ -538,7 +915,7 @@ def test_handle_execute_response_can_handle_with_direct_results(self): ttypes.TOperationState.FINISHED_STATE, ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_use_arrow_schema_if_available(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value arrow_schema_mock = MagicMock(name="Arrow schema mock") @@ -548,7 +925,8 @@ def test_use_arrow_schema_if_available(self, tcli_service_class): status=self.okay_status, resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET, schema=hive_schema_mock, - arrowSchema=arrow_schema_mock) + arrowSchema=arrow_schema_mock, + ) t_execute_resp = ttypes.TExecuteStatementResp( status=self.okay_status, @@ -556,13 +934,17 @@ def test_use_arrow_schema_if_available(self, tcli_service_class): operationHandle=self.operation_handle, ) - tcli_service_instance.GetResultSetMetadata.return_value = t_get_result_set_metadata_resp + tcli_service_instance.GetResultSetMetadata.return_value = ( + t_get_result_set_metadata_resp + ) thrift_backend = self._make_fake_thrift_backend() - execute_response = thrift_backend._handle_execute_response(t_execute_resp, Mock()) + execute_response = thrift_backend._handle_execute_response( + t_execute_resp, Mock() + ) self.assertEqual(execute_response.arrow_schema_bytes, arrow_schema_mock) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_fall_back_to_hive_schema_if_no_arrow_schema(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value hive_schema_mock = MagicMock(name="Hive schema mock") @@ -571,7 +953,8 @@ def test_fall_back_to_hive_schema_if_no_arrow_schema(self, tcli_service_class): status=self.okay_status, resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET, arrowSchema=None, - schema=hive_schema_mock) + schema=hive_schema_mock, + ) t_execute_resp = ttypes.TExecuteStatementResp( status=self.okay_status, @@ -583,14 +966,21 @@ def test_fall_back_to_hive_schema_if_no_arrow_schema(self, tcli_service_class): thrift_backend = self._make_fake_thrift_backend() thrift_backend._handle_execute_response(t_execute_resp, Mock()) - self.assertEqual(hive_schema_mock, - thrift_backend._hive_schema_to_arrow_schema.call_args[0][0]) + self.assertEqual( + hive_schema_mock, + thrift_backend._hive_schema_to_arrow_schema.call_args[0][0], + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch( + "databricks.sql.utils.ResultSetQueueFactory.build_queue", return_value=Mock() + ) + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_handle_execute_response_reads_has_more_rows_in_direct_results( - self, tcli_service_class): - for has_more_rows, resp_type in itertools.product([True, False], - self.execute_response_types): + self, tcli_service_class, build_queue + ): + for has_more_rows, resp_type in itertools.product( + [True, False], self.execute_response_types + ): with self.subTest(has_more_rows=has_more_rows, resp_type=resp_type): tcli_service_instance = tcli_service_class.return_value results_mock = Mock() @@ -606,24 +996,35 @@ def test_handle_execute_response_reads_has_more_rows_in_direct_results( hasMoreRows=has_more_rows, results=results_mock, ), - closeOperation=Mock()) + closeOperation=Mock(), + ) execute_resp = resp_type( status=self.okay_status, directResults=direct_results_message, - operationHandle=self.operation_handle) + operationHandle=self.operation_handle, + ) - tcli_service_instance.GetResultSetMetadata.return_value = self.metadata_resp + tcli_service_instance.GetResultSetMetadata.return_value = ( + self.metadata_resp + ) thrift_backend = self._make_fake_thrift_backend() - execute_response = thrift_backend._handle_execute_response(execute_resp, Mock()) + execute_response = thrift_backend._handle_execute_response( + execute_resp, Mock() + ) self.assertEqual(has_more_rows, execute_response.has_more_rows) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch( + "databricks.sql.utils.ResultSetQueueFactory.build_queue", return_value=Mock() + ) + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_handle_execute_response_reads_has_more_rows_in_result_response( - self, tcli_service_class): - for has_more_rows, resp_type in itertools.product([True, False], - self.execute_response_types): + self, tcli_service_class, build_queue + ): + for has_more_rows, resp_type in itertools.product( + [True, False], self.execute_response_types + ): with self.subTest(has_more_rows=has_more_rows, resp_type=resp_type): tcli_service_instance = tcli_service_class.return_value results_mock = MagicMock() @@ -632,22 +1033,31 @@ def test_handle_execute_response_reads_has_more_rows_in_result_response( execute_resp = resp_type( status=self.okay_status, directResults=None, - operationHandle=self.operation_handle) + operationHandle=self.operation_handle, + ) fetch_results_resp = ttypes.TFetchResultsResp( status=self.okay_status, hasMoreRows=has_more_rows, results=results_mock, + resultSetMetadata=ttypes.TGetResultSetMetadataResp( + resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET + ), ) operation_status_resp = ttypes.TGetOperationStatusResp( status=self.okay_status, operationState=ttypes.TOperationState.FINISHED_STATE, - errorMessage="some information about the error") + errorMessage="some information about the error", + ) tcli_service_instance.FetchResults.return_value = fetch_results_resp - tcli_service_instance.GetOperationStatus.return_value = operation_status_resp - tcli_service_instance.GetResultSetMetadata.return_value = self.metadata_resp + tcli_service_instance.GetOperationStatus.return_value = ( + operation_status_resp + ) + tcli_service_instance.GetResultSetMetadata.return_value = ( + self.metadata_resp + ) thrift_backend = self._make_fake_thrift_backend() thrift_backend._handle_execute_response(execute_resp, Mock()) @@ -658,11 +1068,12 @@ def test_handle_execute_response_reads_has_more_rows_in_result_response( expected_row_start_offset=0, lz4_compressed=False, arrow_schema_bytes=Mock(), - description=Mock()) + description=Mock(), + ) self.assertEqual(has_more_rows, has_more_rows_resp) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_arrow_batches_row_count_are_respected(self, tcli_service_class): # make some semi-real arrow batches and check the number of rows is correct in the queue tcli_service_instance = tcli_service_class.return_value @@ -673,17 +1084,36 @@ def test_arrow_batches_row_count_are_respected(self, tcli_service_class): startRowOffset=0, rows=[], arrowBatches=[ - ttypes.TSparkArrowBatch(batch=bytearray(), rowCount=15) for _ in range(10) - ])) + ttypes.TSparkArrowBatch(batch=bytearray(), rowCount=15) + for _ in range(10) + ], + ), + resultSetMetadata=ttypes.TGetResultSetMetadataResp( + resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET + ), + ) tcli_service_instance.FetchResults.return_value = t_fetch_results_resp - schema = pyarrow.schema([ - pyarrow.field("column1", pyarrow.int32()), - pyarrow.field("column2", pyarrow.string()), - pyarrow.field("column3", pyarrow.float64()), - pyarrow.field("column3", pyarrow.binary()) - ]).serialize().to_pybytes() - - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + schema = ( + pyarrow.schema( + [ + pyarrow.field("column1", pyarrow.int32()), + pyarrow.field("column2", pyarrow.string()), + pyarrow.field("column3", pyarrow.float64()), + pyarrow.field("column3", pyarrow.binary()), + ] + ) + .serialize() + .to_pybytes() + ) + + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) arrow_queue, has_more_results = thrift_backend.fetch_results( op_handle=Mock(), max_rows=1, @@ -691,16 +1121,26 @@ def test_arrow_batches_row_count_are_respected(self, tcli_service_class): expected_row_start_offset=0, lz4_compressed=False, arrow_schema_bytes=schema, - description=MagicMock()) + description=MagicMock(), + ) self.assertEqual(arrow_queue.n_valid_rows, 15 * 10) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_execute_statement_calls_client_and_handle_execute_response(self, tcli_service_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_execute_statement_calls_client_and_handle_execute_response( + self, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.ExecuteStatement.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -711,14 +1151,25 @@ def test_execute_statement_calls_client_and_handle_execute_response(self, tcli_s self.assertEqual(req.getDirectResults, get_direct_results) self.assertEqual(req.statement, "foo") # Check response handling - thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) + thrift_backend._handle_execute_response.assert_called_with( + response, cursor_mock + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_get_catalogs_calls_client_and_handle_execute_response(self, tcli_service_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_get_catalogs_calls_client_and_handle_execute_response( + self, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.GetCatalogs.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -728,14 +1179,25 @@ def test_get_catalogs_calls_client_and_handle_execute_response(self, tcli_servic get_direct_results = ttypes.TSparkGetDirectResults(maxRows=100, maxBytes=200) self.assertEqual(req.getDirectResults, get_direct_results) # Check response handling - thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) + thrift_backend._handle_execute_response.assert_called_with( + response, cursor_mock + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_get_schemas_calls_client_and_handle_execute_response(self, tcli_service_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_get_schemas_calls_client_and_handle_execute_response( + self, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.GetSchemas.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -745,7 +1207,8 @@ def test_get_schemas_calls_client_and_handle_execute_response(self, tcli_service 200, cursor_mock, catalog_name="catalog_pattern", - schema_name="schema_pattern") + schema_name="schema_pattern", + ) # Check call to client req = tcli_service_instance.GetSchemas.call_args[0][0] get_direct_results = ttypes.TSparkGetDirectResults(maxRows=100, maxBytes=200) @@ -753,14 +1216,25 @@ def test_get_schemas_calls_client_and_handle_execute_response(self, tcli_service self.assertEqual(req.catalogName, "catalog_pattern") self.assertEqual(req.schemaName, "schema_pattern") # Check response handling - thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) + thrift_backend._handle_execute_response.assert_called_with( + response, cursor_mock + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_get_tables_calls_client_and_handle_execute_response(self, tcli_service_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_get_tables_calls_client_and_handle_execute_response( + self, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.GetTables.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -772,7 +1246,8 @@ def test_get_tables_calls_client_and_handle_execute_response(self, tcli_service_ catalog_name="catalog_pattern", schema_name="schema_pattern", table_name="table_pattern", - table_types=["type1", "type2"]) + table_types=["type1", "type2"], + ) # Check call to client req = tcli_service_instance.GetTables.call_args[0][0] get_direct_results = ttypes.TSparkGetDirectResults(maxRows=100, maxBytes=200) @@ -782,14 +1257,25 @@ def test_get_tables_calls_client_and_handle_execute_response(self, tcli_service_ self.assertEqual(req.tableName, "table_pattern") self.assertEqual(req.tableTypes, ["type1", "type2"]) # Check response handling - thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) + thrift_backend._handle_execute_response.assert_called_with( + response, cursor_mock + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_get_columns_calls_client_and_handle_execute_response(self, tcli_service_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_get_columns_calls_client_and_handle_execute_response( + self, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.GetColumns.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -801,7 +1287,8 @@ def test_get_columns_calls_client_and_handle_execute_response(self, tcli_service catalog_name="catalog_pattern", schema_name="schema_pattern", table_name="table_pattern", - column_name="column_pattern") + column_name="column_pattern", + ) # Check call to client req = tcli_service_instance.GetColumns.call_args[0][0] get_direct_results = ttypes.TSparkGetDirectResults(maxRows=100, maxBytes=200) @@ -811,41 +1298,73 @@ def test_get_columns_calls_client_and_handle_execute_response(self, tcli_service self.assertEqual(req.tableName, "table_pattern") self.assertEqual(req.columnName, "column_pattern") # Check response handling - thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) + thrift_backend._handle_execute_response.assert_called_with( + response, cursor_mock + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_open_session_user_provided_session_id_optional(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value tcli_service_instance.OpenSession.return_value = self.open_session_resp - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend.open_session({}, None, None) self.assertEqual(len(tcli_service_instance.OpenSession.call_args_list), 1) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_op_handle_respected_in_close_command(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend.close_command(self.operation_handle) - self.assertEqual(tcli_service_instance.CloseOperation.call_args[0][0].operationHandle, - self.operation_handle) + self.assertEqual( + tcli_service_instance.CloseOperation.call_args[0][0].operationHandle, + self.operation_handle, + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_session_handle_respected_in_close_session(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) thrift_backend.close_session(self.session_handle) - self.assertEqual(tcli_service_instance.CloseSession.call_args[0][0].sessionHandle, - self.session_handle) + self.assertEqual( + tcli_service_instance.CloseSession.call_args[0][0].sessionHandle, + self.session_handle, + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_non_arrow_non_column_based_set_triggers_exception(self, tcli_service_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_non_arrow_non_column_based_set_triggers_exception( + self, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value results_mock = Mock() results_mock.startRowOffset = 0 execute_statement_resp = ttypes.TExecuteStatementResp( - status=self.okay_status, directResults=None, operationHandle=self.operation_handle) + status=self.okay_status, + directResults=None, + operationHandle=self.operation_handle, + ) metadata_resp = ttypes.TGetResultSetMetadataResp( status=self.okay_status, @@ -855,7 +1374,8 @@ def test_non_arrow_non_column_based_set_triggers_exception(self, tcli_service_cl operation_status_resp = ttypes.TGetOperationStatusResp( status=self.okay_status, operationState=ttypes.TOperationState.FINISHED_STATE, - errorMessage="some information about the error") + errorMessage="some information about the error", + ) tcli_service_instance.ExecuteStatement.return_value = execute_statement_resp tcli_service_instance.GetResultSetMetadata.return_value = metadata_resp @@ -864,19 +1384,36 @@ def test_non_arrow_non_column_based_set_triggers_exception(self, tcli_service_cl with self.assertRaises(OperationalError) as cm: thrift_backend.execute_command("foo", Mock(), 100, 100, Mock(), Mock()) - self.assertIn("Expected results to be in Arrow or column based format", str(cm.exception)) + self.assertIn( + "Expected results to be in Arrow or column based format", str(cm.exception) + ) def test_create_arrow_table_raises_error_for_unsupported_type(self): t_row_set = ttypes.TRowSet() - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(OperationalError): thrift_backend._create_arrow_table(t_row_set, Mock(), None, Mock()) - @patch.object(ThriftBackend, "_convert_arrow_based_set_to_arrow_table") - @patch.object(ThriftBackend, "_convert_column_based_set_to_arrow_table") - def test_create_arrow_table_calls_correct_conversion_method(self, convert_col_mock, - convert_arrow_mock): - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + @patch("databricks.sql.thrift_backend.convert_arrow_based_set_to_arrow_table") + @patch("databricks.sql.thrift_backend.convert_column_based_set_to_arrow_table") + def test_create_arrow_table_calls_correct_conversion_method( + self, convert_col_mock, convert_arrow_mock + ): + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) convert_arrow_mock.return_value = (MagicMock(), Mock()) convert_col_mock.return_value = (MagicMock(), Mock()) @@ -887,49 +1424,79 @@ def test_create_arrow_table_calls_correct_conversion_method(self, convert_col_mo description = Mock() t_col_set = ttypes.TRowSet(columns=cols) - thrift_backend._create_arrow_table(t_col_set, lz4_compressed, schema, description) + thrift_backend._create_arrow_table( + t_col_set, lz4_compressed, schema, description + ) convert_arrow_mock.assert_not_called() convert_col_mock.assert_called_once_with(cols, description) t_arrow_set = ttypes.TRowSet(arrowBatches=arrow_batches) thrift_backend._create_arrow_table(t_arrow_set, lz4_compressed, schema, Mock()) - convert_arrow_mock.assert_called_once_with(arrow_batches, lz4_compressed, schema) + convert_arrow_mock.assert_called_once_with( + arrow_batches, lz4_compressed, schema + ) @patch("lz4.frame.decompress") @patch("pyarrow.ipc.open_stream") - def test_convert_arrow_based_set_to_arrow_table(self, open_stream_mock, lz4_decompress_mock): - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) - - lz4_decompress_mock.return_value = bytearray('Testing','utf-8') - - schema = pyarrow.schema([ - pyarrow.field("column1", pyarrow.int32()), - ]).serialize().to_pybytes() - - arrow_batches = [ttypes.TSparkArrowBatch(batch=bytearray('Testing','utf-8'), rowCount=1) for _ in range(10)] - thrift_backend._convert_arrow_based_set_to_arrow_table(arrow_batches, False, schema) + def test_convert_arrow_based_set_to_arrow_table( + self, open_stream_mock, lz4_decompress_mock + ): + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) + + lz4_decompress_mock.return_value = bytearray("Testing", "utf-8") + + schema = ( + pyarrow.schema( + [ + pyarrow.field("column1", pyarrow.int32()), + ] + ) + .serialize() + .to_pybytes() + ) + + arrow_batches = [ + ttypes.TSparkArrowBatch(batch=bytearray("Testing", "utf-8"), rowCount=1) + for _ in range(10) + ] + utils.convert_arrow_based_set_to_arrow_table(arrow_batches, False, schema) lz4_decompress_mock.assert_not_called() - thrift_backend._convert_arrow_based_set_to_arrow_table(arrow_batches, True, schema) + utils.convert_arrow_based_set_to_arrow_table(arrow_batches, True, schema) lz4_decompress_mock.assert_called() - def test_convert_column_based_set_to_arrow_table_without_nulls(self): # Deliberately duplicate the column name to check that dups work field_names = ["column1", "column2", "column3", "column3"] - description = [(name, ) for name in field_names] + description = [(name,) for name in field_names] t_cols = [ ttypes.TColumn(i32Val=ttypes.TI32Column(values=[1, 2, 3], nulls=bytes(1))), ttypes.TColumn( - stringVal=ttypes.TStringColumn(values=["s1", "s2", "s3"], nulls=bytes(1))), - ttypes.TColumn(doubleVal=ttypes.TDoubleColumn(values=[1.15, 2.2, 3.3], nulls=bytes(1))), + stringVal=ttypes.TStringColumn( + values=["s1", "s2", "s3"], nulls=bytes(1) + ) + ), + ttypes.TColumn( + doubleVal=ttypes.TDoubleColumn(values=[1.15, 2.2, 3.3], nulls=bytes(1)) + ), ttypes.TColumn( - binaryVal=ttypes.TBinaryColumn(values=[b'\x11', b'\x22', b'\x33'], nulls=bytes(1))) + binaryVal=ttypes.TBinaryColumn( + values=[b"\x11", b"\x22", b"\x33"], nulls=bytes(1) + ) + ), ] - arrow_table, n_rows = ThriftBackend._convert_column_based_set_to_arrow_table( - t_cols, description) + arrow_table, n_rows = utils.convert_column_based_set_to_arrow_table( + t_cols, description + ) self.assertEqual(n_rows, 3) # Check schema, column names and types @@ -947,48 +1514,68 @@ def test_convert_column_based_set_to_arrow_table_without_nulls(self): self.assertEqual(arrow_table.column(0).to_pylist(), [1, 2, 3]) self.assertEqual(arrow_table.column(1).to_pylist(), ["s1", "s2", "s3"]) self.assertEqual(arrow_table.column(2).to_pylist(), [1.15, 2.2, 3.3]) - self.assertEqual(arrow_table.column(3).to_pylist(), [b'\x11', b'\x22', b'\x33']) + self.assertEqual(arrow_table.column(3).to_pylist(), [b"\x11", b"\x22", b"\x33"]) def test_convert_column_based_set_to_arrow_table_with_nulls(self): field_names = ["column1", "column2", "column3", "column3"] - description = [(name, ) for name in field_names] + description = [(name,) for name in field_names] t_cols = [ - ttypes.TColumn(i32Val=ttypes.TI32Column(values=[1, 2, 3], nulls=bytes([1]))), ttypes.TColumn( - stringVal=ttypes.TStringColumn(values=["s1", "s2", "s3"], nulls=bytes([2]))), + i32Val=ttypes.TI32Column(values=[1, 2, 3], nulls=bytes([1])) + ), + ttypes.TColumn( + stringVal=ttypes.TStringColumn( + values=["s1", "s2", "s3"], nulls=bytes([2]) + ) + ), ttypes.TColumn( - doubleVal=ttypes.TDoubleColumn(values=[1.15, 2.2, 3.3], nulls=bytes([4]))), + doubleVal=ttypes.TDoubleColumn( + values=[1.15, 2.2, 3.3], nulls=bytes([4]) + ) + ), ttypes.TColumn( binaryVal=ttypes.TBinaryColumn( - values=[b'\x11', b'\x22', b'\x33'], nulls=bytes([3]))) + values=[b"\x11", b"\x22", b"\x33"], nulls=bytes([3]) + ) + ), ] - arrow_table, n_rows = ThriftBackend._convert_column_based_set_to_arrow_table( - t_cols, description) + arrow_table, n_rows = utils.convert_column_based_set_to_arrow_table( + t_cols, description + ) self.assertEqual(n_rows, 3) # Check data self.assertEqual(arrow_table.column(0).to_pylist(), [None, 2, 3]) self.assertEqual(arrow_table.column(1).to_pylist(), ["s1", None, "s3"]) self.assertEqual(arrow_table.column(2).to_pylist(), [1.15, 2.2, None]) - self.assertEqual(arrow_table.column(3).to_pylist(), [None, None, b'\x33']) + self.assertEqual(arrow_table.column(3).to_pylist(), [None, None, b"\x33"]) def test_convert_column_based_set_to_arrow_table_uses_types_from_col_set(self): field_names = ["column1", "column2", "column3", "column3"] - description = [(name, ) for name in field_names] + description = [(name,) for name in field_names] t_cols = [ ttypes.TColumn(i32Val=ttypes.TI32Column(values=[1, 2, 3], nulls=bytes(1))), ttypes.TColumn( - stringVal=ttypes.TStringColumn(values=["s1", "s2", "s3"], nulls=bytes(1))), - ttypes.TColumn(doubleVal=ttypes.TDoubleColumn(values=[1.15, 2.2, 3.3], nulls=bytes(1))), + stringVal=ttypes.TStringColumn( + values=["s1", "s2", "s3"], nulls=bytes(1) + ) + ), ttypes.TColumn( - binaryVal=ttypes.TBinaryColumn(values=[b'\x11', b'\x22', b'\x33'], nulls=bytes(1))) + doubleVal=ttypes.TDoubleColumn(values=[1.15, 2.2, 3.3], nulls=bytes(1)) + ), + ttypes.TColumn( + binaryVal=ttypes.TBinaryColumn( + values=[b"\x11", b"\x22", b"\x33"], nulls=bytes(1) + ) + ), ] - arrow_table, n_rows = ThriftBackend._convert_column_based_set_to_arrow_table( - t_cols, description) + arrow_table, n_rows = utils.convert_column_based_set_to_arrow_table( + t_cols, description + ) self.assertEqual(n_rows, 3) # Check schema, column names and types @@ -1006,9 +1593,9 @@ def test_convert_column_based_set_to_arrow_table_uses_types_from_col_set(self): self.assertEqual(arrow_table.column(0).to_pylist(), [1, 2, 3]) self.assertEqual(arrow_table.column(1).to_pylist(), ["s1", "s2", "s3"]) self.assertEqual(arrow_table.column(2).to_pylist(), [1.15, 2.2, 3.3]) - self.assertEqual(arrow_table.column(3).to_pylist(), [b'\x11', b'\x22', b'\x33']) + self.assertEqual(arrow_table.column(3).to_pylist(), [b"\x11", b"\x22", b"\x33"]) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_cancel_command_uses_active_op_handle(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value @@ -1016,8 +1603,10 @@ def test_cancel_command_uses_active_op_handle(self, tcli_service_class): active_op_handle_mock = Mock() thrift_backend.cancel_command(active_op_handle_mock) - self.assertEqual(tcli_service_instance.CancelOperation.call_args[0][0].operationHandle, - active_op_handle_mock) + self.assertEqual( + tcli_service_instance.CancelOperation.call_args[0][0].operationHandle, + active_op_handle_mock, + ) def test_handle_execute_response_sets_active_op_handle(self): thrift_backend = self._make_fake_thrift_backend() @@ -1031,11 +1620,16 @@ def test_handle_execute_response_sets_active_op_handle(self): self.assertEqual(mock_resp.operationHandle, mock_cursor.active_op_handle) - @patch("thrift.transport.THttpClient.THttpClient") - @patch("databricks.sql.thrift_api.TCLIService.TCLIService.Client.GetOperationStatus") - @patch("databricks.sql.thrift_backend._retry_policy", new_callable=retry_policy_factory) + @patch("databricks.sql.auth.thrift_http_client.THttpClient") + @patch( + "databricks.sql.thrift_api.TCLIService.TCLIService.Client.GetOperationStatus" + ) + @patch( + "databricks.sql.thrift_backend._retry_policy", new_callable=retry_policy_factory + ) def test_make_request_will_retry_GetOperationStatus( - self, mock_retry_policy, mock_GetOperationStatus, t_transport_class): + self, mock_retry_policy, mock_GetOperationStatus, t_transport_class + ): import thrift, errno from databricks.sql.thrift_api.TCLIService.TCLIService import Client @@ -1044,7 +1638,9 @@ def test_make_request_will_retry_GetOperationStatus( this_gos_name = "GetOperationStatus" mock_GetOperationStatus.__name__ = this_gos_name - mock_GetOperationStatus.side_effect = OSError(errno.ETIMEDOUT, "Connection timed out") + mock_GetOperationStatus.side_effect = OSError( + errno.ETIMEDOUT, "Connection timed out" + ) protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(t_transport_class) client = Client(protocol) @@ -1059,22 +1655,32 @@ def test_make_request_will_retry_GetOperationStatus( thrift_backend = ThriftBackend( "foobar", 443, - "path", [], + "path", + [], auth_provider=AuthProvider(), + ssl_options=SSLOptions(), _retry_stop_after_attempts_count=EXPECTED_RETRIES, - _retry_delay_default=1) - + _retry_delay_default=1, + ) with self.assertRaises(RequestError) as cm: thrift_backend.make_request(client.GetOperationStatus, req) - self.assertEqual(NoRetryReason.OUT_OF_ATTEMPTS.value, cm.exception.context["no-retry-reason"]) - self.assertEqual(f'{EXPECTED_RETRIES}/{EXPECTED_RETRIES}', cm.exception.context["attempt"]) + self.assertEqual( + NoRetryReason.OUT_OF_ATTEMPTS.value, cm.exception.context["no-retry-reason"] + ) + self.assertEqual( + f"{EXPECTED_RETRIES}/{EXPECTED_RETRIES}", cm.exception.context["attempt"] + ) # Unusual OSError code - mock_GetOperationStatus.side_effect = OSError(errno.EEXIST, "File does not exist") + mock_GetOperationStatus.side_effect = OSError( + errno.EEXIST, "File does not exist" + ) - with self.assertLogs("databricks.sql.thrift_backend", level=logging.WARNING) as cm: + with self.assertLogs( + "databricks.sql.thrift_backend", level=logging.WARNING + ) as cm: with self.assertRaises(RequestError): thrift_backend.make_request(client.GetOperationStatus, req) @@ -1085,27 +1691,69 @@ def test_make_request_will_retry_GetOperationStatus( self.assertEqual(cm.output[1], cm.output[0]) # The warnings should include this text - self.assertIn(f"{this_gos_name} failed with code {errno.EEXIST} and will attempt to retry", cm.output[0]) + self.assertIn( + f"{this_gos_name} failed with code {errno.EEXIST} and will attempt to retry", + cm.output[0], + ) + @patch( + "databricks.sql.thrift_api.TCLIService.TCLIService.Client.GetOperationStatus" + ) + @patch( + "databricks.sql.thrift_backend._retry_policy", new_callable=retry_policy_factory + ) + def test_make_request_will_retry_GetOperationStatus_for_http_error( + self, mock_retry_policy, mock_gos + ): - @patch("thrift.transport.THttpClient.THttpClient") - def test_make_request_wont_retry_if_headers_not_present(self, t_transport_class): - t_transport_instance = t_transport_class.return_value - t_transport_instance.code = 429 - t_transport_instance.headers = {"foo": "bar"} - mock_method = Mock() - mock_method.__name__ = "method name" - mock_method.side_effect = Exception("This method fails") + import urllib3.exceptions - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + mock_gos.side_effect = urllib3.exceptions.HTTPError("Read timed out") - with self.assertRaises(OperationalError) as cm: - thrift_backend.make_request(mock_method, Mock()) + import thrift, errno + from databricks.sql.thrift_api.TCLIService.TCLIService import Client + from databricks.sql.exc import RequestError + from databricks.sql.utils import NoRetryReason + from databricks.sql.auth.thrift_http_client import THttpClient - self.assertIn("This method fails", str(cm.exception.message_with_context())) + this_gos_name = "GetOperationStatus" + mock_gos.__name__ = this_gos_name + + protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(THttpClient) + client = Client(protocol) + + req = ttypes.TGetOperationStatusReq( + operationHandle=self.operation_handle, + getProgressUpdate=False, + ) + + EXPECTED_RETRIES = 2 + + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + _retry_stop_after_attempts_count=EXPECTED_RETRIES, + _retry_delay_default=1, + ) + + with self.assertRaises(RequestError) as cm: + thrift_backend.make_request(client.GetOperationStatus, req) + + self.assertEqual( + NoRetryReason.OUT_OF_ATTEMPTS.value, cm.exception.context["no-retry-reason"] + ) + self.assertEqual( + f"{EXPECTED_RETRIES}/{EXPECTED_RETRIES}", cm.exception.context["attempt"] + ) @patch("thrift.transport.THttpClient.THttpClient") - def test_make_request_wont_retry_if_error_code_not_429_or_503(self, t_transport_class): + def test_make_request_wont_retry_if_error_code_not_429_or_503( + self, t_transport_class + ): t_transport_instance = t_transport_class.return_value t_transport_instance.code = 430 t_transport_instance.headers = {"Retry-After": "1"} @@ -1113,7 +1761,14 @@ def test_make_request_wont_retry_if_error_code_not_429_or_503(self, t_transport_ mock_method.__name__ = "method name" mock_method.side_effect = Exception("This method fails") - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(OperationalError) as cm: thrift_backend.make_request(mock_method, Mock()) @@ -1121,9 +1776,12 @@ def test_make_request_wont_retry_if_error_code_not_429_or_503(self, t_transport_ self.assertIn("This method fails", str(cm.exception.message_with_context())) @patch("databricks.sql.auth.thrift_http_client.THttpClient") - @patch("databricks.sql.thrift_backend._retry_policy", new_callable=retry_policy_factory) + @patch( + "databricks.sql.thrift_backend._retry_policy", new_callable=retry_policy_factory + ) def test_make_request_will_retry_stop_after_attempts_count_if_retryable( - self, mock_retry_policy, t_transport_class): + self, mock_retry_policy, t_transport_class + ): t_transport_instance = t_transport_class.return_value t_transport_instance.code = 429 t_transport_instance.headers = {"Retry-After": "0"} @@ -1134,11 +1792,14 @@ def test_make_request_will_retry_stop_after_attempts_count_if_retryable( thrift_backend = ThriftBackend( "foobar", 443, - "path", [], + "path", + [], auth_provider=AuthProvider(), + ssl_options=SSLOptions(), _retry_stop_after_attempts_count=14, _retry_delay_max=0, - _retry_delay_min=0) + _retry_delay_min=0, + ) with self.assertRaises(OperationalError) as cm: thrift_backend.make_request(mock_method, Mock()) @@ -1149,23 +1810,40 @@ def test_make_request_will_retry_stop_after_attempts_count_if_retryable( self.assertEqual(mock_method.call_count, 14) @patch("databricks.sql.auth.thrift_http_client.THttpClient") - def test_make_request_will_read_error_message_headers_if_set(self, t_transport_class): + def test_make_request_will_read_error_message_headers_if_set( + self, t_transport_class + ): t_transport_instance = t_transport_class.return_value mock_method = Mock() mock_method.__name__ = "method name" mock_method.side_effect = Exception("This method fails") - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) - error_headers = [[("x-thriftserver-error-message", "thrift server error message")], - [("x-databricks-error-or-redirect-message", "databricks error message")], - [("x-databricks-error-or-redirect-message", "databricks error message"), - ("x-databricks-reason-phrase", "databricks error reason")], - [("x-thriftserver-error-message", "thrift server error message"), - ("x-databricks-error-or-redirect-message", "databricks error message"), - ("x-databricks-reason-phrase", "databricks error reason")], - [("x-thriftserver-error-message", "thrift server error message"), - ("x-databricks-error-or-redirect-message", "databricks error message")]] + error_headers = [ + [("x-thriftserver-error-message", "thrift server error message")], + [("x-databricks-error-or-redirect-message", "databricks error message")], + [ + ("x-databricks-error-or-redirect-message", "databricks error message"), + ("x-databricks-reason-phrase", "databricks error reason"), + ], + [ + ("x-thriftserver-error-message", "thrift server error message"), + ("x-databricks-error-or-redirect-message", "databricks error message"), + ("x-databricks-reason-phrase", "databricks error reason"), + ], + [ + ("x-thriftserver-error-message", "thrift server error message"), + ("x-databricks-error-or-redirect-message", "databricks error message"), + ], + ] for headers in error_headers: t_transport_instance.headers = dict(headers) @@ -1176,16 +1854,23 @@ def test_make_request_will_read_error_message_headers_if_set(self, t_transport_c self.assertIn(header[1], str(cm.exception)) @staticmethod - def make_table_and_desc(height, n_decimal_cols, width, precision, scale, int_constant, - decimal_constant): + def make_table_and_desc( + height, n_decimal_cols, width, precision, scale, int_constant, decimal_constant + ): int_col = [int_constant for _ in range(height)] decimal_col = [decimal_constant for _ in range(height)] - data = OrderedDict({"col{}".format(i): int_col for i in range(width - n_decimal_cols)}) - decimals = OrderedDict({"col_dec{}".format(i): decimal_col for i in range(n_decimal_cols)}) + data = OrderedDict( + {"col{}".format(i): int_col for i in range(width - n_decimal_cols)} + ) + decimals = OrderedDict( + {"col_dec{}".format(i): decimal_col for i in range(n_decimal_cols)} + ) data.update(decimals) - int_desc = ([("", "int")] * (width - n_decimal_cols)) - decimal_desc = ([("", "decimal", None, None, precision, scale, None)] * n_decimal_cols) + int_desc = [("", "int")] * (width - n_decimal_cols) + decimal_desc = [ + ("", "decimal", None, None, precision, scale, None) + ] * n_decimal_cols description = int_desc + decimal_desc table = pyarrow.Table.from_pydict(data) @@ -1201,33 +1886,53 @@ def test_arrow_decimal_conversion(self): for n_decimal_cols in [0, 1, 10]: for height in [0, 1, 10]: with self.subTest(n_decimal_cols=n_decimal_cols, height=height): - table, description = self.make_table_and_desc(height, n_decimal_cols, width, - precision, scale, int_constant, - decimal_constant) - decimal_converted_table = ThriftBackend._convert_decimals_in_arrow_table( - table, description) + table, description = self.make_table_and_desc( + height, + n_decimal_cols, + width, + precision, + scale, + int_constant, + decimal_constant, + ) + decimal_converted_table = utils.convert_decimals_in_arrow_table( + table, description + ) for i in range(width): if height > 0: if i < width - n_decimal_cols: self.assertEqual( - decimal_converted_table.field(i).type, pyarrow.int64()) + decimal_converted_table.field(i).type, + pyarrow.int64(), + ) else: self.assertEqual( decimal_converted_table.field(i).type, - pyarrow.decimal128(precision=precision, scale=scale)) + pyarrow.decimal128( + precision=precision, scale=scale + ), + ) int_col = [int_constant for _ in range(height)] decimal_col = [Decimal(decimal_constant) for _ in range(height)] expected_result = OrderedDict( - {"col{}".format(i): int_col - for i in range(width - n_decimal_cols)}) + { + "col{}".format(i): int_col + for i in range(width - n_decimal_cols) + } + ) decimals = OrderedDict( - {"col_dec{}".format(i): decimal_col - for i in range(n_decimal_cols)}) + { + "col_dec{}".format(i): decimal_col + for i in range(n_decimal_cols) + } + ) expected_result.update(decimals) - self.assertEqual(decimal_converted_table.to_pydict(), expected_result) + self.assertEqual( + decimal_converted_table.to_pydict(), expected_result + ) @patch("thrift.transport.THttpClient.THttpClient") def test_retry_args_passthrough(self, mock_http_client): @@ -1235,32 +1940,51 @@ def test_retry_args_passthrough(self, mock_http_client): "_retry_delay_min": 6, "_retry_delay_max": 10, "_retry_stop_after_attempts_count": 1, - "_retry_stop_after_attempts_duration": 100 + "_retry_stop_after_attempts_duration": 100, } - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), **retry_delay_args) - for (arg, val) in retry_delay_args.items(): + backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + **retry_delay_args, + ) + for arg, val in retry_delay_args.items(): self.assertEqual(getattr(backend, arg), val) @patch("thrift.transport.THttpClient.THttpClient") def test_retry_args_bounding(self, mock_http_client): retry_delay_test_args_and_expected_values = {} - for (k, (_, _, min, max)) in databricks.sql.thrift_backend._retry_policy.items(): - retry_delay_test_args_and_expected_values[k] = ((min - 1, min), (max + 1, max)) + for k, (_, _, min, max) in databricks.sql.thrift_backend._retry_policy.items(): + retry_delay_test_args_and_expected_values[k] = ( + (min - 1, min), + (max + 1, max), + ) for i in range(2): retry_delay_args = { k: v[i][0] for (k, v) in retry_delay_test_args_and_expected_values.items() } - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), **retry_delay_args) + backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + **retry_delay_args, + ) retry_delay_expected_vals = { k: v[i][1] for (k, v) in retry_delay_test_args_and_expected_values.items() } - for (arg, val) in retry_delay_expected_vals.items(): + for arg, val in retry_delay_expected_vals.items(): self.assertEqual(getattr(backend, arg), val) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_configuration_passthrough(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value tcli_service_instance.OpenSession.return_value = self.open_session_resp @@ -1269,21 +1993,35 @@ def test_configuration_passthrough(self, tcli_client_class): "spark.thriftserver.arrowBasedRowSet.timestampAsString": "false", "foo": "bar", "baz": "True", - "42": "42" + "42": "42", } - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) backend.open_session(mock_config, None, None) open_session_req = tcli_client_class.return_value.OpenSession.call_args[0][0] self.assertEqual(open_session_req.configuration, expected_config) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_cant_set_timestamp_as_string_to_true(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value tcli_service_instance.OpenSession.return_value = self.open_session_resp mock_config = {"spark.thriftserver.arrowBasedRowSet.timestampAsString": True} - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(databricks.sql.Error) as cm: backend.open_session(mock_config, None, None) @@ -1295,42 +2033,71 @@ def _construct_open_session_with_namespace(self, can_use_multiple_cats, cat, sch status=self.okay_status, serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4, canUseMultipleCatalogs=can_use_multiple_cats, - initialNamespace=ttypes.TNamespace(catalogName=cat, schemaName=schem)) + initialNamespace=ttypes.TNamespace(catalogName=cat, schemaName=schem), + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_initial_namespace_passthrough_to_open_session(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) initial_cat_schem_args = [("cat", None), (None, "schem"), ("cat", "schem")] for cat, schem in initial_cat_schem_args: with self.subTest(cat=cat, schem=schem): - tcli_service_instance.OpenSession.return_value = \ + tcli_service_instance.OpenSession.return_value = ( self._construct_open_session_with_namespace(True, cat, schem) + ) backend.open_session({}, cat, schem) - open_session_req = tcli_client_class.return_value.OpenSession.call_args[0][0] + open_session_req = tcli_client_class.return_value.OpenSession.call_args[ + 0 + ][0] self.assertEqual(open_session_req.initialNamespace.catalogName, cat) self.assertEqual(open_session_req.initialNamespace.schemaName, schem) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_can_use_multiple_catalogs_is_set_in_open_session_req(self, tcli_client_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_can_use_multiple_catalogs_is_set_in_open_session_req( + self, tcli_client_class + ): tcli_service_instance = tcli_client_class.return_value tcli_service_instance.OpenSession.return_value = self.open_session_resp - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) backend.open_session({}, None, None) open_session_req = tcli_client_class.return_value.OpenSession.call_args[0][0] self.assertTrue(open_session_req.canUseMultipleCatalogs) - @patch("databricks.sql.thrift_backend.TCLIService.Client") - def test_can_use_multiple_catalogs_is_false_fails_with_initial_catalog(self, tcli_client_class): + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_can_use_multiple_catalogs_is_false_fails_with_initial_catalog( + self, tcli_client_class + ): tcli_service_instance = tcli_client_class.return_value - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) # If the initial catalog is set, but server returns canUseMultipleCatalogs=False, we # expect failure. If the initial catalog isn't set, then canUseMultipleCatalogs=False # is fine @@ -1338,48 +2105,63 @@ def test_can_use_multiple_catalogs_is_false_fails_with_initial_catalog(self, tcl passing_ns_args = [(None, None), (None, "schem")] for cat, schem in failing_ns_args: - tcli_service_instance.OpenSession.return_value = \ + tcli_service_instance.OpenSession.return_value = ( self._construct_open_session_with_namespace(False, cat, schem) + ) with self.assertRaises(InvalidServerResponseError) as cm: backend.open_session({}, cat, schem) - self.assertIn("server does not support multiple catalogs", str(cm.exception), - "incorrect error thrown for initial namespace {}".format((cat, schem))) + self.assertIn( + "server does not support multiple catalogs", + str(cm.exception), + "incorrect error thrown for initial namespace {}".format((cat, schem)), + ) for cat, schem in passing_ns_args: - tcli_service_instance.OpenSession.return_value = \ + tcli_service_instance.OpenSession.return_value = ( self._construct_open_session_with_namespace(False, cat, schem) + ) backend.open_session({}, cat, schem) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_protocol_v3_fails_if_initial_namespace_set(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value - tcli_service_instance.OpenSession.return_value = \ - ttypes.TOpenSessionResp( - status=self.okay_status, - serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V3, - canUseMultipleCatalogs=True, - initialNamespace=ttypes.TNamespace(catalogName="cat", schemaName="schem") - ) + tcli_service_instance.OpenSession.return_value = ttypes.TOpenSessionResp( + status=self.okay_status, + serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V3, + canUseMultipleCatalogs=True, + initialNamespace=ttypes.TNamespace(catalogName="cat", schemaName="schem"), + ) - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + ) with self.assertRaises(InvalidServerResponseError) as cm: backend.open_session({}, "cat", "schem") - self.assertIn("Setting initial namespace not supported by the DBR version", - str(cm.exception)) + self.assertIn( + "Setting initial namespace not supported by the DBR version", + str(cm.exception), + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) @patch("databricks.sql.thrift_backend.ThriftBackend._handle_execute_response") - def test_execute_command_sets_complex_type_fields_correctly(self, mock_handle_execute_response, - tcli_service_class): + def test_execute_command_sets_complex_type_fields_correctly( + self, mock_handle_execute_response, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value # Iterate through each possible combination of native types (True, False and unset) - for (complex, timestamp, decimals) in itertools.product( - [True, False, None], [True, False, None], [True, False, None]): + for complex, timestamp, decimals in itertools.product( + [True, False, None], [True, False, None], [True, False, None] + ): complex_arg_types = {} if complex is not None: complex_arg_types["_use_arrow_native_complex_types"] = complex @@ -1388,18 +2170,36 @@ def test_execute_command_sets_complex_type_fields_correctly(self, mock_handle_ex if decimals is not None: complex_arg_types["_use_arrow_native_decimals"] = decimals - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), **complex_arg_types) + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + **complex_arg_types, + ) thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock()) - t_execute_statement_req = tcli_service_instance.ExecuteStatement.call_args[0][0] + t_execute_statement_req = tcli_service_instance.ExecuteStatement.call_args[ + 0 + ][0] # If the value is unset, the native type should default to True - self.assertEqual(t_execute_statement_req.useArrowNativeTypes.timestampAsArrow, - complex_arg_types.get("_use_arrow_native_timestamps", True)) - self.assertEqual(t_execute_statement_req.useArrowNativeTypes.decimalAsArrow, - complex_arg_types.get("_use_arrow_native_decimals", True)) - self.assertEqual(t_execute_statement_req.useArrowNativeTypes.complexTypesAsArrow, - complex_arg_types.get("_use_arrow_native_complex_types", True)) - self.assertFalse(t_execute_statement_req.useArrowNativeTypes.intervalTypesAsArrow) + self.assertEqual( + t_execute_statement_req.useArrowNativeTypes.timestampAsArrow, + complex_arg_types.get("_use_arrow_native_timestamps", True), + ) + self.assertEqual( + t_execute_statement_req.useArrowNativeTypes.decimalAsArrow, + complex_arg_types.get("_use_arrow_native_decimals", True), + ) + self.assertEqual( + t_execute_statement_req.useArrowNativeTypes.complexTypesAsArrow, + complex_arg_types.get("_use_arrow_native_complex_types", True), + ) + self.assertFalse( + t_execute_statement_req.useArrowNativeTypes.intervalTypesAsArrow + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main()