Simple token based chunker #10

	name: CI

	on: pull_request

	jobs:
	run-linters:
	name: Run linters
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v3
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	version: 1.3.2
	- name: install dependencies
	run: poetry install --with dev
	- name: Run flake8 Check
	run: poetry run flake8 .
	- name: Run mypy check
	run: poetry run mypy .

	run-tests:
	name: Run tests
	runs-on: ubuntu-latest
	needs: run-linters
	strategy:
	matrix:
	python-version: [3.8, 3.9, '3.10', 3.11]

	steps:
	- uses: actions/checkout@v3
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	version: 1.3.2
	- name: install dependencies
	run: poetry install --with dev --all-extras
	- name: Run pytest
	run: poetry run pytest -v --html=report.html tests/unit
	- name: upload pytest report.html
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: dataset-pytest-report-py${{ matrix.python-version }}
	path: report.html

Provide feedback