update: linting / deployement

.github/dependabot.yml

@@ -6,6 +6,6 @@
 version: 2
 updates:
   - package-ecosystem: "poetry"
-    directory: "/" # Location of package manifests
+    directory: "/"  # Location of package manifests
     schedule:
       interval: "weekly"

.github/workflows/api-test.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"] # Support starts at v3.9
+        python-version: ["3.9", "3.10", "3.11"]  # Support starts at v3.9
       fail-fast: false
 
     steps:

.github/workflows/linting.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"] # Support starts at v3.9
+        python-version: ["3.9", "3.10", "3.11"]  # Support starts at v3.9
       fail-fast: false
 
     steps:

.github/workflows/unittest.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"] # Support starts at v3.9
+        python-version: ["3.9", "3.10", "3.11"]  # Support starts at v3.9
       fail-fast: false
 
     steps:

.gitignore

@@ -4,6 +4,7 @@ debug.log
 spend.log
 .env.config
 .env.secrets
+dictionary.dic
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.pyspelling.yaml

@@ -0,0 +1,19 @@
+matrix:
+  - name: Markdown
+    aspell:
+      lang: en
+      # ignore-case: true
+    dictionary:
+      encoding: utf-8
+      wordlists:
+        - .wordlist
+    pipeline:
+      - pyspelling.filters.markdown:
+      - pyspelling.filters.html:
+          comments: false
+          ignores:
+            - code
+            - pre
+    sources:
+      - '!**/INFO.md|!**/TODO.md|!venv/**|**/*.md'
+    default_encoding: utf-8

.wordlist

@@ -0,0 +1,14 @@
+CLI
+ConfigMap
+Dockerfile
+embeddings
+json
+LLM
+Makefile
+md
+openapi
+OpenAPI
+pyspelling
+repo
+UI
+volumeMount

.yamllint

@@ -0,0 +1,16 @@
+extends: default
+
+rules:
+  document-start: disable
+  line-length: disable
+  comments-indentation: disable
+  new-line-at-end-of-file: disable
+  indentation:
+    indent-sequences: whatever
+    # ignore: kludgeops/
+  truthy: disable
+  trailing-spaces: enable
+
+ignore:
+  - venv/
+  - .venv/

API.md

@@ -4,10 +4,10 @@ The Deep Thought API is a REST API that provides access to the Deep Thought.
 
 ## Endpoint Details
 
-Endpoint specifications are defined in the OpenAPI spec. Each version of the API has its own spec. The specs can be found in the `/specs` directory.
+Endpoint specifications are defined in the OpenAPI spec. Each version of the API has its own spec. The specs can be found in the `src/specs` directory.
 
-- [v1](./specs/openapi-v1.json)
-- [v2](./specs/openapi-v2.json)
+- [v1](src/specs/openapi-v1.json)
+- [v2](src/specs/openapi-v2.json)
 
 ### Find Sources
 
@@ -18,7 +18,7 @@ Description: Find sources for a given query
 With curl:
 
 ```bash
-curl -X 'POST' 'http://127.0.0.1:8000/v1/find_sources' -H 'Content-Type: application/json' -d '{"query": "test_query", "num_results": 2}'
+curl -X 'POST' 'http://127.0.0.1:8080/v1/find_sources' -H 'Content-Type: application/json' -d '{"query": "test_query", "num_results": 2}'
 ```
 
 <!-- With Postman:
@@ -36,7 +36,7 @@ Description: Find the most likely fact for a given query
 With curl:
 
 ```bash
-curl -X 'POST' 'http://127.0.0.1:8000/v1/' -H 'Content-Type: application/json' -d '{"user_input": "whats for dinner"}'
+curl -X 'POST' 'http://127.0.0.1:8080/v1/' -H 'Content-Type: application/json' -d '{"user_input": "whats for dinner"}'
 ```
 
 <!-- With Postman:
@@ -54,13 +54,13 @@ Description: Ask a question and get an answer with links to sources
 With curl:
 
 ```bash
-curl -X 'POST' 'http://127.0.0.1:8000/v1/ask' -H 'Content-Type: application/json' -d '{"query": "step by step instructions to install a new operator", "num_results": 1}'
+curl -X 'POST' 'http://127.0.0.1:8080/v1/ask' -H 'Content-Type: application/json' -d '{"query": "step by step instructions to install a new operator", "num_results": 1}'
 ```
 
 <!-- With Postman:
 
 ```bash
-POST http://localhost:8000/v1/synthesize_response
+POST http://localhost:8080/v1/synthesize_response
 ``` -->
 
 ## OpenAPI (Swagger) Specification
@@ -84,7 +84,7 @@ This API follows [Semantic Versioning](https://semver.org/), but only the major
 To test versioning, you can hit two separate endpoints:
 
 ```bash
-curl -X 'GET' 'http://127.0.0.1:8000/v1/items/' -H 'Content-Type: application/json' -d '{"query": "test_query", "num_results": 2}'
+curl -X 'GET' 'http://127.0.0.1:8080/v1/items/' -H 'Content-Type: application/json' -d '{"query": "test_query", "num_results": 2}'
 ```
 
 Which should return:
@@ -100,7 +100,7 @@ Which should return:
 And then:
 
 ```bash
-curl -X 'GET' 'http://127.0.0.1:8000/v2/api_version_test/' -H 'Content-Type: application/json' -d '{"query": "test_query", "num_results": 2}'
+curl -X 'GET' 'http://127.0.0.1:8080/v2/api_version_test/' -H 'Content-Type: application/json' -d '{"query": "test_query", "num_results": 2}'
 ```
 
 Which should return:

Makefile

@@ -1,25 +1,51 @@
+.ONESHELL:
+
 API_VERSIONS = v1 v2
-SERVER_URL = http://127.0.0.1:8000
+SERVER_URL = http://127.0.0.1:8080
 SPEC_PATH = specs
 
 .PHONY: test run test-api test-all $(API_VERSIONS)
 
+venv/bin/activate: requirements-devel.txt
+	python3 -m venv venv
+	. venv/bin/activate
+	pip install --upgrade pip setuptools wheel
+	pip install -r requirements-devel.txt
+	poetry install
+
+clean-prereqs:
+	rm -rf venv
+.PHONY: clean-prereqs
+
+prereqs: venv/bin/activate
+# ifndef VIRTUAL_ENV
+# 	$(error run: . venv/bin/activate)
+# endif
+.PHONY: prereqs
+
+lint: prereqs
+	. venv/bin/activate
+	yamllint .
+	pyspelling
+	# black .
+	pylint src
+.PHONY: prereqs lint
+
 run:
 	@echo "Current virtualenv: $(VIRTUAL_ENV)"
-	@uvicorn src.app:app --reload
+	@uvicorn src.app:app --port 8080 --reload
 
-install:
+install: prereqs
+	. venv/bin/activate
 	poetry install
 
 upgrade-dependencies:
 	./src/scripts/upgrade-dependencies.sh
 
-lint:
-	poetry run pylint src
-
 test-all: lint test test-api
 
 test:
+	. .venv/bin/activate
 	poetry run pytest
 
 test-api: $(API_VERSIONS)

README.md

@@ -15,6 +15,6 @@ Information about the API can be found in the [API documentation](API.md).
 
 Information for deploying Deep Thought to OpenShift can be found in the [deployment documentation](deployment/README.md).
 
-## Contributing
+## Contributing / Development
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).

deployment/README.md

@@ -22,17 +22,11 @@ To deploy the deep thought application you need to perform a few steps as outlin
 - Create the first secret using `oc apply -f secrets.yaml -n deep-thought-example`.
 - Create the second secret using `oc apply -f google-application-credentials -n deep-thought-example`.
 
-Note: Secret values must be in base64 format. To convert a value to base64, run `echo -n "<value>" | base64 | tr -d '\n'` and use as your .
-
 ### 5. Create a Build Configuration
 
 - ~~If your repo contains a Dockerfile, use `oc new-build --strategy=docker --binary --name=<app-name>` to create a build configuration.~~
 - If you're using a Source-to-Image (S2I) build, use `oc new-app https://github.com/HunterGerlach/deep-thought --name=deep-thought -n deep-thought-example`.
 
-### 6. Edit the Service
-
-- Run `oc edit svc deep-thought` and update all instances of the port number to `8000`.
-
 ### 7. Edit the Deployment
 
 - Run `oc edit deployment deep-thought` and add the ConfigMap and Secrets as environment variables and mount the credentials file as a volumeMount. See `deployment.yaml` for an example. The volumes section needs to be separate from the rest.
@@ -79,7 +73,7 @@ Note: Secret values must be in base64 format. To convert a value to base64, run
 - Click on the `+Add` button.
 - Select ~~`From Dockerfile` or~~ `Import from Git`.
 - Enter a URL to your repo (e.g. `https://github.com/HunterGerlach/deep-thought.git`). OpenShift should automatically identify the language and choose the appropriate builder image.
-- Set the port to `8000`.
+- Set the port to `8080`.
 - Click `Create`.
 
 ### 5. Start the Build

deployment/configmap.yaml

@@ -2,7 +2,7 @@ kind: ConfigMap
 apiVersion: v1
 metadata:
   name: deep-thought-config
-  #namespace: deep-thought-dev
+  # namespace: deep-thought-dev
 data:
   PYTHONPATH: /
   SPENDING_WARNING_PCT: '0.8'

deployment/deployment.yaml

@@ -1,22 +1,19 @@
 kind: Deployment
 apiVersion: apps/v1
 metadata:
-  name: deep-thought # change to your app name
+  name: deep-thought
 spec:
   selector:
     matchLabels:
-      app: deep-thought # change to your app name
+      app: deep-thought
   template:
     metadata:
       labels:
-        app: deep-thought # change to your app name
+        app: deep-thought
     spec:
       containers:
         - name: deep-thought
           image: image-registry.openshift-image-registry.svc:5000/deep-thought-example/deep-thought:latest
-
-          # Copy all the following lines
-          # and paste them into the end of the container definition
           volumeMounts:
             - name: deepthought-json
               mountPath: /etc/42

deployment/google-application-credentials.yaml

@@ -3,6 +3,6 @@ apiVersion: v1
 metadata:
   name: google-application-credentials
   # namespace: deep-thought-dev
-data:
-  deepthought-dev.json: INSERT_JSON # insert deep thought json file. This current resides in /etc/innovation-office/deepthought/deepthought-local.json
+stringData:
+  deepthought-dev.json: INSERT_JSON  # insert deep thought json file. This current resides in /etc/innovation-office/deepthought/deepthought-local.json
 type: Opaque

deployment/route.yaml

@@ -8,7 +8,7 @@ spec:
     name: deep-thought
     weight: 100
   port:
-    targetPort: 8000-tcp
+    targetPort: 8080-tcp
   tls:
     termination: edge
     insecureEdgeTerminationPolicy: Redirect

deployment/secrets.yaml

@@ -3,7 +3,7 @@ apiVersion: v1
 metadata:
   name: deep-thought-secrets
   # namespace: deep-thought-dev
-data:
-  CONNECTION_STRING: postgresql://... # insert connection string to postgresql vector db
-  OPENAI_API_KEY: sk-... # insert openai api key
+stringData:
+  CONNECTION_STRING: postgresql://...  # insert connection string to postgresql vector db
+  OPENAI_API_KEY: sk-...  # insert openai api key
 type: Opaque

deployment/service.yaml

@@ -4,10 +4,10 @@ metadata:
   name: deep-thought
 spec:
   ports:
-    - name: 8000-tcp
+    - name: 8080-tcp
       protocol: TCP
-      port: 8000
-      targetPort: 8000
+      port: 8080
+      targetPort: 8080
   # selector:
   #   app: deep-thought
   #   deployment: deep-thought

pyproject.toml

@@ -5,6 +5,8 @@ description = "A question-answering system that uses embeddings and a language m
 authors = ["Hunter Gerlach <[email protected]>"]
 license = "GPL-3.0"
 readme = "README.md"
+packages = [{ include="src", from="." }]
+
 
 [tool.poetry.dependencies]
 python = ">=3.9,<3.13"

requirements-devel.txt

@@ -0,0 +1,5 @@
+black
+poetry
+pylint
+pyspelling
+yamllint

src/app.py

@@ -7,10 +7,10 @@
 from fastapi.responses import JSONResponse
 
 
-from src.config import Config
-from src.logging_setup import setup_logger
-from src.v1.endpoints import router as v1_router
-from src.v2.endpoints import router as v2_router
+from config import Config
+from logging_setup import setup_logger
+from v1.endpoints import router as v1_router
+from v2.endpoints import router as v2_router
 
 config = Config()
 logger = setup_logger()
@@ -68,4 +68,4 @@ def handle_exception(exc):
 
 if __name__ == '__main__':
     import uvicorn
-    uvicorn.run(app, host="0.0.0.0")
+    uvicorn.run("__main__:app", host="0.0.0.0", port=8080, reload=True)

src/embeddings.py

@@ -6,8 +6,8 @@
 from langchain.vectorstores.pgvector import PGVector
 from langchain.embeddings import HuggingFaceEmbeddings
 
-from src.config import Config
-from src.logging_setup import setup_logger
+from config import Config
+from logging_setup import setup_logger
 
 config = Config()
 logger = setup_logger()

src/logging_setup.py

@@ -3,7 +3,7 @@
 import logging
 from termcolor import colored
 
-from src.config import Config
+from config import Config
 
 config = Config()
 log_level = config.get("LOG_LEVEL", "INFO")