Add monitoring for APIs (#98)

- Add monitoring for the APIs in sequence.
- Create [notion
doc](https://www.notion.so/astronomerio/Ask-Astro-Monitoring-Observability-d22964e6166f4580837e924339cedf16?pvs=4)
for observability and monitoring.
- Integrated the monitoring for Open AI.
<img width="1372" alt="Screenshot 2023-11-09 at 1 22 34 AM"
src="https://github.com/astronomer/ask-astro/assets/8670962/6b591c02-03bd-4fa8-a014-089f5f9cf975">
I have tested this with slack integration:

<img width="688" alt="Screenshot 2023-11-09 at 5 04 26 PM"
src="https://github.com/astronomer/ask-astro/assets/8670962/fff1be81-5437-46b8-8357-17dbd77ae3be">



closes: #59

airflow/Dockerfile

@@ -1,5 +1,5 @@
 # syntax=quay.io/astronomer/airflow-extensions:latest
 
-FROM quay.io/astronomer/astro-runtime:9.3.0-base
+FROM quay.io/astronomer/astro-runtime:9.5.0-base
 
 COPY include/airflow_provider_weaviate-0.0.1-py3-none-any.whl /tmp

airflow/dags/ingestion/ask-astro-load-blogs.py

@@ -9,6 +9,7 @@
 ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "")
 
 _WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
+WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")
 
 blog_cutoff_date = datetime.strptime("2023-01-19", "%Y-%m-%d")
 
@@ -30,7 +31,7 @@ def ask_astro_load_blogs():
         weaviate_conn_id=_WEAVIATE_CONN_ID,
         retries=10,
         retry_delay=30,
-    ).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs])
+    ).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs])
 
 
 ask_astro_load_blogs()

airflow/dags/ingestion/ask-astro-load-github.py

@@ -10,7 +10,7 @@
 
 _WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
 _GITHUB_CONN_ID = "github_ro"
-
+WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")
 markdown_docs_sources = [
     {"doc_dir": "learn", "repo_base": "astronomer/docs"},
     {"doc_dir": "astro", "repo_base": "astronomer/docs"},
@@ -71,7 +71,7 @@ def ask_astro_load_github():
         weaviate_conn_id=_WEAVIATE_CONN_ID,
         retries=10,
         retry_delay=30,
-    ).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs, split_code_docs])
+    ).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs, split_code_docs])
 
     issues_docs >> md_docs >> rst_docs >> code_samples
 

airflow/dags/ingestion/ask-astro-load-registry.py

@@ -9,6 +9,7 @@
 ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "")
 
 _WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
+WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")
 
 
 @dag(schedule_interval="0 5 * * *", start_date=datetime(2023, 9, 27), catchup=False, is_paused_upon_creation=True)
@@ -32,7 +33,7 @@ def ask_astro_load_registry():
         weaviate_conn_id=_WEAVIATE_CONN_ID,
         retries=10,
         retry_delay=30,
-    ).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs, split_code_docs])
+    ).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs, split_code_docs])
 
 
 ask_astro_load_registry()

airflow/dags/ingestion/ask-astro-load-slack.py

@@ -9,7 +9,7 @@
 ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "")
 
 _WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
-
+WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")
 slack_channel_sources = [
     {
         "channel_name": "troubleshooting",
@@ -38,7 +38,7 @@ def ask_astro_load_slack():
         weaviate_conn_id=_WEAVIATE_CONN_ID,
         retries=10,
         retry_delay=30,
-    ).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs])
+    ).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs])
 
 
 ask_astro_load_slack()

airflow/dags/ingestion/ask-astro-load-stackoverflow.py

@@ -9,7 +9,7 @@
 ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "")
 
 _WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
-
+WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")
 stackoverflow_cutoff_date = "2021-09-01"
 
 stackoverflow_tags = [
@@ -38,7 +38,7 @@ def ask_astro_load_stackoverflow():
         weaviate_conn_id=_WEAVIATE_CONN_ID,
         retries=10,
         retry_delay=30,
-    ).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs])
+    ).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs])
 
 
 ask_astro_load_stackoverflow()

airflow/dags/ingestion/ask-astro-load.py

@@ -15,6 +15,7 @@
 
 _WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
 _GITHUB_CONN_ID = "github_ro"
+WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")
 
 markdown_docs_sources = [
     {"doc_dir": "learn", "repo_base": "astronomer/docs"},
@@ -83,7 +84,7 @@ def ask_astro_load_bulk():
         task_id="create_schema",
         weaviate_conn_id=_WEAVIATE_CONN_ID,
         class_object_data="file://include/data/schema.json",
-        existing="fail",
+        existing="ignore",
     )
 
     @task.branch(trigger_rule="none_failed")
@@ -238,12 +239,12 @@ def extract_astro_blogs():
     split_code_docs = task(split.split_python).expand(dfs=python_code_tasks)
 
     task.weaviate_import(ingest.import_data, weaviate_conn_id=_WEAVIATE_CONN_ID, retries=10, retry_delay=30).partial(
-        class_name="Docs"
+        class_name=WEAVIATE_CLASS
     ).expand(dfs=[split_md_docs, split_code_docs])
 
     _import_baseline = task.weaviate_import(
         ingest.import_baseline, trigger_rule="none_failed", weaviate_conn_id=_WEAVIATE_CONN_ID
-    )(class_name="Docs", seed_baseline_url=seed_baseline_url)
+    )(class_name=WEAVIATE_CLASS, seed_baseline_url=seed_baseline_url)
 
     _check_schema >> [_check_seed_baseline, _create_schema]