Skip to content

Commit

Permalink
Add process/service usage to API endpoint
Browse files Browse the repository at this point in the history
  • Loading branch information
dormant-user committed Sep 30, 2024
1 parent 9b4365b commit 434f7f2
Show file tree
Hide file tree
Showing 10 changed files with 152 additions and 34 deletions.
8 changes: 6 additions & 2 deletions docs/genindex.html
Original file line number Diff line number Diff line change
Expand Up @@ -278,10 +278,10 @@ <h2 id="G">G</h2>
</li>
<li><a href="index.html#pyninja.executors.routes.get_ip_address">get_ip_address() (in module pyninja.executors.routes)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#pyninja.modules.models.get_library">get_library() (in module pyninja.modules.models)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#pyninja.executors.routes.get_memory_utilization">get_memory_utilization() (in module pyninja.executors.routes)</a>
</li>
<li><a href="index.html#pyninja.features.cpu.get_name">get_name() (in module pyninja.features.cpu)</a>
Expand All @@ -300,6 +300,8 @@ <h2 id="G">G</h2>
<li><a href="index.html#pyninja.features.process.get_process_status">(in module pyninja.features.process)</a>
</li>
</ul></li>
<li><a href="index.html#pyninja.executors.routes.get_process_usage">get_process_usage() (in module pyninja.executors.routes)</a>
</li>
<li><a href="index.html#pyninja.executors.routes.get_processor_name">get_processor_name() (in module pyninja.executors.routes)</a>
</li>
<li><a href="index.html#pyninja.executors.database.get_record">get_record() (in module pyninja.executors.database)</a>
Expand All @@ -320,6 +322,8 @@ <h2 id="G">G</h2>
<li><a href="index.html#pyninja.features.service.get_service_status">(in module pyninja.features.service)</a>
</li>
</ul></li>
<li><a href="index.html#pyninja.executors.routes.get_service_usage">get_service_usage() (in module pyninja.executors.routes)</a>
</li>
<li><a href="index.html#pyninja.monitor.resources.get_system_metrics">get_system_metrics() (in module pyninja.monitor.resources)</a>
</li>
<li><a href="index.html#pyninja.modules.models.EnvConfig.gpu_lib">gpu_lib (pyninja.modules.models.EnvConfig attribute)</a>
Expand Down
48 changes: 43 additions & 5 deletions docs/index.html

Large diffs are not rendered by default.

Binary file modified docs/objects.inv
Binary file not shown.
2 changes: 1 addition & 1 deletion docs/searchindex.js

Large diffs are not rendered by default.

76 changes: 74 additions & 2 deletions pyninja/executors/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from pydantic import PositiveFloat, PositiveInt

from pyninja.executors import auth, squire
from pyninja.features import cpu, disks, dockerized, process, service
from pyninja.features import cpu, disks, dockerized, operations, process, service
from pyninja.modules import exceptions, models
from pyninja.monitor import resources

Expand Down Expand Up @@ -241,12 +241,72 @@ async def get_process_status(
)


async def get_service_usage(
request: Request,
service_names: List[str],
apikey: HTTPAuthorizationCredentials = Depends(BEARER_AUTH),
):
"""**API function to monitor a service.**
**Args:**
- request: Reference to the FastAPI request object.
- service_names: Name of the service to check status.
- apikey: API Key to authenticate the request.
**Raises:**
APIResponse:
Raises the HTTPStatus object with a status code and detail as response.
"""
await auth.level_1(request, apikey)
response = await operations.service_monitor(service_names)
if len(service_names) == 1:
response = response[0]
if response.get("PID") == 0000:
raise exceptions.APIResponse(
status_code=HTTPStatus.NOT_FOUND.real,
detail=f"{service_names[0]!r} not found or not running",
)
raise exceptions.APIResponse(status_code=HTTPStatus.OK.real, detail=response)


async def get_process_usage(
request: Request,
process_names: List[str],
apikey: HTTPAuthorizationCredentials = Depends(BEARER_AUTH),
):
"""**API function to monitor a process.**
**Args:**
- request: Reference to the FastAPI request object.
- process_names: Name of the service to check status.
- apikey: API Key to authenticate the request.
**Raises:**
APIResponse:
Raises the HTTPStatus object with a status code and detail as response.
"""
await auth.level_1(request, apikey)
response = await operations.process_monitor(process_names)
if len(process_names) == 1:
response = response[0]
if response.get("PID") == 0000:
raise exceptions.APIResponse(
status_code=HTTPStatus.NOT_FOUND.real,
detail=f"{process_names[0]!r} not found or not running",
)
raise exceptions.APIResponse(status_code=HTTPStatus.OK.real, detail=response)


async def get_service_status(
request: Request,
service_name: str,
apikey: HTTPAuthorizationCredentials = Depends(BEARER_AUTH),
):
"""**API function to monitor a service.**
"""**API function to get the status of a service.**
**Args:**
Expand Down Expand Up @@ -512,12 +572,24 @@ def get_all_routes(dependencies: List[Depends]) -> List[APIRoute]:
methods=["GET"],
dependencies=dependencies,
),
APIRoute(
path="/service-usage",
endpoint=get_service_usage,
methods=["POST"],
dependencies=dependencies,
),
APIRoute(
path="/process-status",
endpoint=get_process_status,
methods=["GET"],
dependencies=dependencies,
),
APIRoute(
path="/process-usage",
endpoint=get_process_usage,
methods=["POST"],
dependencies=dependencies,
),
APIRoute(
path="/docker-container",
endpoint=get_docker_containers,
Expand Down
19 changes: 9 additions & 10 deletions pyninja/features/operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import logging
import subprocess
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
from typing import Dict, List, Optional

Expand All @@ -28,6 +27,7 @@ def default(name: str):
}


# todo: Remove redundancy between features/operations.py and features/services.py [OR] features/process.py
def get_process_info(
proc: psutil.Process, process_name: str = None
) -> Dict[str, str | int]:
Expand Down Expand Up @@ -68,7 +68,7 @@ def get_process_info(
return default(process_name or proc.name())


async def process_monitor(executor: ThreadPoolExecutor) -> List[Dict[str, str]]:
async def process_monitor(processes: List[str]) -> List[Dict[str, str]]:
"""Function to monitor processes and return their usage statistics.
See Also:
Expand All @@ -86,15 +86,14 @@ async def process_monitor(executor: ThreadPoolExecutor) -> List[Dict[str, str]]:
for proc in psutil.process_iter(
["pid", "name", "cpu_percent", "memory_info", "create_time"]
):
if any(
name in proc.name() or name == str(proc.pid)
for name in models.env.processes
):
tasks.append(loop.run_in_executor(executor, get_process_info, *(proc,)))
if any(name in proc.name() or name == str(proc.pid) for name in processes):
tasks.append(
loop.run_in_executor(models.EXECUTOR, get_process_info, *(proc,))
)
return [await task for task in asyncio.as_completed(tasks)]


async def service_monitor(executor: ThreadPoolExecutor) -> List[Dict[str, str]]:
async def service_monitor(services: List[str]) -> List[Dict[str, str]]:
"""Function to monitor services and return their usage statistics.
See Also:
Expand All @@ -111,7 +110,7 @@ async def service_monitor(executor: ThreadPoolExecutor) -> List[Dict[str, str]]:
loop = asyncio.get_event_loop()
tasks = []
usages = []
for service_name in models.env.services:
for service_name in services:
pid = get_service_pid(service_name)
if not pid:
LOGGER.debug(f"Failed to get PID for service: {service_name}")
Expand All @@ -124,7 +123,7 @@ async def service_monitor(executor: ThreadPoolExecutor) -> List[Dict[str, str]]:
usages.append(default(service_name))
continue
tasks.append(
loop.run_in_executor(executor, get_process_info, proc, service_name)
loop.run_in_executor(models.EXECUTOR, get_process_info, proc, service_name)
)
for task in asyncio.as_completed(tasks):
usages.append(await task)
Expand Down
10 changes: 5 additions & 5 deletions pyninja/features/process.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import logging
import os
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import as_completed
from typing import Dict, List

import psutil
from pydantic import PositiveInt

from pyninja.modules import models

LOGGER = logging.getLogger("uvicorn.default")


Expand All @@ -24,11 +25,10 @@ def get_process_status(
"""
result = []
futures = {}
executor = ThreadPoolExecutor(max_workers=os.cpu_count())
with executor:
with models.EXECUTOR:
for proc in psutil.process_iter(["pid", "name"]):
if proc.name().lower() == process_name.lower():
future = executor.submit(
future = models.EXECUTOR.submit(
get_performance, process=proc, cpu_interval=cpu_interval
)
futures[future] = proc.name()
Expand Down
6 changes: 3 additions & 3 deletions pyninja/features/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def get_service_status(service_name: str) -> models.ServiceStatus:
ServiceStatus:
Returns an instance of the ServiceStatus object.
"""
if models.OPERATING_SYSTEM == "Linux":
if models.OPERATING_SYSTEM == "linux":
try:
output = subprocess.check_output(
[models.env.service_lib, "is-active", service_name],
Expand All @@ -102,7 +102,7 @@ def get_service_status(service_name: str) -> models.ServiceStatus:
LOGGER.error("%d - %s", 404, error)
return unavailable(service_name)

if models.OPERATING_SYSTEM == "Darwin":
if models.OPERATING_SYSTEM == "darwin":
try:
output = subprocess.check_output(
[models.env.service_lib, "list"], text=True
Expand All @@ -116,7 +116,7 @@ def get_service_status(service_name: str) -> models.ServiceStatus:
LOGGER.error("%d - %s", 404, error)
return unavailable(service_name)

if models.OPERATING_SYSTEM == "Windows":
if models.OPERATING_SYSTEM == "windows":
try:
output = subprocess.check_output(
[models.env.service_lib, "query", service_name],
Expand Down
4 changes: 4 additions & 0 deletions pyninja/modules/models.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import os
import pathlib
import platform
import re
import socket
import sqlite3
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, List, Set, Tuple, Type

from pydantic import (
Expand All @@ -18,6 +20,8 @@
from pyninja.modules import exceptions

MINIMUM_CPU_UPDATE_INTERVAL = 1
# Use a ThreadPoolExecutor to run blocking functions in separate threads
EXECUTOR = ThreadPoolExecutor(max_workers=os.cpu_count())
OPERATING_SYSTEM = platform.system().lower()
if OPERATING_SYSTEM not in ("darwin", "linux", "windows"):
exceptions.raise_os_error(OPERATING_SYSTEM)
Expand Down
13 changes: 7 additions & 6 deletions pyninja/monitor/resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,13 @@
import json
import logging
import os
from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List

import psutil

from pyninja.features import operations
from pyninja.modules import models

# Use a ThreadPoolExecutor to run blocking functions in separate threads
EXECUTOR = ThreadPoolExecutor(max_workers=os.cpu_count())
LOGGER = logging.getLogger("uvicorn.default")


Expand Down Expand Up @@ -96,13 +93,17 @@ async def system_resources() -> Dict[str, dict]:
"""
system_metrics_task = asyncio.create_task(get_system_metrics())
docker_stats_task = asyncio.create_task(get_docker_stats())
service_stats_task = asyncio.create_task(operations.service_monitor(EXECUTOR))
process_stats_task = asyncio.create_task(operations.process_monitor(EXECUTOR))
service_stats_task = asyncio.create_task(
operations.service_monitor(models.env.services)
)
process_stats_task = asyncio.create_task(
operations.process_monitor(models.env.processes)
)

# CPU percent check is a blocking call and cannot be awaited, so run it in a separate thread
loop = asyncio.get_event_loop()
cpu_usage_task = loop.run_in_executor(
EXECUTOR, get_cpu_percent, *(models.MINIMUM_CPU_UPDATE_INTERVAL,)
models.EXECUTOR, get_cpu_percent, *(models.MINIMUM_CPU_UPDATE_INTERVAL,)
)

system_metrics = await system_metrics_task
Expand Down

0 comments on commit 434f7f2

Please sign in to comment.