Merge branch 'master' into deploy-cue

.config/dictionaries/project.dic

@@ -27,6 +27,7 @@ GOMODCACHE
 gopls
 gosec
 graphviz
+jorm
 jormungandr
 Kroki
 kubeconfig

.gitignore

@@ -10,7 +10,7 @@ bin/
 node_modules
 
 # Developer only files
-/.vscode
+.vscode
 
 # Built docs
 /earthly/docs/site
@@ -21,4 +21,4 @@ target/
 Cargo.lock
 
 # Python junk
-**/*.pyc
+**/*.pyc

cspell.json

@@ -43,5 +43,6 @@
         "cspell.json",
         "**/**/go.mod",
         "**/**/go.sum",
+        "**/**/pyproject.toml",
     ]
 }

flake.nix

@@ -3,7 +3,7 @@
   description = "Catalyst CI";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     devenv.url = "github:cachix/devenv";
     mk-shell-bin.url = "github:rrbutani/nix-mk-shell-bin";
   };
@@ -50,6 +50,7 @@
             nodePackages.typescript
 
             # Python
+            ruff
             poetry
             python312
           ];

services/jorm-metrics-server/.gitignore

@@ -0,0 +1 @@
+test.py

services/jorm-metrics-server/Earthfile

@@ -0,0 +1,59 @@
+VERSION 0.7
+
+deps:
+    FROM python:3.12.1-slim-bookworm
+    WORKDIR /work
+
+    ARG version = 1.7.1
+
+    ENV POETRY_HOME=/opt/poetry
+    ENV POETRY_VERSION=$version
+    ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+    ENV PATH=$POETRY_HOME/bin:$PATH
+
+    RUN apt-get update && \
+        apt-get install -y --no-install-recommends \
+        build-essential \
+        curl
+
+    RUN curl -sSL https://install.python-poetry.org | python3 -
+
+    COPY poetry.lock pyproject.toml README.md .
+
+    RUN poetry install --only main --no-root
+
+src:
+    FROM +deps
+
+    COPY --dir jorm_metrics_server .
+
+check:
+    FROM +src
+
+    RUN poetry install --only dev --no-root
+    RUN poetry run ruff check .
+    RUN poetry run ruff format --check .
+
+build:
+    FROM +src
+
+    COPY --dir jorm_metrics_server .
+    RUN poetry install --only main
+
+    SAVE ARTIFACT .venv venv
+    SAVE ARTIFACT . src
+
+publish:
+    FROM python:3.12.1-slim-bookworm
+
+    WORKDIR /app
+
+    ENV tag=latest
+    ENV PATH=/app/.venv/bin:$PATH
+
+    COPY +build/venv .
+    COPY +build/src .
+
+    ENTRYPOINT ["python", "-m", "jorm_metrics_server.main"]
+
+    SAVE IMAGE jorm-metrics-server:$tag

services/jorm-metrics-server/README.md

@@ -0,0 +1,50 @@
+# jorm-metrics-server
+
+> A small Prometheus exporter for aggregating Jormungandr metrics
+
+This service scrapes the Jormungandr node API on a routine basis and exposes useful metrics in the Prometheus format.
+Once pointed at a node, it will expose a predetermined list of Prometheus metrics at `/metrics`.
+
+## Usage
+
+Build the container:
+
+```terminal
+earthly +publish
+```
+
+Point it at the root address of a Jormungandr node and run it:
+
+```terminal
+docker run -p 8080:8080 -e API_URL="https://core.projectcatalyst.io" jorm-metrics-server
+```
+
+The metrics will be exposed at `http://localhost:8080/metrics`.
+
+## Configuration
+
+The service can be configured with the following environment variables:
+
+| Name                | Description                                                                     | Required | Default      |
+| ------------------- | ------------------------------------------------------------------------------- | -------- | ------------ |
+| `API_URL`           | The root address of the Jormungandr node to scrape                              | Yes      | N/A          |
+| `ADDRESS`           | The address the metrics server should listen on                                 | No       | `0.0.0.0`    |
+| `PORT`              | The port the metrics server should listen on                                    | No       | `8080`       |
+| `INTERVAL`          | The interval at which the service will scrape the Jormungandr node (in seconds) | No       | `60`         |
+| `STORAGE`           | A path to a directory where the metrics server will store a cache               | No       | `/tmp/cache` |
+| `METRICS`           | Comma-separated list of metrics to scrape (see table below).                    | No       | (all)        |
+| `DISABLE_JSON_LOGS` | Disable JSON log output                                                         | No       | `false`      |
+
+## Metrics
+
+The following metrics are currently exposed:
+
+| Name                 | Description                                                              | Expensive |
+| -------------------- | ------------------------------------------------------------------------ | --------- |
+| `num_proposal_votes` | The number of votes per proposal                                         | Yes       |
+| `num_unique_voters`  | The number of unique voters that have cast at least one vote on the node | No        |
+| `voting_power`       | A histogram of the voting power distribution amongst active voters       | No        |
+
+The expensive column denotes how expensive the metric is to scrape in terms of the numbers of series generated.
+The more series generated, the more expensive the metric is to scrape.
+For non-production environments, avoid scraping these expensive metrics to save costs.

services/jorm-metrics-server/jorm_metrics_server/client.py

@@ -0,0 +1,201 @@
+"""This module provides a client for interacting with the Jormungandr API."""
+
+# cspell: words aiohttp loguru pydantic
+
+import asyncio
+import os
+from typing import Any
+from urllib.parse import urljoin
+
+import aiohttp
+from loguru import logger
+from pydantic import TypeAdapter
+
+from .models.account_by_id import AccountByID
+from .models.account_votes import AccountVotes
+from .models.account_votes_count import AccountVotesCount
+from .models.active_plans import VotePlan
+from .models.proposals import Proposal
+from .models.voter import Voter
+
+
+class ApiClient:
+    """A client for interacting with the Jormungandr API.
+
+    Attributes:
+        base_url (str): The base URL of the Jormungandr API.
+        session (aiohttp.ClientSession): The HTTP session used for requests.
+        storage (str): A path to a directory to store cached data.
+    """
+
+    base_url: str
+    session: aiohttp.ClientSession
+    storage: str
+
+    _proposals_cache: dict[int, Proposal]
+    _proposals_cache_path: str
+
+    _voter_cache: dict[str, Voter]
+    _voter_cache_path: str
+
+    def __init__(self, base_url: str, storage: str):
+        self.base_url = base_url
+        self.session = aiohttp.ClientSession()
+        self.storage = storage
+
+        self._proposals_cache_path = os.path.join(storage, "proposals.json")
+        self._voter_cache_path = os.path.join(storage, "voters.json")
+
+        adapter = TypeAdapter(dict[str, Voter])
+        if os.path.exists(self._voter_cache_path):
+            with open(self._voter_cache_path) as f:
+                self._voter_cache = adapter.validate_json(f.read())
+        else:
+            self._voter_cache = {}
+
+        adapter = TypeAdapter(dict[str, Proposal])
+        if os.path.exists(self._proposals_cache_path):
+            with open(self._proposals_cache_path) as f:
+                self._proposals_cache = adapter.validate_json(f.read())
+        else:
+            self._proposals_cache = {}
+
+    async def close(self):
+        """Closes the web client."""
+        await self.session.close()
+
+    async def get(self, path: str) -> Any:
+        """Make a GET request to the Jormungandr API.
+
+        Args:
+            path (str): The relative path to request.
+
+        Returns:
+            Any: The response from the Jormungandr API.
+        """
+        async with self.session.get(urljoin(self.base_url, path)) as response:
+            return await response.json()
+
+    async def get_account_by_id(self, account_id: str) -> AccountByID:
+        """Get the account information for a given account ID.
+
+        Args:
+            account_id (str): The account ID to get information for.
+
+        Returns:
+            Any: The account information for the given account ID.
+        """
+        return AccountByID.model_validate(
+            await self.get(f"api/v0/account/{account_id}")
+        )
+
+    async def get_account_votes(self, account_id: str):
+        """Get the proposal internal IDs that the account has voted on.
+
+        Args:
+            account_id (str): The account ID to get votes for.
+
+        Returns:
+            list[int]: The proposal internal IDs that the account has voted on.
+        """
+        adapter = TypeAdapter(list[AccountVotes])
+        resp = adapter.validate_python(
+            await self.get(f"api/v1/votes/plan/account-votes/{account_id}")
+        )
+
+        return sum([r.votes for r in resp], [])
+
+    async def get_accounts_votes(self) -> AccountVotesCount:
+        """Get the number of votes per wallet.
+
+        Returns:
+            AccountVotes: The number of votes per wallet.
+        """
+        return AccountVotesCount.model_validate(
+            {"votes": await self.get("api/v1/votes/plan/accounts-votes-count")}
+        )
+
+    async def get_active_plans(self) -> list[VotePlan]:
+        """Get all active plans.
+
+        Returns:
+            list[VotePlan]: All active plans.
+        """
+        adapter = TypeAdapter(list[VotePlan])
+        return adapter.validate_python(
+            await self.get("api/v0/vote/active/plans")
+        )
+
+    async def get_proposals(self) -> dict[str, Proposal]:
+        """Get all proposals.
+
+        Returns:
+            list[Proposal]: All proposals.
+        """
+        if self._proposals_cache:
+            return self._proposals_cache
+
+        adapter = TypeAdapter(list[Proposal])
+        proposals_data = adapter.validate_python(
+            await self.get("api/v0/proposals")
+        )
+        for proposal in proposals_data:
+            self._proposals_cache[proposal.chain_proposal_id] = proposal
+
+        adapter = TypeAdapter(dict[str, Proposal])
+        with open(self._proposals_cache_path, "wb") as f:
+            f.write(adapter.dump_json(self._proposals_cache))
+
+        return self._proposals_cache
+
+    async def get_proposal_by_id(self, id: int) -> Proposal:
+        """Get a proposal by its internal ID.
+
+        Args:
+            id (int): The internal ID of the proposal to get.
+
+        Returns:
+            Proposal: The proposal with the given internal ID.
+        """
+        return (await self.get_proposals())[id]
+
+    async def get_voters(self) -> dict[str, Voter]:
+        """Get all voters that have voted.
+
+        Note that this method uses a cache to avoid making too many requests to
+        the Jormungandr API. It will only make requests for voters that are
+        not already in the cache.
+
+        Returns:
+            dict[str, Voter]: A dictionary of voters
+        """
+        accounts_ids = (await self.get_accounts_votes()).votes.keys()
+        tasks = []
+        for account_id in accounts_ids:
+            if account_id not in self._voter_cache:
+                tasks.append(
+                    asyncio.create_task(self.update_voter_cache(account_id))
+                )
+
+        await asyncio.gather(*tasks)
+
+        adapter = TypeAdapter(dict[str, Voter])
+        with open(self._voter_cache_path, "wb") as f:
+            f.write(adapter.dump_json(self._voter_cache))
+
+        return self._voter_cache
+
+    async def update_voter_cache(self, account_id: str):
+        """Update the voter cache for a given account ID.
+
+        Args:
+            account_id (str): The account ID to update the voter cache with.
+        """
+        logger.info(f"Fetching voter details for {account_id}")
+        resp = await self.get_account_by_id(account_id)
+        voter = Voter(
+            id=account_id,
+            data=resp,
+            votes=await self.get_account_votes(account_id),
+        )
+        self._voter_cache[account_id] = voter