Feat/bootstrap ansible

ansible/inventory-bootstrap.yaml

@@ -0,0 +1,56 @@
+all:
+  children:
+    tnet:
+      hosts:
+        bootstrap-tnet-rust-ceramic-1.3box.io:
+          peers:
+            - /dns4/bootstrap-tnet-rust-ceramic-2.3box.io/tcp/4101/p2p/12D3KooWPFGbRHWfDaWt5MFFeqAHBBq3v5BqeJ4X7pmn2V1t6uNs
+          rust_ceramic_pk_name: "bootstrap-tnet-rust-ceramic-1-pk"
+        bootstrap-tnet-rust-ceramic-2.3box.io:
+          peers:
+            - /dns4/bootstrap-tnet-rust-ceramic-1.3box.io/tcp/4101/p2p/12D3KooWMqCFj5bnwuNi6D6KLhYiK4C8Eh9xSUKv2E6Jozs4nWEE
+          rust_ceramic_pk_name: "bootstrap-tnet-rust-ceramic-2-pk"
+      vars:
+        ceramic_network: testnet-clay
+        gcp_project: tnet-prod-2024
+        ssh_key_cd_pub: "{{ lookup('gcp_secret', gcp_project, 'bootstrap-tnet-prod-id_rsa-pub') }}"
+    mainnet:
+      hosts:
+        bootstrap-mainnet-rust-ceramic-1.3box.io:
+          peers:
+            - /dns4/bootstrap-mainnet-rust-ceramic-2.3box.io/tcp/4101/p2p/12D3KooWCuS388c1im7KkmdrpsLMziihF8mbcv2w6HPCp4Qmww6m
+          rust_ceramic_pk_name: "bootstrap-mainnet-rust-ceramic-1-pk"
+        bootstrap-mainnet-rust-ceramic-2.3box.io:
+          peers:
+            - /dns4/bootstrap-mainnet-rust-ceramic-1.3box.io/tcp/4101/p2p/12D3KooWJC1yR4KiCnocV9kuAEwtsMNh7Xmu2vzqpBvk2o3MrYd6
+          rust_ceramic_pk_name: "bootstrap-mainnet-rust-ceramic-2-pk"
+      vars:
+        ceramic_network: mainnet
+        gcp_project: tnet-prod-2024
+        ssh_key_cd_pub: "{{ lookup('gcp_secret', gcp_project, 'bootstrap-tnet-prod-id_rsa-pub') }}"
+    devqa:
+      hosts:
+        bootstrap-devqa-rust-ceramic-1.3box.io:
+          peers:
+            - /dns4/bootstrap-devqa-rust-ceramic-2.3box.io/tcp/4101/p2p/12D3KooWFCf7sKeW8NHoT35EutjJX5vCpPekYqa4hB4tTUpYrcam
+          rust_ceramic_pk_name: "bootstrap-devqa-rust-ceramic-1-pk"
+        bootstrap-devqa-rust-ceramic-2.3box.io:
+          peers:
+            - /dns4/bootstrap-devqa-rust-ceramic-1.3box.io/tcp/4101/p2p/12D3KooWJmYPnXgst4gW5GoyAYzRB3upLgLVR1oDVGwjiS9Ce7sA
+          rust_ceramic_pk_name: "bootstrap-devqa-rust-ceramic-2-pk"
+      vars:
+        ceramic_network: dev-unstable
+        gcp_project: dev-qa-2023
+        ssh_key_cd_pub: "{{ lookup('gcp_secret', gcp_project, 'bootstrap-devqa-id_rsa-pub') }}"
+
+  vars:
+    caddy_proxy_port: 8000
+    caddy_tls_email: [email protected]
+    internal_hostname: "{{ inventory_hostname.split('.')[0].replace('gitcoin-gcp-', '') }}"
+    ceramic_versions_path: /tmp
+    rust_ceramic_data_block_path: /dev/disk/by-id/google-rust-ceramic-data
+    rust_ceramic_data_mount_path: /rust_ceramic_data_disk
+    rust_ceramic_store_path: /rust_ceramic_data_disk/ceramic-one
+    rust_ceramic_pk_dir: /rust_ceramic_data_disk/keys
+    rust_ceramic_version: v0.25.0
+    rust_ceramic_download_url: "https://github.com/ceramicnetwork/rust-ceramic/releases/download/{{ rust_ceramic_version }}/ceramic-one_x86_64-unknown-linux-gnu.tar.gz"

ansible/pip.requirements.txt

@@ -1,10 +1,29 @@
 ansible==7.2.0
 ansible-core==2.14.2
+base58==2.1.1
+cachetools==5.3.3
+certifi==2024.7.4
 cffi==1.15.1
+charset-normalizer==3.3.2
 cryptography==39.0.1
+google-api-core==2.19.1
+google-auth==2.32.0
+google-cloud-secret-manager==2.20.1
+googleapis-common-protos==1.63.2
+grpc-google-iam-v1==0.13.1
+grpcio==1.65.0
+grpcio-status==1.65.0
+idna==3.7
 Jinja2==3.1.2
 MarkupSafe==2.1.2
 packaging==23.0
+proto-plus==1.24.0
+protobuf==5.27.2
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
 pycparser==2.21
 PyYAML==6.0
+requests==2.32.3
 resolvelib==0.8.1
+rsa==4.9
+urllib3==2.2.2

ansible/playbooks/bootstrap-update.yaml

@@ -0,0 +1,18 @@
+---
+- hosts: devqa
+  serial: 1
+  become: true
+  roles:
+    - rust-ceramic
+
+- hosts: tnet
+  serial: 1
+  become: true
+  roles:
+    - rust-ceramic
+
+- hosts: mainnet
+  serial: 1
+  become: true
+  roles:
+    - rust-ceramic

ansible/playbooks/bootstrap.yaml

@@ -0,0 +1,26 @@
+---
+- hosts: all
+  become: true
+  roles:
+    - name: ceramic-prep
+      import_role:
+        name: ceramic-prep
+      tags: ceramic-prep
+
+- hosts: all
+  become: true
+  roles:
+    - name: rust-ceramic
+      import_role:
+        name: rust-ceramic
+      tags: rust-ceramic
+
+    - name: bootstrap-ui
+      import_role:
+        name: bootstrap-ui
+      tags: bootstrap-ui
+
+    - name: caddy
+      import_role:
+        name: caddy
+      tags: caddy

ansible/playbooks/lookup_plugins/gcp_secret.py

@@ -0,0 +1,36 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import subprocess
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+from ansible.utils.display import Display
+
+display = Display()
+
+class LookupModule(LookupBase):
+
+    def run(self, terms, variables=None, **kwargs):
+        display.debug("GCP Secret lookup plugin called")
+        if len(terms) != 2:
+            raise AnsibleError("gcp_secret lookup expects 2 arguments: [project_id, secret_name]")
+
+        project_id, secret_name = terms
+        display.debug(f"Looking up secret {secret_name} in project {project_id}")
+
+        try:
+            cmd = [
+                "gcloud", "secrets", "versions", "access", "latest",
+                f"--secret={secret_name}",
+                f"--project={project_id}",
+                "--quiet"
+            ]
+            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+            display.debug("Secret accessed successfully using gcloud")
+        except subprocess.CalledProcessError as e:
+            display.error(f"Error accessing secret: {e}")
+            raise AnsibleError(f"Error accessing secret: {e.stderr}")
+
+        secret_value = result.stdout.strip()
+        display.v(f"Retrieved secret '{secret_name}' from project '{project_id}'")
+        return [secret_value]

ansible/roles/bootstrap-ui/files/bootstrap-ui.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Bootstrap UI
+After=network.target
+
+[Service]
+WorkingDirectory=/opt/ceramic-one-ui
+ExecStart=/opt/ceramic-one-ui/.venv/bin/python /opt/ceramic-one-ui/main.py
+Restart=always
+RestartSec=10
+Environment=PYTHONUNBUFFERED=1
+MemoryLimit=100M
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/bootstrap-ui/files/main.py

@@ -0,0 +1,165 @@
+import asyncio
+import signal
+from datetime import datetime
+from version import get_ceramic_one_version
+from peers import get_swarm_peers
+from fastapi import FastAPI
+from fastapi.responses import JSONResponse, HTMLResponse
+import uvicorn
+import aiohttp
+
+# Add configuration option for sleep interval (in seconds)
+CACHE_UPDATE_INTERVAL = 60  # Default: 30 seconds
+
+# Create a global event to signal shutdown
+shutdown_event = asyncio.Event()
+
+# Create a global dictionary to store the cache state
+cache_state = {}
+
+app = FastAPI()
+
+
+def handle_shutdown_signal():
+    print("Shutdown signal received. Exiting gracefully...")
+    shutdown_event.set()
+
+
+async def get_ceramic_id():
+    async with aiohttp.ClientSession() as session:
+        try:
+            async with session.post('http://localhost:5101/api/v0/id') as response:
+                if response.status == 200:
+                    data = await response.json()
+                    return {
+                        "ID": data.get("ID"),
+                        "Addresses": data.get("Addresses", [])
+                    }
+                else:
+                    return {"error": f"Failed to get Ceramic ID. Status: {response.status}"}
+        except aiohttp.ClientError as e:
+            return {"error": f"Failed to connect to Ceramic node: {str(e)}"}
+
+
+async def update_cache():
+    while not shutdown_event.is_set():
+        timestamp = datetime.now().isoformat()
+
+        # Get Ceramic version
+        ceramic_version = get_ceramic_one_version()
+
+        # Get swarm peers
+        swarm_peers = get_swarm_peers()
+
+        # Get Ceramic ID
+        ceramic_id = await get_ceramic_id()
+
+        # Create cache entry
+        cache_entry = {
+            "timestamp": timestamp,
+            "ceramic_version": ceramic_version,
+            "swarm_peers": swarm_peers,
+            "ceramic_id": ceramic_id
+        }
+        # Save cache entry to global dictionary
+        cache_state[timestamp] = cache_entry
+        print(f"Cache updated at {timestamp}")
+
+        # Use the configured interval
+        await asyncio.sleep(CACHE_UPDATE_INTERVAL)
+
+
+@app.get("/latest")
+async def get_latest_cache():
+    if not cache_state:
+        return JSONResponse(content={"error": "Cache is empty"}, status_code=404)
+    latest_timestamp = max(cache_state.keys())
+    return JSONResponse(content=cache_state[latest_timestamp])
+
+
+@app.get("/", response_class=HTMLResponse)
+async def get_root():
+    html_content = """
+    <!DOCTYPE html>
+    <html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>Ceramic bootstrap node</title>
+        <script>
+            async function fetchLatestCache() {
+                try {
+                    const response = await fetch('/latest');
+                    const data = await response.json();
+                    const formattedTimestamp = new Date(data.timestamp).toLocaleString();
+                    const formattedData = `
+                        <h2>Latest Cache Data</h2>
+                        <p><strong>Timestamp:</strong> ${formattedTimestamp}</p>
+                        <p><strong>Ceramic Version:</strong> ${data.ceramic_version}</p>
+                        <h3>Ceramic ID:</h3>
+                        <p><strong>ID:</strong> ${data.ceramic_id.ID}</p>
+                        <h4>Addresses:</h4>
+                        <ul>
+                            ${data.ceramic_id.Addresses ?
+                                data.ceramic_id.Addresses.map(address => `<li>${address}</li>`).join('') :
+                                '<li>No addresses available</li>'
+                            }
+                        </ul>
+                        <h3>Swarm Peers:</h3>
+                        <ul>
+                            ${data.swarm_peers.Peers ?
+                                data.swarm_peers.Peers.map(peer => `
+                                    <li>
+                                        <strong>Peer:</strong> ${peer.Peer}<br>
+                                        <strong>Addr:</strong> ${peer.Addr}<br>
+                                        <strong>Direction:</strong> ${peer.Direction}
+                                    </li>
+                                `).join('') :
+                                '<li>No peers available</li>'
+                            }
+                        </ul>
+                    `;
+                    document.getElementById('cacheData').innerHTML = formattedData;
+                } catch (error) {
+                    console.error('Error fetching cache data:', error);
+                    document.getElementById('cacheData').innerHTML = '<p>Error fetching cache data</p>';
+                }
+            }
+
+            // Fetch data immediately and then every 30 seconds
+            fetchLatestCache();
+            setInterval(fetchLatestCache, 30000);
+        </script>
+    </head>
+    <body>
+        <h1>Ceramic bootstrap node Cache Status</h1>
+        <div id="cacheData">Loading...</div>
+    </body>
+    </html>
+    """
+    return HTMLResponse(content=html_content)
+
+
+async def run_fastapi():
+    config = uvicorn.Config(app, host="127.0.0.1", port=8000, loop="asyncio")
+    server = uvicorn.Server(config)
+    await server.serve()
+
+
+async def main():
+    update_task = asyncio.create_task(update_cache())
+    fastapi_task = asyncio.create_task(run_fastapi())
+    await asyncio.gather(update_task, fastapi_task)
+
+if __name__ == "__main__":
+    print("Starting cache updater and FastAPI server...")
+    print(f"Cache update interval: {CACHE_UPDATE_INTERVAL} seconds")
+
+    # Register signal handlers
+    signal.signal(signal.SIGTERM, lambda s, f: handle_shutdown_signal())
+    signal.signal(signal.SIGINT, lambda s, f: handle_shutdown_signal())
+
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("Keyboard interrupt received. Exiting gracefully...")

ansible/roles/bootstrap-ui/files/peers.py

@@ -0,0 +1,36 @@
+import requests
+import json
+
+
+def get_swarm_peers():
+    url = "http://localhost:5101/api/v0/swarm/peers"
+
+    try:
+        # Send POST request
+        response = requests.post(url, timeout=10)
+
+        # Raise an exception for bad status codes
+        response.raise_for_status()
+
+        # Parse JSON response
+        json_response = response.json()
+
+        return json_response
+
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred while making the request: {e}")
+        return None
+
+    except json.JSONDecodeError as e:
+        print(f"Error decoding JSON response: {e}")
+        return None
+
+
+if __name__ == "__main__":
+    result = get_swarm_peers()
+
+    if result is not None:
+        print("Response received:")
+        print(json.dumps(result, indent=2))
+    else:
+        print("Failed to get a valid response.")