Dbp 1037 setup privacyidea monitring

roles/privacyidea_ha/defaults/main.yml

@@ -80,4 +80,8 @@ certificate_sync_target_directory: "/synced_certificates"
 ha_cluster_exporter_version: 1.3.3
 
 # The public key used to verify subscription signatures
-subscription_auth_public_key: ""
+subscription_auth_public_key: ""
+privacyidea_mariadb_root_password: ""
+
+exporter_username: "exporter"
+exporter_password: ""

roles/privacyidea_ha/handlers/main.yml

@@ -17,20 +17,26 @@
     enabled: true
     state: reloaded
 
-- name: reload nginx
+- name: restart uwsgi
   command:
-    cmd: /usr/sbin/pcs resource restart nginx
+    cmd: /usr/sbin/pcs resource restart uwsgi
   become: true
   ignore_errors: true
 
-- name: restart nginx
+- name: Restart exporter
   command:
-    cmd: /usr/sbin/pcs resource restart nginx
+    cmd: /usr/sbin/pcs resource restart privacyidea-exporter
   become: true
   ignore_errors: true
 
-- name: restart uwsgi
+- name: reload nginx
   command:
-    cmd: /usr/sbin/pcs resource restart uwsgi
+    cmd: /usr/sbin/pcs resource reload nginx
   become: true
   ignore_errors: true
+
+- name: restart nginx
+  command:
+    cmd: /usr/sbin/pcs resource restart nginx
+  become: true
+  ignore_errors: true

roles/privacyidea_ha/tasks/configure_privacyidea.yml

@@ -56,13 +56,6 @@
     group: www-data
     mode: '0644'
 
-#- name: Copy NetKnight pem file
-#  copy:
-#    src: NetKnights.pem
-#    dest: /etc/privacyidea/NetKnights.pem
-#    owner: www-data
-#    group: www-data
-#    mode: 0644
 
 # /etc/privacyidea/enckey is copied from /data/privacyidea
 - name: Set PrivacyIDEA file permissions

roles/privacyidea_ha/tasks/generate_keys.yml

@@ -63,3 +63,4 @@
         path: "/tmp/certificate_sync_key_*" 
         state: absent
       delegate_to: localhost
+      ignore_errors: true

roles/privacyidea_ha/tasks/hacluster_nginx.yml

@@ -62,3 +62,70 @@
     resource1: "nginx"
     resource2: "galera"
   when: is_main_node
+
+- name: Create the PrivacyIDEA cronjob
+  copy:
+    dest: /etc/privacyidea/privacyidea_cron
+    content: |
+      # Script to run scheduled tasks for PrivacyIDEA
+      */5 * * * * www-data /opt/privacyidea/virtualenv/bin/privacyidea-cron run_scheduled
+    mode: '0644'
+
+- name: Create Symlink for Cron Job
+  pcs_resource:
+    name: "privacyidea_cron_symlink"
+    resource_type: 'ocf:heartbeat:symlink'
+    options: >-
+      target=/etc/privacyidea/privacyidea_cron
+      link=/etc/cron.d/privacyidea_cron
+      backup_suffix=.disabled
+      op monitor interval=10s timeout=20s
+      op start timeout=60s interval=0s
+      op stop timeout=60s interval=0s
+      meta failure-timeout=15m migration-threshold=2
+      --group {{ cluster_name }}_group
+  when: is_main_node
+
+- name: Add Order Constraint for Symlink after uWSGI
+  pcs_constraint_order:
+    resource1: "uwsgi"
+    resource2: "privacyidea_cron_symlink"
+  when: is_main_node
+  become: true
+
+- name: Add Colocation Constraint for Symlink with uWSGI
+  pcs_constraint_colocation:
+    resource1: "privacyidea_cron_symlink"
+    resource2: "uwsgi"
+    score: "INFINITY"
+  when: is_main_node
+  become: true
+
+- name: Add custom exporter resource
+  pcs_resource:
+     name: "custom-exporter"
+     resource_type: 'systemd:privacyidea-exporter'
+     options: >
+       op monitor timeout=1s interval=1s 
+       meta failure-timeout=15m migration-threshold=2
+       --group {{ cluster_name }}_group
+  when: is_main_node
+
+- name: Add colocation for custom exporter with uwsgi
+  pcs_constraint_colocation:
+    resource1: "custom-exporter"
+    resource2: "uwsgi"
+    score: "INFINITY"
+  when: is_main_node
+
+- name: Add order for galera and then custom-exporter
+  pcs_constraint_order:
+    resource1: "galera"
+    resource2: "custom-exporter"
+  when: is_main_node
+
+#- name: Start custom exporter PCS resource
+#  pcs_resource:
+#    name: "custom-exporter"
+#    state: started
+#  when: is_main_node

roles/privacyidea_ha/tasks/import_data.yml

@@ -7,7 +7,9 @@
     group: www-data
     mode: '0755'
 
-# - name: Execute the privacyidea_data_import_and_cleanup.sh script
-#   shell: /etc/privacyidea/privacyidea_data_import_and_cleanup.sh
-#   args:
-#     executable: /bin/bash
+#- name: Execute the privacyidea_data_import_and_cleanup.sh script
+#  shell: /etc/privacyidea/privacyidea_data_import_and_cleanup.sh
+#  args:
+#    executable: /bin/bash
+#  notify: restart uwsgi
+

roles/privacyidea_ha/tasks/install_packages.yml

@@ -5,3 +5,7 @@
     state: present
     update_cache: yes
 
+- name: Install prometheus_client in PrivacyIDEA
+  ansible.builtin.command:
+  cmd: sudo -H /opt/privacyidea/virtualenv/bin/pip install prometheus_client
+  become: true

roles/privacyidea_ha/tasks/main.yml

@@ -2,10 +2,10 @@
 - import_tasks: prepare.yml
 - import_tasks: setup_directories.yml
 - import_tasks: install_privacyidea.yml
-- import_tasks: setup_database.yml
 - import_tasks: generate_keys.yml
 - import_tasks: configure_privacyidea.yml
 - import_tasks: create_admin.yml
+- import_tasks: privacyidea_exporter.yml
 - import_tasks: setup_nginx.yml
 - import_tasks: security.yml
 - import_tasks: hacluster.yml

roles/privacyidea_ha/tasks/privacyidea_exporter.yml

@@ -0,0 +1,35 @@
+
+---
+- name: Deploy custom exporter script
+  template:  
+    src: custom_exporter.py.j2 
+    dest: /etc/privacyidea/custom_exporter.py
+    owner: www-data
+    group: www-data
+    mode: '0775'
+
+- name: Create uWSGI configuration for custom exporter
+  template:
+    src: privacyidea-exporter.ini.j2
+    dest: /etc/uwsgi/apps-available/privacyidea-exporter.ini
+    owner: www-data
+    group: www-data
+    mode: '0644'
+
+- name: Link uWSGI config for custom exporter
+  file:
+    src: /etc/uwsgi/apps-available/privacyidea-exporter.ini
+    dest: /etc/uwsgi/apps-enabled/privacyidea-exporter.ini
+    state: link
+
+- name: Deploy custom exporter Systemd service
+  template:
+    src: privacyidea-exporter.service.j2
+    dest: /etc/systemd/system/privacyidea-exporter.service
+    owner: www-data
+    group: www-data
+    mode: '0644'
+
+- name: Reload systemd daemon
+  systemd:
+    daemon_reload: true

roles/privacyidea_ha/tasks/setup_directories.yml

@@ -12,6 +12,8 @@
     - { path: '/var/log/privacyidea', mode: '0755' }
     - { path: '/var/log/uwsgi', mode: '0755' }
     - { path: '/usr/lib/ocf/resource.d/privacyidea/', mode: '0755' }
+    - { path: '/run/uwsgi/app/privacyidea-exporter/', mode: '0775' }
+
   become: true
 
 - name: Create uWSGI and privacyidea log files
@@ -26,4 +28,5 @@
     - { path: '/var/log/uwsgi/error.log' }
     - { path: '/var/log/uwsgi/request.log' }
     - { path: '/var/log/privacyidea/privacyidea.log' }
-  become: true
+    - { path: '/var/log/uwsgi/privacyidea-exporter.log' }
+  become: true

roles/privacyidea_ha/templates/custom_exporter.py.j2

@@ -0,0 +1,116 @@
+import os
+import sys
+import socket
+import time
+import logging
+from prometheus_client import start_http_server, Gauge, Counter, generate_latest
+from privacyidea.app import create_app
+from flask import Flask, Response, request
+import sqlalchemy
+from sqlalchemy.pool import QueuePool
+from functools import wraps
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Get the hostname
+hostname = socket.gethostname()
+
+config_name =  "{{ config_name }}"
+debug_mode = {{ debug_mode }}
+
+# Create the PrivacyIDEA application
+application = create_app(config_name=config_name, config_file="/etc/privacyidea/pi.cfg")
+application.debug = debug_mode
+
+# Get credentials from environment variables
+EXPORTER_USERNAME = os.environ.get('EXPORTER_USERNAME')
+EXPORTER_PASSWORD = os.environ.get('EXPORTER_PASSWORD')
+
+def check_auth(username, password):
+    """Check if a username/password combination is valid."""
+    return username == EXPORTER_USERNAME and password == EXPORTER_PASSWORD
+
+def authenticate():
+    """Sends a 401 response that enables basic auth"""
+    return Response(
+        'Could not verify your access level for that URL.\n'
+        'You have to login with proper credentials', 401,
+        {'WWW-Authenticate': 'Basic realm="Login Required"'})
+
+def requires_auth(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        auth = request.authorization
+        if not auth or not check_auth(auth.username, auth.password):
+            return authenticate()
+        return f(*args, **kwargs)
+    return decorated
+
+# Define the /metrics endpoint function outside the PrivacyIDEAExporter class
+@requires_auth
+def metrics():
+    exporter.collect_metrics()
+    return Response(generate_latest(), mimetype="text/plain")
+
+class PrivacyIDEAExporter:
+    def __init__(self, app, port=9101):
+        self.app = app
+        self.port = port
+        self.flask_app = Flask(__name__)
+
+        # Define Prometheus metrics
+        self.auth_success = Gauge('privacyidea_auth_success_total', 'Total successful authentications')
+        self.auth_failure = Gauge('privacyidea_auth_failure_total', 'Total failed authentications')
+        self.tokens_total = Gauge('privacyidea_tokens_total', 'Total number of tokens')
+        self.hardware_tokens = Gauge('privacyidea_hardware_tokens', 'Number of hardware tokens')
+        self.software_tokens = Gauge('privacyidea_software_tokens', 'Number of software tokens')
+        self.unassigned_hardware_tokens = Gauge('privacyidea_unassigned_hardware_tokens', 'Number of unassigned hardware tokens')
+        self.assigned_tokens = Gauge('privacyidea_assigned_tokens', 'Number of assigned tokens')
+        self.new_users = Gauge('privacyidea_new_users', 'Number of new users')
+        self.token_init = Gauge('privacyidea_token_init', 'Number of token initializations')
+        self.token_assign = Gauge('privacyidea_token_assign', 'Number of token assignments')
+        self.token_unassign = Gauge('privacyidea_token_unassign', 'Number of token unassignments')
+
+        # Create a connection pool
+        self.engine = sqlalchemy.create_engine(
+            application.config['SQLALCHEMY_DATABASE_URI'],
+            poolclass=QueuePool,
+            pool_size=5,
+            max_overflow=10
+        )
+
+    def fetch_latest_stat(self, connection, stats_key):
+        query = """
+        SELECT stats_value 
+        FROM monitoringstats 
+        WHERE stats_key = :key 
+        ORDER BY timestamp DESC 
+        LIMIT 1
+        """
+        result = connection.execute(sqlalchemy.text(query), {"key": stats_key})
+        return result.scalar() or 0
+
+    def collect_metrics(self):
+        with self.app.app_context():
+            try:
+                with self.engine.connect() as connection:
+                    # Fetch and set metrics
+                    self.tokens_total.set(self.fetch_latest_stat(connection, 'total_tokens'))
+                    self.hardware_tokens.set(self.fetch_latest_stat(connection, 'hardware_tokens'))
+                    self.software_tokens.set(self.fetch_latest_stat(connection, 'software_tokens'))
+                    self.unassigned_hardware_tokens.set(self.fetch_latest_stat(connection, 'unassigned_hardware_tokens'))
+                    self.assigned_tokens.set(self.fetch_latest_stat(connection, 'assigned_tokens'))
+                    self.auth_success.set(self.fetch_latest_stat(connection, 'successful_auth_counter'))
+                    self.auth_failure.set(self.fetch_latest_stat(connection, 'failed_auth_counter'))
+                    self.new_users.set(self.fetch_latest_stat(connection, 'new_users_counter'))
+                    self.token_init.set(self.fetch_latest_stat(connection, 'token_init_counter'))
+                    self.token_assign.set(self.fetch_latest_stat(connection, 'token_assign_counter'))
+                    self.token_unassign.set(self.fetch_latest_stat(connection, 'token_unassign_counter'))
+
+                logging.info("Metrics collected successfully")
+            except Exception as e:
+                logging.error(f"Error fetching metrics: {e}", exc_info=True)
+
+exporter = PrivacyIDEAExporter(application)
+application.add_url_rule('/metrics', 'metrics', metrics)

roles/privacyidea_ha/templates/nginx_privacyidea.conf.j2

@@ -19,7 +19,7 @@
 #    #}
 
 #    location / {
-#        include uwsgi_params;
+#        include /etc/nginx/uwsgi_params;
 #        uwsgi_pass unix:/run/uwsgi/app/privacyidea/privacyidea.socket;
 #    }
 #}
@@ -44,8 +44,14 @@ server {
     {% endif %}
     access_log /var/log/nginx/privacyidea.access.log privacyidea;
     error_log /var/log/nginx/privacyidea.error.log;
+
+    location /metrics {
+        include /etc/nginx/uwsgi_params;
+        uwsgi_pass unix:/run/uwsgi/app/privacyidea-exporter/socket;
+    }
+
     location / {
-        include uwsgi_params;
+        include /etc/nginx/uwsgi_params;
         uwsgi_pass unix:/run/uwsgi/app/privacyidea/privacyidea.socket;
     }
     location /static {

roles/privacyidea_ha/templates/pi.cfg.j2

@@ -24,4 +24,6 @@ PI_ENCFILE = '/etc/privacyidea/enckey'
 # This is used to encrypt the admin passwords
 PI_PEPPER = '{{ privacyidea_pi_pepper }}'
 SECRET_KEY = '{{ privacyidea_secret_key }}'
-PI_PREFERRED_LANGUAGE = ["de"]
+PI_PREFERRED_LANGUAGE = ["de"]
+PI_AUDIT_SQL_TRUNCATE = True
+PI_AUDIT_SQL_COLUMN_LENGTH = {"info": 1000, "action": 100, "action_detail": 100, "policies": 1000}