Add rpm name search and integration testing

.github/workflows/tang-actions.yaml

@@ -9,13 +9,59 @@ on:
     paths-ignore:
       - '**.md'
 jobs:
-  govet:
-    name: Vet
+  golangci:
+    name: Lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
           go-version: "1.20"
-      - run: |
-          go vet ./...
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.54.2
+          skip-go-installation: true
+          args: --timeout=5m
+  gotest:
+    name: Test
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "1.20"
+      - name: start pulp
+        uses: isbang/[email protected]
+        with:
+          compose-file: ./compose_files/pulp/docker-compose.yml
+          down-flags: --volumes
+      - name: Wait for pulp
+        run: |
+          docker run --network=host --rm -v ${PWD}:/local curlimages/curl  \
+          curl --retry-all-errors --fail --retry-delay 10 --retry 32 --retry-max-time 240 http://localhost:8087/pulp/default/api/v3/repositories/rpm/rpm/  -u admin:password
+          sleep 30
+      - name: integration tests
+        run: |
+          make test-integration
+        env:
+          DATABASE_HOST: localhost
+          DATABASE_PORT: 5434
+          DATABASE_USER: pulp
+          DATABASE_NAME: pulp
+          DATABASE_PASSWORD: password
+          SERVER_URL: http://localhost:8087
+          SERVER_USERNAME: admin
+          SERVER_PASSWORD: password

.gitignore

@@ -29,4 +29,7 @@ release/
 Containerfile
 
 # Pulp Oci Images
-compose_files/pulp/pulp-oci-images
+compose_files/pulp/pulp-oci-images
+
+#config
+configs/config.yaml

.golangci.yaml

@@ -0,0 +1,14 @@
+# Configuration for golangci-lint. See https://golangci-lint.run/usage/configuration/.
+linters:
+  disable-all: false # use default linters
+  enable:
+    - gofmt
+    - whitespace
+    - govet
+    - misspell
+    - forcetypeassert
+    - gci
+    - bodyclose
+issues:
+  exclude:
+    - composite

Makefile

@@ -0,0 +1,23 @@
+##
+# Entrypoint for the Makefile
+#
+# It is composed at mk/includes.mk by including
+# small make files which provides all the necessary
+# rules.
+#
+# Some considerations:
+#
+# - Variables customization can be
+#   stored at 'config.env', 'mk/private.mk' files.
+# - By default the 'help' rule is executed.
+# - No parallel jobs are executed from the main Makefile,
+#   so that multiple rules from the command line will be
+#   executed in serial.
+##
+
+include mk/includes.mk
+
+.NOT_PARALLEL:
+
+# Set the default rule
+.DEFAULT_GOAL := help

README.md

@@ -1 +1,63 @@
 # tang
+
+The tangy package provides methods to read from a [pulp](https://pulpproject.org/) database.
+
+## Installation
+`go get github.com/content-services/tang`
+
+## Usage
+The tangy package is meant to be imported into an existing project that is using pulp. It can be used like this:
+```go
+// Pulp database configuration information
+dbConfig := tangy.Database{
+    Name:       "pulp",
+    Host:       "localhost",
+    Port:       5434,
+    User:       "pulp",
+    Password:   "password",
+    CACertPath: "",
+    PoolLimit:  20,
+}
+
+// Create new Tangy instance using database config
+t, err := tangy.New(dbConfig, tangy.Logger{Enabled: false})
+if err != nil {
+    return err
+}
+defer t.Close()
+
+// Use Tangy to search for RPMs, by name, that are associated to a specific repository version, returning up to the first 100 results
+versionHref := "/pulp/e1c6bee3/api/v3/repositories/rpm/rpm/018c1c95-4281-76eb-b277-842cbad524f4/versions/1/"
+rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja", 100)
+if err != nil {
+return err
+}
+```
+See example.go for a complete example.
+
+## Developing
+To develop for tangy, there are a few more things to know.
+
+### Create your configuration
+`$ cp ./configs/config.yaml.example ./configs/config.yaml`
+
+### Connecting to pulp
+
+#### Connect to an existing pulp server
+To connect to an existing pulp server, put the corresponding connection information in `configs/config.yaml`.
+
+#### Create a new pulp server
+To create a new pulp server, you can use the provided make commands. You will need to have podman & podman-compose (or docker) installed.
+The default values provided in config.yaml.example will work with this server.
+
+##### Start containers
+`make compose-up`
+
+#### Stop containers
+`make compose-down`
+
+#### Clean container volumes
+`make compose-clean`
+
+### Mocking
+Tangy also exports a mock interface you can regenerate using the [mockery](https://github.com/vektra/mockery) tool.

compose_files/pulp/assets/bin/nginx.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# This logic enables us to have multiple servers, and check to see
+# if they are scaled every 10 seconds.
+# https://serverfault.com/a/821625/189494
+# https://www.nginx.com/blog/dns-service-discovery-nginx-plus#domain-name-variable
+
+set -e
+
+if [ "$container" != "podman" ]; then
+  # the nameserver list under podman is unreliable.
+  # It will look like "10.89.1.1 192.168.1.1 192.168.1.1", but only the 1st IP works.
+  # This doesn't mess up `nslookup`, but it messes up `getent hosts` and nginx.
+  export NAMESERVER=`cat /etc/resolv.conf | grep "nameserver" | awk '{print $2}' | head -n1`
+else
+  export NAMESERVER=`cat /etc/resolv.conf | grep "nameserver" | awk '{print $2}' | tr '\n' ' '`
+fi
+
+echo "Nameserver is: $NAMESERVER"
+
+echo "Generating nginx config"
+envsubst '$NAMESERVER' < /etc/opt/rh/rh-nginx116/nginx/nginx.conf.template > /etc/opt/rh/rh-nginx116/nginx/nginx.conf
+
+# We cannot use upstream server groups with a DNS resolver without nginx plus
+# So we modifying the files to use the variables rather than the upstream server groups
+for file in /opt/app-root/etc/nginx.default.d/*.conf ; do
+  echo "Modifying $file"
+  sed -i 's/pulp-api/$pulp_api:24817/' $file
+  sed -i 's/pulp-content/$pulp_content:24816/' $file
+done
+
+echo "Starting nginx"
+exec nginx  -g "daemon off;"

compose_files/pulp/assets/certs/database_fields.symmetric.key

@@ -0,0 +1 @@
+DNmNdwgyZugTax9S64J0FITTr9IHPxbuoF1F1CGPr68=

compose_files/pulp/assets/nginx/nginx.conf.template

@@ -0,0 +1,89 @@
+error_log /dev/stdout info;
+worker_processes 1;
+events {
+    worker_connections 1024; # increase if you have lots of clients
+    accept_mutex off; # set to 'on' if nginx worker_processes > 1
+}
+
+http {
+    access_log /dev/stdout;
+    include mime.types;
+    # fallback in case we can't determine a type
+    default_type application/octet-stream;
+    sendfile on;
+
+    # If left at the default of 1024, nginx emits a warning about being unable
+    # to build optimal hash types.
+    types_hash_max_size 4096;
+
+    server {
+        # This logic enables us to have multiple servers, and check to see
+        # if they are scaled every 10 seconds.
+        # https://www.nginx.com/blog/dns-service-discovery-nginx-plus#domain-name-variable
+        # https://serverfault.com/a/821625/189494
+        resolver $NAMESERVER valid=10s;
+        set $pulp_api pulp_api;
+        set $pulp_content pulp_content;
+
+        # Gunicorn docs suggest the use of the "deferred" directive on Linux.
+        listen 8080 default_server deferred;
+        listen [::]:8080 default_server deferred;
+
+        # If you have a domain name, this is where to add it
+        server_name $hostname;
+
+        # The default client_max_body_size is 1m. Clients uploading
+        # files larger than this will need to chunk said files.
+        client_max_body_size 10m;
+
+        # Gunicorn docs suggest this value.
+        keepalive_timeout 5;
+
+        # static files that can change dynamically, or are needed for TLS
+        # purposes are served through the webserver.
+        root /opt/app-root/src;
+
+        location /pulp/content/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://$pulp_content:24816;
+        }
+
+        location /pulp/api/v3/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://$pulp_api:24817;
+        }
+
+        location /auth/login/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://$pulp_api:24817;
+        }
+
+        include /opt/app-root/etc/nginx.default.d/*.conf;
+
+        location / {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://$pulp_api:24817;
+            # static files are served through whitenoise - http://whitenoise.evans.io/en/stable/
+        }
+    }
+}

compose_files/pulp/assets/postgres/passwd

@@ -0,0 +1,20 @@
+root:x:0:0:root:/root:/bin/bash
+daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin
+bin:x:2:2:bin:/bin:/usr/sbin/nologin
+sys:x:3:3:sys:/dev:/usr/sbin/nologin
+sync:x:4:65534:sync:/bin:/bin/sync
+games:x:5:60:games:/usr/games:/usr/sbin/nologin
+man:x:6:12:man:/var/cache/man:/usr/sbin/nologin
+lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin
+mail:x:8:8:mail:/var/mail:/usr/sbin/nologin
+news:x:9:9:news:/var/spool/news:/usr/sbin/nologin
+uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin
+proxy:x:13:13:proxy:/bin:/usr/sbin/nologin
+www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin
+backup:x:34:34:backup:/var/backups:/usr/sbin/nologin
+list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin
+irc:x:39:39:ircd:/run/ircd:/usr/sbin/nologin
+gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin
+nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin
+_apt:x:100:65534::/nonexistent:/usr/sbin/nologin
+postgres:x:26:26::/var/lib/postgresql:/bin/bash

compose_files/pulp/assets/settings.py

@@ -0,0 +1,18 @@
+SECRET_KEY = "aabbcc"
+CONTENT_ORIGIN = "http://pulp_content:24816"
+DATABASES = {"default": {"HOST": "postgres", "ENGINE": "django.db.backends.postgresql", "NAME": "pulp", "USER": "pulp", "PASSWORD": "password", "PORT": "5432", "CONN_MAX_AGE": 0, "OPTIONS": {"sslmode": "prefer"}}}
+CACHE_ENABLED = True
+REDIS_HOST = "redis"
+REDIS_PORT = 6379
+REDIS_PASSWORD = ""
+ANSIBLE_API_HOSTNAME = "http://pulp_api:24817"
+ANSIBLE_CONTENT_HOSTNAME = "http://pulp_content:24816/pulp/content"
+ALLOWED_IMPORT_PATHS = ["/tmp"]
+ALLOWED_EXPORT_PATHS = ["/tmp"]
+TOKEN_SERVER = "http://pulp_api:24817/token/"
+TOKEN_AUTH_DISABLED = False
+TOKEN_SIGNATURE_ALGORITHM = "ES256"
+PUBLIC_KEY_PATH = "/etc/pulp/keys/container_auth_public_key.pem"
+PRIVATE_KEY_PATH = "/etc/pulp/keys/container_auth_private_key.pem"
+TELEMETRY = False
+STATIC_ROOT = "/var/lib/operator/static/"