Merge branch 'main' of ssh://github.com/linuxfoundation/lfx-v2-query-service into andrest50/query-direct-fields

Author: andrest50

CLAUDE.md

@@ -110,3 +110,76 @@ Environment variables control implementation selection:
 - Unit tests use mock implementations
 - Integration tests can switch between real and mock implementations
 - Test files follow `*_test.go` pattern alongside implementation files
+
+## CEL Filter Feature
+
+The service supports Common Expression Language (CEL) filtering for post-query resource filtering.
+
+### Overview
+
+CEL filtering allows API consumers to filter resources on arbitrary data fields using a safe, non-Turing complete expression language. The filter is applied after the OpenSearch query but before access control checks.
+
+### Implementation Details
+
+**Location**: `internal/infrastructure/filter/cel_filter.go`
+
+**Key Components**:
+- **ResourceFilter Interface** (`internal/domain/port/filter.go`): Domain interface for filtering
+- **CELFilter Implementation**: Uses `google/cel-go` library for expression evaluation
+- **Expression Caching**: LRU cache with TTL for compiled CEL programs
+- **Security Features**: Max expression length (1000 chars), evaluation timeout (100ms per resource)
+
+**Integration Point**: `internal/service/resource_search.go` (lines 84-102)
+- CEL filter applied after OpenSearch query
+- Filters resources before access control checks
+- Reduces number of access control checks needed
+
+### Available Variables in CEL Expressions
+
+- `data` (map): Resource data object
+- `resource_type` (string): Resource type
+- `id` (string): Resource ID
+
+Note: `type` is a reserved word in CEL, so we use `resource_type` instead.
+
+### Example Usage
+
+```go
+// API call
+GET /query/resources?type=project&cel_filter=data.slug == "tlf"
+
+// Expression is evaluated against each resource after OpenSearch query
+// Only matching resources proceed to access control checks
+```
+
+### Adding CEL Filter Tests
+
+When writing tests that involve resource search:
+
+```go
+// Use MockResourceFilter for testing
+mockFilter := mock.NewMockResourceFilter()
+
+// Pass to service constructor
+service := service.NewResourceSearch(mockSearcher, mockAccessChecker, mockFilter)
+```
+
+### Common CEL Operations
+
+- Equality: `data.status == "active"`
+- Comparison: `data.priority > 5`
+- Boolean logic: `data.status == "active" && data.priority > 5`
+- String operations: `data.name.contains("LF")`
+- List membership: `data.category in ["security", "networking"]`
+- Field existence: `has(data.archived)`
+
+### Performance Considerations
+
+- Compiled CEL programs are cached (100 max entries, 5-minute TTL)
+- Each resource evaluation has 100ms timeout
+- Post-query filtering means pagination may return fewer results than page size
+- For best performance, use specific OpenSearch criteria first, then CEL for refinement
+
+### Important Limitations
+
+**Pagination**: CEL filters apply only to results from each OpenSearch page. If the target resource is not in the first page of OpenSearch results, it won't be found even if it matches the CEL filter. Always use specific primary search criteria (`type`, `name`, `parent`) to narrow OpenSearch results first.

Makefile

@@ -26,6 +26,8 @@ GOLANGCI_LINT_VERSION := v2.2.2
 LINT_TIMEOUT := 10m
 LINT_TOOL=$(shell go env GOPATH)/bin/golangci-lint
 
+GOA_VERSION := v3.22.6
+
 ##@ Development
 
 .PHONY: setup-dev
@@ -43,7 +45,7 @@ setup: ## Setup development environment
 .PHONY: deps
 deps: ## Install dependencies
 	@echo "Installing dependencies..."
-	go install goa.design/goa/v3/cmd/goa@latest
+	go install goa.design/goa/v3/cmd/goa@$(GOA_VERSION)
 
 .PHONY: apigen
 apigen: deps #@ Generate API code using Goa

README.md

@@ -214,7 +214,9 @@ Authorization: Bearer <jwt_token>
 - `name`: Resource name or alias (supports typeahead search)
 - `type`: Resource type to filter by
 - `parent`: Parent resource for hierarchical queries
-- `tags`: Array of tags to filter by
+- `tags`: Array of tags to filter by (OR logic)
+- `tags_all`: Array of tags where all must match (AND logic)
+- `cel_filter`: CEL expression for advanced post-query filtering (see [CEL Filter](#cel-filter) section)
 - `sort`: Sort order (name_asc, name_desc, updated_asc, updated_desc)
 - `page_token`: Pagination token
 - `v`: API version (required)
@@ -239,6 +241,119 @@ Authorization: Bearer <jwt_token>
 }
 ```
 
+#### CEL Filter
+
+The `cel_filter` query parameter enables advanced filtering of search results using Common Expression Language (CEL). CEL is a non-Turing complete expression language designed for safe, fast evaluation of expressions in performance-critical applications.
+
+**Why CEL Filter?**
+
+CEL filtering was added to provide flexible, dynamic filtering capabilities on arbitrary resource data fields without modifying the OpenSearch query structure. This allows API consumers to:
+
+- Filter on any field within the resource data
+- Combine multiple conditions with boolean logic
+- Perform complex comparisons beyond simple equality checks
+- Apply filters without requiring backend code changes
+
+**What is CEL?**
+
+CEL (Common Expression Language) is an open-source expression language developed by Google. It provides:
+
+- **Safety**: Non-Turing complete, no side effects, no infinite loops
+- **Performance**: Linear time evaluation with compilation and caching
+- **Portability**: Language-agnostic with implementations in multiple languages
+- **Security**: Execution timeouts and resource constraints
+
+Learn more: [CEL Specification](https://github.com/google/cel-spec) | [CEL-Go Documentation](https://github.com/google/cel-go)
+
+**How It Works**
+
+CEL filters are applied **after** the OpenSearch query executes but **before** access control checks. This means:
+
+1. OpenSearch returns initial results based on primary search criteria (`type`, `name`, `parent`, `tags`)
+2. CEL filter evaluates each resource and removes non-matching items
+3. Access control checks are performed only on filtered results (improved performance)
+4. Final results are returned to the client
+
+**Available Variables**
+
+CEL expressions have access to the following variables for each resource:
+
+- `data` (map): The resource's data object containing all custom fields
+- `resource_type` (string): The type of the resource (e.g., "project", "committee")
+- `id` (string): The unique identifier of the resource
+
+**Security Constraints**
+
+- **Maximum expression length**: 1000 characters
+- **Evaluation timeout**: 100ms per resource
+- **Expression caching**: Compiled programs cached with LRU and 5-minute TTL
+- **No external access**: Cannot make network calls or access filesystem
+
+**Usage Examples**
+
+Filter projects by slug:
+```
+GET /query/resources?type=project&cel_filter=data.slug == "tlf"&v=1
+```
+
+Filter by status and priority:
+```
+GET /query/resources?type=project&cel_filter=data.status == "active" && data.priority > 5&v=1
+```
+
+Filter by resource type:
+```
+GET /query/resources?parent=org:123&cel_filter=resource_type == "committee"&v=1
+```
+
+Complex boolean logic:
+```
+GET /query/resources?type=project&cel_filter=data.status == "active" || (data.priority > 8 && data.category == "security")&v=1
+```
+
+String operations:
+```
+GET /query/resources?type=project&cel_filter=data.name.contains("LF") && data.description.startsWith("Open")&v=1
+```
+
+Check field existence:
+```
+GET /query/resources?type=project&cel_filter=has(data.archived) && data.archived == false&v=1
+```
+
+List membership:
+```
+GET /query/resources?type=project&cel_filter=data.category in ["security", "networking", "storage"]&v=1
+```
+
+Nested field access:
+```
+GET /query/resources?type=project&cel_filter=data.metadata.owner == "admin" && data.metadata.region == "us-west"&v=1
+```
+
+**Supported Operators**
+
+- **Comparison**: `==`, `!=`, `<`, `<=`, `>`, `>=`
+- **Logical**: `&&` (AND), `||` (OR), `!` (NOT)
+- **Arithmetic**: `+`, `-`, `*`, `/`, `%`
+- **String**: `contains()`, `startsWith()`, `endsWith()`, `matches()` (regex)
+- **Membership**: `in`
+- **Field check**: `has()`
+
+**Important Limitations**
+
+⚠️ **Pagination Consideration**: CEL filters are applied to the results from each OpenSearch page. If you're looking for a specific resource that matches your CEL filter but it's not in the first page of OpenSearch results, it may not be found. For best results when using CEL filters, use more specific primary search parameters (`type`, `name`, `parent`, `tags`) to narrow down the OpenSearch results first.
+
+**Error Handling**
+
+Invalid CEL expressions return a 400 Bad Request with details:
+
+```json
+{
+  "error": "filter expression failed: ERROR: <input>:1:6: Syntax error: mismatched input 'invalid' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}"
+}
+```
+
 #### Organization Search API
 
 **Query Organizations:**

charts/lfx-v2-query-service/Chart.yaml

@@ -5,5 +5,5 @@ apiVersion: v2
 name: lfx-v2-query-service
 description: LFX Platform V2 Query Service chart
 type: application
-version: 0.4.9
+version: 0.4.10
 appVersion: "latest"

cmd/main.go

@@ -59,13 +59,14 @@ func main() {
 	accessControlChecker := service.AccessControlCheckerImpl(ctx)
 	organizationSearcher := service.OrganizationSearcherImpl(ctx)
 	authService := service.AuthServiceImpl(ctx)
+	resourceFilter := service.ResourceFilterImpl(ctx)
 
 	// Initialize the services.
 	var (
 		querySvcSvc querysvc.Service
 	)
 	{
-		querySvcSvc = service.NewQuerySvc(resourceSearcher, accessControlChecker, organizationSearcher, authService)
+		querySvcSvc = service.NewQuerySvc(resourceSearcher, accessControlChecker, resourceFilter, organizationSearcher, authService)
 	}
 
 	// Wrap the services in endpoints that can be invoked from other services

cmd/service/converters.go

@@ -58,6 +58,7 @@ func (s *querySvcsrvc) payloadToCriteria(ctx context.Context, p *querysvc.QueryR
 		Tags:         p.Tags,
 		TagsAll:      p.TagsAll,
 		Filters:      filters,
+		CelFilter:    p.CelFilter,
 		SortBy:       p.Sort,
 		PageToken:    p.PageToken,
 		PageSize:     constants.DefaultPageSize,

cmd/service/converters_test.go

@@ -20,7 +20,7 @@ func TestPayloadToCriteria(t *testing.T) {
 	mockAccessChecker := mock.NewMockAccessControlChecker()
 	mockOrgSearcher := mock.NewMockOrganizationSearcher()
 	mockAuth := mock.NewMockAuthService()
-	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mockOrgSearcher, mockAuth)
+	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mock.NewMockResourceFilter(), mockOrgSearcher, mockAuth)
 	svc := service.(*querySvcsrvc)
 
 	// Setup environment variable for page token secret
@@ -165,7 +165,7 @@ func TestDomainResultToResponse(t *testing.T) {
 	mockAccessChecker := mock.NewMockAccessControlChecker()
 	mockOrgSearcher := mock.NewMockOrganizationSearcher()
 	mockAuth := mock.NewMockAuthService()
-	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mockOrgSearcher, mockAuth)
+	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mock.NewMockResourceFilter(), mockOrgSearcher, mockAuth)
 	svc := service.(*querySvcsrvc)
 
 	tests := []struct {
@@ -290,7 +290,7 @@ func TestPayloadToOrganizationCriteria(t *testing.T) {
 	mockAccessChecker := mock.NewMockAccessControlChecker()
 	mockOrgSearcher := mock.NewMockOrganizationSearcher()
 	mockAuth := mock.NewMockAuthService()
-	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mockOrgSearcher, mockAuth)
+	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mock.NewMockResourceFilter(), mockOrgSearcher, mockAuth)
 	svc := service.(*querySvcsrvc)
 
 	tests := []struct {
@@ -357,7 +357,7 @@ func TestDomainOrganizationToResponse(t *testing.T) {
 	mockAccessChecker := mock.NewMockAccessControlChecker()
 	mockOrgSearcher := mock.NewMockOrganizationSearcher()
 	mockAuth := mock.NewMockAuthService()
-	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mockOrgSearcher, mockAuth)
+	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mock.NewMockResourceFilter(), mockOrgSearcher, mockAuth)
 	svc := service.(*querySvcsrvc)
 
 	tests := []struct {
@@ -437,7 +437,7 @@ func TestPayloadToOrganizationSuggestionCriteria(t *testing.T) {
 	mockAccessChecker := mock.NewMockAccessControlChecker()
 	mockOrgSearcher := mock.NewMockOrganizationSearcher()
 	mockAuth := mock.NewMockAuthService()
-	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mockOrgSearcher, mockAuth)
+	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mock.NewMockResourceFilter(), mockOrgSearcher, mockAuth)
 	svc := service.(*querySvcsrvc)
 
 	tests := []struct {
@@ -493,7 +493,7 @@ func TestDomainOrganizationSuggestionsToResponse(t *testing.T) {
 	mockAccessChecker := mock.NewMockAccessControlChecker()
 	mockOrgSearcher := mock.NewMockOrganizationSearcher()
 	mockAuth := mock.NewMockAuthService()
-	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mockOrgSearcher, mockAuth)
+	service := NewQuerySvc(mockResourceSearcher, mockAccessChecker, mock.NewMockResourceFilter(), mockOrgSearcher, mockAuth)
 	svc := service.(*querySvcsrvc)
 
 	tests := []struct {

cmd/service/providers.go

@@ -14,6 +14,7 @@ import (
 	"github.com/linuxfoundation/lfx-v2-query-service/internal/domain/port"
 	"github.com/linuxfoundation/lfx-v2-query-service/internal/infrastructure/auth"
 	"github.com/linuxfoundation/lfx-v2-query-service/internal/infrastructure/clearbit"
+	"github.com/linuxfoundation/lfx-v2-query-service/internal/infrastructure/filter"
 	"github.com/linuxfoundation/lfx-v2-query-service/internal/infrastructure/mock"
 	"github.com/linuxfoundation/lfx-v2-query-service/internal/infrastructure/nats"
 	"github.com/linuxfoundation/lfx-v2-query-service/internal/infrastructure/opensearch"
@@ -242,3 +243,15 @@ func OrganizationSearcherImpl(ctx context.Context) port.OrganizationSearcher {
 
 	return organizationSearcher
 }
+
+// ResourceFilterImpl injects the resource filter implementation
+func ResourceFilterImpl(ctx context.Context) port.ResourceFilter {
+	slog.InfoContext(ctx, "initializing CEL resource filter")
+
+	celFilter, err := filter.NewCELFilter()
+	if err != nil {
+		log.Fatalf("failed to initialize CEL filter: %v", err)
+	}
+
+	return celFilter
+}

cmd/service/service.go

@@ -160,10 +160,11 @@ func (s *querySvcsrvc) Livez(ctx context.Context) (res []byte, err error) {
 // NewQuerySvc returns the query-svc service implementation.
 func NewQuerySvc(resourceSearcher port.ResourceSearcher,
 	accessControlChecker port.AccessControlChecker,
+	resourceFilter port.ResourceFilter,
 	organizationSearcher port.OrganizationSearcher,
 	auth port.Authenticator,
 ) querysvc.Service {
-	resourceService := service.NewResourceSearch(resourceSearcher, accessControlChecker)
+	resourceService := service.NewResourceSearch(resourceSearcher, accessControlChecker, resourceFilter)
 	organizationService := service.NewOrganizationSearch(organizationSearcher)
 	return &querySvcsrvc{
 		resourceService:     resourceService,