feat: add github/openssf scrapers

api/context.go

@@ -161,3 +161,7 @@ func (ctx ScrapeContext) Namespace() string {
 func (ctx ScrapeContext) IsTrace() bool {
 	return ctx.scrapeConfig.Spec.IsTrace()
 }
+
+func (ctx ScrapeContext) IsDebug() bool {
+	return ctx.scrapeConfig.Spec.IsDebug()
+}

api/v1/common.go

@@ -6,11 +6,14 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/flanksource/clicky"
+	"github.com/flanksource/clicky/api"
 	"github.com/flanksource/duty"
 	"github.com/flanksource/duty/connection"
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/types"
 	"github.com/flanksource/gomplate/v3"
+	"github.com/samber/lo"
 )
 
 // List of types which should not have scraper_id
@@ -63,6 +66,34 @@ func (s Script) String() string {
 	return ""
 }
 
+func (s Script) PrettyShort() api.Text {
+	t := clicky.Text("")
+	if s.GoTemplate != "" {
+		t = t.Append("go: ", "text-green-600").Append(clicky.CodeBlock("go", lo.Ellipsis(s.GoTemplate, 200)))
+	} else if s.JSONPath != "" {
+		t = t.Append("jsonpath: ", "text-blue-600").Append(clicky.CodeBlock("jsonpath", lo.Ellipsis(s.JSONPath, 200)))
+	} else if s.Expression != "" {
+		t = t.Append("expr: ", "text-yellow-600").Append(clicky.CodeBlock("cel", lo.Ellipsis(s.Expression, 200)))
+	} else if s.Javascript != "" {
+		t = t.Append("js: ", "text-purple-600").Append(clicky.CodeBlock("javascript", lo.Ellipsis(s.Javascript, 200)))
+	}
+	return t
+}
+
+func (s Script) Pretty() api.Text {
+	t := clicky.Text("")
+	if s.GoTemplate != "" {
+		t = t.Append("go: ", "text-green-600").Append(clicky.CodeBlock("go", s.GoTemplate))
+	} else if s.JSONPath != "" {
+		t = t.Append("jsonpath: ", "text-blue-600").Append(clicky.CodeBlock("jsonpath", s.JSONPath))
+	} else if s.Expression != "" {
+		t = t.Append("expr: ", "text-yellow-600").Append(clicky.CodeBlock("cel", s.Expression))
+	} else if s.Javascript != "" {
+		t = t.Append("js: ", "text-purple-600").Append(clicky.CodeBlock("javascript", s.Javascript))
+	}
+	return t
+}
+
 type Mask struct {
 	// Selector is a CEL expression that selects on what config items to apply the mask.
 	Selector string `json:"selector,omitempty"`
@@ -110,6 +141,13 @@ type ChangeMapping struct {
 	Action ChangeAction `json:"action,omitempty"`
 	// Summary replaces the existing change summary.
 	Summary string `json:"summary,omitempty"`
+	// ConfigID is a CEL expression that returns the target config's external ID
+	// for redirecting changes to a different config item.
+	ConfigID string `json:"config_id,omitempty"`
+	// ConfigType is the target config type for redirecting changes.
+	ConfigType string `json:"config_type,omitempty"`
+	// ScraperID is the scraper ID for the target config. Use "all" for cross-scraper lookups.
+	ScraperID string `json:"scraper_id,omitempty"`
 }
 
 type TransformChange struct {
@@ -389,6 +427,15 @@ func (c Connection) GetEndpoint() string {
 	return sanitizeEndpoints(c.Connection)
 }
 
+func (c Connection) Pretty() api.Text {
+	t := clicky.Text("")
+	if c.Connection != "" {
+		clicky.RedactSecretValues()
+		t = t.Append(sanitizeEndpoints(c.Connection))
+	}
+	return t
+}
+
 // Obfuscate passwords of the form ' password=xxxxx ' from connectionString since
 // connectionStrings are used as metric labels and we don't want to leak passwords
 // Returns the Connection string with the password replaced by '###'

api/v1/github_security.go

@@ -0,0 +1,53 @@
+package v1
+
+import (
+	"time"
+
+	"github.com/flanksource/duty/types"
+)
+
+// GitHubSecurity scraper fetches security alerts from GitHub repositories
+// including Dependabot alerts, code scanning alerts, secret scanning alerts,
+// and security advisories.
+type GitHubSecurity struct {
+	BaseScraper `json:",inline" yaml:",inline"`
+
+	// Repositories is the list of repositories to scan
+	Repositories []GitHubSecurityRepository `yaml:"repositories" json:"repositories"`
+
+	// PersonalAccessToken for GitHub API authentication
+	// Required scopes: repo (full) or security_events (read)
+	PersonalAccessToken types.EnvVar `yaml:"personalAccessToken,omitempty" json:"personalAccessToken,omitempty"`
+
+	// ConnectionName, if provided, will be used to populate personalAccessToken
+	ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
+
+	// Filters for security alerts
+	Filters GitHubSecurityFilters `yaml:"filters,omitempty" json:"filters,omitempty"`
+}
+
+// GitHubSecurityRepository specifies a repository to scan
+type GitHubSecurityRepository struct {
+	Owner string `yaml:"owner" json:"owner"`
+	Repo  string `yaml:"repo" json:"repo"`
+}
+
+// GitHubSecurityFilters defines filtering options for security alerts
+type GitHubSecurityFilters struct {
+	// Severity filters: critical, high, medium, low
+	Severity []string `yaml:"severity,omitempty" json:"severity,omitempty"`
+
+	// State filters: open, closed, dismissed, fixed
+	State []string `yaml:"state,omitempty" json:"state,omitempty"`
+
+	// MaxAge filters alerts by age (e.g., "90d", "30d")
+	MaxAge string `yaml:"maxAge,omitempty" json:"maxAge,omitempty"`
+}
+
+// ParseMaxAge converts the MaxAge string to a time.Duration
+func (f GitHubSecurityFilters) ParseMaxAge() (time.Duration, error) {
+	if f.MaxAge == "" {
+		return 0, nil
+	}
+	return time.ParseDuration(f.MaxAge)
+}

api/v1/interface.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/flanksource/clicky"
+	"github.com/flanksource/clicky/api"
 	"github.com/flanksource/commons/collections/set"
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty"
@@ -741,6 +743,110 @@ type ScrapeResult struct {
 	GCPStructPB *structpb.Struct `json:"-"`
 }
 
+func (s ScrapeResult) Debug() api.Text {
+	t := clicky.Text("")
+
+	t = t.Append("ID: ", "text-muted").Append(s.ID)
+	t = t.Append(" Name: ", "text-muted").Append(s.Name)
+	t = t.Append(" Type: ", "text-muted").Append(s.Type)
+	t = t.Append(" Status: ", "text-muted").Append(s.Status)
+	if s.Health != "" && s.Health != models.HealthUnknown {
+		t = t.Append(" Health: ", "text-muted").Append(string(s.Health))
+	}
+	if s.Source != "" {
+		t = t.NewLine().Append("Source: ", "text-muted").Append(s.Source)
+	}
+	if s.ScraperLess {
+		t = t.NewLine().Append("Scraper Less: ", "text-muted").Append("true")
+	}
+	if s.Icon != "" {
+		t = t.NewLine().Append("Icon: ", "text-muted").Append(s.Icon)
+	}
+
+	if s.Error != nil {
+		t = t.Append(" Error: ", "text-red-500").Append(s.Error.Error())
+	}
+	if s.Aliases != nil && len(s.Aliases) > 0 {
+		t = t.Append(" Aliases: ", "text-muted").Append(strings.Join(s.Aliases, ", "))
+	}
+	if s.Description != "" {
+		t = t.NewLine().Append("Description: ", "text-muted").Append(s.Description)
+	}
+
+	if s.Locations != nil && len(s.Locations) > 0 {
+		t = t.NewLine().Append("Locations: ", "text-muted").Append(strings.Join(s.Locations, ", "))
+	}
+
+	if s.Labels != nil && len(s.Labels) > 0 {
+		t = t.NewLine().Append("Labels: ", "text-muted").Append(clicky.Map(s.Labels))
+	}
+	if s.Tags != nil && len(s.Tags) > 0 {
+		t = t.NewLine().Append("Tags: ", "text-muted").Append(clicky.Map(s.Tags.AsMap()))
+	}
+	if s.Properties != nil && len(s.Properties) > 0 {
+		t = t.NewLine().Append("Properties: ", "text-muted").Append(clicky.Map(s.Properties.AsMap()))
+	}
+
+	if s.Changes != nil && len(s.Changes) > 0 {
+		t = t.NewLine().Append("Changes: ", "text-muted")
+		for _, change := range s.Changes {
+			t = t.NewLine().Append(fmt.Sprintf(" - %s: %s", change.ChangeType, change.Summary))
+		}
+	}
+
+	switch v := s.Config.(type) {
+	case string:
+		if s.Format == "json" || s.Format == "" {
+			t = t.NewLine().Append(clicky.CodeBlock("json", v)).NewLine()
+		} else {
+			t = t.NewLine().Append(clicky.CodeBlock(s.Format, v)).NewLine()
+		}
+	case map[string]any:
+		t = t.NewLine().Append(clicky.Map(v), "max-w-[100ch]").NewLine()
+	case map[string]string:
+		t = t.NewLine().Append(clicky.Map(v), "max-w-[100ch]").NewLine()
+	default:
+		t = t.NewLine().Append(fmt.Sprintf("%v", v)).NewLine()
+	}
+
+	return t
+}
+
+func (s ScrapeResult) PrettyRow(options interface{}) map[string]api.Text {
+	row := make(map[string]api.Text)
+	row["ID"] = clicky.Text(s.ID)
+	row["Name"] = clicky.Text(s.Name)
+	row["Type"] = clicky.Text(s.Type)
+	row["Health"] = clicky.Text(string(s.Health))
+	row["Status"] = clicky.Text(s.Status)
+	if s.Error != nil {
+		row["Error"] = clicky.Text(s.Error.Error())
+	} else {
+		row["Error"] = clicky.Text("")
+	}
+	clicky.MustPrint([]map[string]api.Text{row}, clicky.FormatOptions{JSON: true})
+	clicky.MustPrint([]map[string]api.Text{row}, clicky.FormatOptions{Pretty: true})
+
+	return row
+}
+
+func (s ScrapeResult) IsMetadataOnly() bool {
+	return s.Config == nil
+}
+
+func (s ScrapeResult) Pretty() api.Text {
+	t := clicky.Text("")
+
+	t = t.Append("ID: ", "text-muted").Append(s.ID)
+	t = t.Append(" Name: ", "text-muted").Append(s.Name)
+	t = t.Append(" Type: ", "text-muted").Append(s.Type)
+	if s.Tags != nil && len(s.Tags) > 0 {
+		t = t.NewLine().Append("Tags: ", "text-muted").Append(clicky.Map(s.Tags.AsMap()))
+	}
+
+	return t
+}
+
 // +kubebuilder:object:generate=false
 type ExternalConfigAccessLog struct {
 	models.ConfigAccessLog
@@ -885,6 +991,58 @@ func (s ScrapeResult) Clone(config interface{}) ScrapeResult {
 	return clone
 }
 
+// +kubebuilder:object:generate=false
+type FullScrapeResults struct {
+	Configs            []api.PrettyRow             `json:"configs,omitempty"`
+	Analysis           []models.ConfigAnalysis     `json:"analysis,omitempty"`
+	Changes            []models.ConfigChange       `json:"changes,omitempty"`
+	Relationships      []models.ConfigRelationship `json:"relationships,omitempty"`
+	ExternalRoles      []models.ExternalRole       `json:"external_roles,omitempty"`
+	ExternalUsers      []models.ExternalUser       `json:"external_users,omitempty"`
+	ExternalGroups     []models.ExternalGroup      `json:"external_groups,omitempty"`
+	ExternalUserGroups []models.ExternalUserGroup  `json:"external_user_groups,omitempty"`
+	ConfigAccess       []ExternalConfigAccess      `json:"config_access,omitempty"`
+	ConfigAccessLogs   []ExternalConfigAccessLog   `json:"config_access_logs,omitempty"`
+}
+
+func MergeScrapeResults(results ...ScrapeResults) FullScrapeResults {
+	full := FullScrapeResults{}
+	for _, res := range results {
+		for _, r := range res {
+			if r.Error != nil {
+				continue
+			}
+
+			if r.AnalysisResult != nil {
+				full.Analysis = append(full.Analysis, r.AnalysisResult.ToConfigAnalysis())
+			}
+
+			for _, change := range r.Changes {
+				configChange := models.ConfigChange{
+					ChangeType: change.ChangeType,
+					Severity:   models.Severity(change.Severity),
+					Source:     change.Source,
+					Summary:    change.Summary,
+					CreatedAt:  change.CreatedAt,
+				}
+				full.Changes = append(full.Changes, configChange)
+			}
+
+			if !r.IsMetadataOnly() {
+				full.Configs = append(full.Configs, r)
+			}
+
+			full.ExternalRoles = append(full.ExternalRoles, r.ExternalRoles...)
+			full.ExternalUsers = append(full.ExternalUsers, r.ExternalUsers...)
+			full.ExternalGroups = append(full.ExternalGroups, r.ExternalGroups...)
+			full.ExternalUserGroups = append(full.ExternalUserGroups, r.ExternalUserGroups...)
+			full.ConfigAccess = append(full.ConfigAccess, r.ConfigAccess...)
+			full.ConfigAccessLogs = append(full.ConfigAccessLogs, r.ConfigAccessLogs...)
+		}
+	}
+	return full
+}
+
 func Ellipses(str string, length int) string {
 	if len(str) > length {
 		if len(str) < 3 || length < 3 {

api/v1/openssf.go

@@ -0,0 +1,18 @@
+package v1
+
+// OpenSSFScorecard scraper fetches security scorecard data from the OpenSSF Scorecard API
+type OpenSSFScorecard struct {
+	BaseScraper `json:",inline" yaml:",inline"`
+
+	// Repositories is the list of repositories to assess
+	Repositories []OpenSSFRepository `yaml:"repositories" json:"repositories"`
+
+	// MinScore optionally filters repositories by minimum score (0-10)
+	MinScore *float64 `yaml:"minScore,omitempty" json:"minScore,omitempty"`
+}
+
+// OpenSSFRepository specifies a repository to assess
+type OpenSSFRepository struct {
+	Owner string `yaml:"owner" json:"owner"`
+	Repo  string `yaml:"repo" json:"repo"`
+}

api/v1/scrapeconfig_types.go

@@ -67,6 +67,14 @@ func (t ScrapeConfig) GetContext() map[string]any {
 	}
 }
 
+func (t ScrapeConfig) IsDebug() bool {
+	return strings.EqualFold(t.Annotations["debug"], "true")
+}
+
+func (t ScrapeConfig) IsTrace() bool {
+	return strings.EqualFold(t.Annotations["trace"], "true")
+}
+
 func (t ScrapeConfig) NamespaceScope() string {
 	return t.Namespace
 }
@@ -93,6 +101,12 @@ func (t *ScrapeConfig) Type() string {
 	if len(t.Spec.GithubActions) != 0 {
 		return "githubactions"
 	}
+	if len(t.Spec.GitHubSecurity) != 0 {
+		return "githubsecurity"
+	}
+	if len(t.Spec.OpenSSFScorecard) != 0 {
+		return "openssf"
+	}
 	if len(t.Spec.Azure) != 0 {
 		return "azure"
 	}

api/v1/types.go

@@ -68,8 +68,10 @@ type ScraperSpec struct {
 	Kubernetes     []Kubernetes     `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty"`
 	KubernetesFile []KubernetesFile `json:"kubernetesFile,omitempty" yaml:"kubernetesFile,omitempty"`
 	AzureDevops    []AzureDevops    `json:"azureDevops,omitempty" yaml:"azureDevops,omitempty"`
-	GithubActions  []GitHubActions  `json:"githubActions,omitempty" yaml:"githubActions,omitempty"`
-	Azure          []Azure          `json:"azure,omitempty" yaml:"azure,omitempty"`
+	GithubActions   []GitHubActions   `json:"githubActions,omitempty" yaml:"githubActions,omitempty"`
+	GitHubSecurity  []GitHubSecurity  `json:"githubSecurity,omitempty" yaml:"githubSecurity,omitempty"`
+	OpenSSFScorecard []OpenSSFScorecard `json:"openssfScorecard,omitempty" yaml:"openssfScorecard,omitempty"`
+	Azure           []Azure           `json:"azure,omitempty" yaml:"azure,omitempty"`
 	SQL            []SQL            `json:"sql,omitempty" yaml:"sql,omitempty"`
 	Slack          []Slack          `json:"slack,omitempty" yaml:"slack,omitempty"`
 	Trivy          []Trivy          `json:"trivy,omitempty" yaml:"trivy,omitempty"`