Fix awsservicemap download, added more logging to all commands/sdk calls

aws/api-gws.go

@@ -30,6 +30,7 @@ type ApiGwModule struct {
 	Goroutines int
 	AWSProfile string
 	WrapTable  bool
+	ServiceMap *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 
 	// Main module data
 	Gateways       []ApiGateway
@@ -178,10 +179,14 @@ func (m *ApiGwModule) executeChecks(r string, wg *sync.WaitGroup, semaphore chan
 	// 	<-semaphore
 	// }()
 
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
-	res, err := servicemap.IsServiceInRegion("apigateway", r)
+	res, err := servicemap.IsServiceInRegion("api-gateway", r)
 	if err != nil {
 		m.modLog.Error(err)
 	}

aws/api-gws_test.go

@@ -10,11 +10,17 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/aws/smithy-go/ptr"
+	"github.com/bishopfox/awsservicemap"
 	"github.com/spf13/afero"
 )
 
 func TestApiGw(t *testing.T) {
 
+	// Create a service map that downloads from AWS
+	serviceMap := &awsservicemap.AwsServiceMap{
+		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	}
+
 	m := ApiGwModule{
 		AWSProfile: "unittesting",
 		AWSRegions: []string{"us-east-1"},
@@ -26,6 +32,7 @@ func TestApiGw(t *testing.T) {
 		WrapTable:          false,
 		APIGatewayClient:   &sdk.MockedAWSAPIGatewayClient{},
 		APIGatewayv2Client: &sdk.MockedAWSAPIGatewayv2Client{},
+		ServiceMap:         serviceMap,
 	}
 
 	fs := internal.MockFileSystem(true)

aws/client-initializers.go

@@ -17,6 +17,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/sns"
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
+	"github.com/sirupsen/logrus"
 )
 
 func InitIamCommandClient(iamSimPPClient sdk.AWSIAMClientInterface, caller sts.GetCallerIdentityOutput, AWSProfile string, Goroutines int) IamSimulatorModule {
@@ -26,6 +27,7 @@ func InitIamCommandClient(iamSimPPClient sdk.AWSIAMClientInterface, caller sts.G
 		Caller:             caller,
 		AWSProfileProvided: AWSProfile,
 		Goroutines:         Goroutines,
+		modLog:             internal.TxtLog.WithFields(logrus.Fields{"module": "iam-simulator"}),
 	}
 
 	return iamSimMod

aws/cloudformation.go

@@ -28,6 +28,7 @@ type CloudformationModule struct {
 	WrapTable     bool
 	AWSOutputType string
 	AWSTableCols  string
+	ServiceMap    *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 
 	// Main module data
 	CFStacks       []CFStack
@@ -192,8 +193,12 @@ func (m *CloudformationModule) PrintCloudformationStacks(outputDirectory string,
 
 func (m *CloudformationModule) executeChecks(r string, wg *sync.WaitGroup, semaphore chan struct{}, dataReceiver chan CFStack) {
 	defer wg.Done()
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	serviceRegions, err := servicemap.GetRegionsForService("cloudformation")
 	if err != nil {

aws/codebuild.go

@@ -29,6 +29,7 @@ type CodeBuildModule struct {
 	AWSProfile     string
 	SkipAdminCheck bool
 	WrapTable      bool
+	ServiceMap     *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 	pmapperMod     PmapperModule
 	pmapperError   error
 	iamSimClient   IamSimulatorModule
@@ -211,8 +212,12 @@ func (m *CodeBuildModule) PrintCodeBuildProjects(outputDirectory string, verbosi
 func (m *CodeBuildModule) executeChecks(r string, wg *sync.WaitGroup, semaphore chan struct{}, dataReceiver chan Project) {
 	defer wg.Done()
 
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	res, err := servicemap.IsServiceInRegion("codebuild", r)
 	if err != nil {

aws/databases.go

@@ -30,6 +30,7 @@ type DatabasesModule struct {
 	Goroutines int
 	AWSProfile string
 	WrapTable  bool
+	ServiceMap *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 
 	Databases      []Database
 	CommandCounter internal.CommandCounter
@@ -231,8 +232,12 @@ func (m *DatabasesModule) Receiver(receiver chan Database, receiverDone chan boo
 func (m *DatabasesModule) executeChecks(r string, wg *sync.WaitGroup, semaphore chan struct{}, dataReceiver chan Database) {
 	defer wg.Done()
 
-	serviceMap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	serviceMap := m.ServiceMap
+	if serviceMap == nil {
+		serviceMap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	m.executeRdsCheck(r, wg, semaphore, dataReceiver, serviceMap) // Also returns Neptune and DocDB
 	m.executeRedshiftCheck(r, wg, semaphore, dataReceiver, serviceMap)

aws/directory-services.go

@@ -31,7 +31,8 @@ type DirectoryModule struct {
 	AWSProfileProvided string
 	AWSProfileStub     string
 	CloudFoxVersion    string
-
+	ServiceMap         *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
+
 	Directories        []Directory
 	CommandCounter     internal.CommandCounter
 	output             internal.OutputData2
@@ -190,8 +191,12 @@ func (m *DirectoryModule) PrintDirectories(outputDirectory string, verbosity int
 func (m *DirectoryModule) executeChecks(r string, wg *sync.WaitGroup, semaphore chan struct{}, dataReceiver chan Directory) {
 	defer wg.Done()
 
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	res, err := servicemap.IsServiceInRegion("clouddirectory", r)
 	if err != nil {

aws/ecr.go

@@ -30,6 +30,7 @@ type ECRModule struct {
 	Goroutines int
 	AWSProfile string
 	WrapTable  bool
+	ServiceMap *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 
 	// Main module data
 	Repositories   []Repository
@@ -197,8 +198,12 @@ func (m *ECRModule) PrintECR(outputDirectory string, verbosity int) {
 func (m *ECRModule) executeChecks(r string, wg *sync.WaitGroup, semaphore chan struct{}, dataReceiver chan Repository) {
 	defer wg.Done()
 
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	res, err := servicemap.IsServiceInRegion("ecr", r)
 	if err != nil {

aws/ecs-tasks.go

@@ -35,6 +35,7 @@ type ECSTasksModule struct {
 	Goroutines     int
 	SkipAdminCheck bool
 	WrapTable      bool
+	ServiceMap     *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 	pmapperMod     PmapperModule
 	pmapperError   error
 	iamSimClient   IamSimulatorModule
@@ -307,8 +308,12 @@ func (m *ECSTasksModule) writeLoot(outputDirectory string) {
 func (m *ECSTasksModule) executeChecks(r string, wg *sync.WaitGroup, dataReceiver chan MappedECSTask) {
 	defer wg.Done()
 
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	res, err := servicemap.IsServiceInRegion("ecs", r)
 	if err != nil {

aws/eks.go

@@ -32,6 +32,7 @@ type EKSModule struct {
 	AWSProfile     string
 	SkipAdminCheck bool
 	WrapTable      bool
+	ServiceMap     *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 	pmapperMod     PmapperModule
 	pmapperError   error
 	iamSimClient   IamSimulatorModule
@@ -242,8 +243,12 @@ func (m *EKSModule) EKS(outputDirectory string, verbosity int) {
 func (m *EKSModule) executeChecks(r string, wg *sync.WaitGroup, semaphore chan struct{}, dataReceiver chan Cluster) {
 	defer wg.Done()
 
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	res, err := servicemap.IsServiceInRegion("eks", r)
 	if err != nil {

aws/elastic-network-interfaces.go

@@ -27,6 +27,7 @@ type ElasticNetworkInterfacesModule struct {
 
 	AWSProfile string
 	WrapTable  bool
+	ServiceMap *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 
 	MappedENIs     []MappedENI
 	CommandCounter internal.CommandCounter
@@ -233,8 +234,12 @@ func (m *ElasticNetworkInterfacesModule) writeLoot(outputDirectory string) {
 
 func (m *ElasticNetworkInterfacesModule) executeChecks(r string, wg *sync.WaitGroup, dataReceiver chan MappedENI) {
 	defer wg.Done()
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	res, err := servicemap.IsServiceInRegion("ec2", r)
 	if err != nil {

aws/endpoints.go

@@ -63,6 +63,7 @@ type EndpointsModule struct {
 	Goroutines int
 	AWSProfile string
 	WrapTable  bool
+	ServiceMap *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 
 	// Main module data
 	Endpoints      []Endpoint
@@ -267,8 +268,12 @@ func (m *EndpointsModule) executeChecks(r string, wg *sync.WaitGroup, semaphore
 	// 	<-semaphore
 	// }()
 
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	res, err := servicemap.IsServiceInRegion("lambda", r)
 	if err != nil {
@@ -328,7 +333,7 @@ func (m *EndpointsModule) executeChecks(r string, wg *sync.WaitGroup, semaphore
 		wg.Add(1)
 		go m.getELBListenersPerRegion(r, wg, semaphore, dataReceiver)
 	}
-	res, err = servicemap.IsServiceInRegion("apigateway", r)
+	res, err = servicemap.IsServiceInRegion("api-gateway", r)
 	if err != nil {
 		m.modLog.Error(err)
 	}

aws/env-vars.go

@@ -43,6 +43,7 @@ type EnvsModule struct {
 
 	Goroutines int
 	WrapTable  bool
+	ServiceMap *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 
 	// Service Clients
 	ECSClient       *ecs.Client
@@ -280,8 +281,12 @@ func (m *EnvsModule) Receiver(receiver chan EnvironmentVariable, receiverDone ch
 
 func (m *EnvsModule) executeChecks(r string, wg *sync.WaitGroup, semaphore chan struct{}, dataReceiver chan EnvironmentVariable) {
 	defer wg.Done()
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	res, _ := servicemap.IsServiceInRegion("ecs", r)
 	if res {

aws/filesystems.go

@@ -36,6 +36,7 @@ type FilesystemsModule struct {
 	Goroutines int
 	AWSProfile string
 	WrapTable  bool
+	ServiceMap *awsservicemap.AwsServiceMap // Shared service map to avoid repeated HTTP requests
 
 	// Main module data
 	Filesystems []FilesystemObject
@@ -218,8 +219,12 @@ func (m *FilesystemsModule) Receiver(receiver chan FilesystemObject, receiverDon
 
 func (m *FilesystemsModule) executeChecks(r string, wg *sync.WaitGroup, semaphore chan struct{}, dataReceiver chan FilesystemObject) {
 	defer wg.Done()
-	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	// Use shared ServiceMap instance if provided, otherwise create a new one
+	servicemap := m.ServiceMap
+	if servicemap == nil {
+		servicemap = &awsservicemap.AwsServiceMap{
+			JsonFileSource: "DOWNLOAD_FROM_AWS",
+		}
 	}
 	res, err := servicemap.IsServiceInRegion("efs", r)
 	if err != nil {

aws/iam-simulator.go

@@ -48,26 +48,23 @@ type SimulatorResult struct {
 	Decision   string
 }
 
-var (
-	defaultActionNames = []string{
-		"sts:AssumeRole",
-		"iam:PassRole",
-		"secretsmanager:GetSecretValue",
-		"ssm:GetParameter",
-		"s3:ListBucket",
-		"s3:GetObject",
-		"ssm:SendCommand",
-		"ssm:StartSession",
-		"ecr:BatchGetImage",
-		"ecr:GetAuthorizationToken",
-		"eks:UpdateClusterConfig",
-		"lambda:ListFunctions",
-		"ec2:DescribeInstanceAttributeInput",
-		"sns:Subscribe",
-		"sqs:SendMessage",
-	}
-	TxtLogger = internal.TxtLogger()
-)
+var defaultActionNames = []string{
+	"sts:AssumeRole",
+	"iam:PassRole",
+	"secretsmanager:GetSecretValue",
+	"ssm:GetParameter",
+	"s3:ListBucket",
+	"s3:GetObject",
+	"ssm:SendCommand",
+	"ssm:StartSession",
+	"ecr:BatchGetImage",
+	"ecr:GetAuthorizationToken",
+	"eks:UpdateClusterConfig",
+	"lambda:ListFunctions",
+	"ec2:DescribeInstanceAttributeInput",
+	"sns:Subscribe",
+	"sqs:SendMessage",
+}
 
 func (m *IamSimulatorModule) PrintIamSimulator(principal string, action string, resource string, outputDirectory string, verbosity int) {
 
@@ -466,11 +463,9 @@ func (m *IamSimulatorModule) isPrincipalAnAdmin(principal *string) bool {
 			},
 		)
 		if err != nil {
-			//m.modLog.Error(err.Error())
-			TxtLogger.Println(err.Error())
+			m.modLog.Error(err)
 			m.CommandCounter.Error++
-			//m.modLog.Error(fmt.Sprintf("Failed admin check on %s\n\n", aws.ToString(principal)))
-			TxtLogger.Printf("Failed admin check on %s\n\n", aws.ToString(principal))
+			m.modLog.Errorf("Failed admin check on %s", aws.ToString(principal))
 			return false
 		}