Merge pull request #165 from Cloud-Architects/refactoring

Moved out AWS specific code to separate files

cloudiscovery/__init__.py

@@ -14,35 +14,26 @@
 limitations under the License.
 """
 
-import argparse
 import gettext
 import sys
 from os.path import dirname
 from typing import List
 
 import pkg_resources
 
+from provider.aws.command import aws_main
+from shared.parameters import generate_parser
+
 """path to pip package"""
 sys.path.append(dirname(__file__))
 
 # pylint: disable=wrong-import-position
-
-from provider.policy.command import Policy
-from provider.vpc.command import Vpc
-from provider.iot.command import Iot
-from provider.all.command import All
-from provider.limit.command import Limit
-from provider.security.command import Security
-
 from shared.common import (
     exit_critical,
     Filterable,
     parse_filters,
-    message_handler,
 )
-from shared.common_aws import aws_verbose, generate_session
 
-# pylint: enable=wrong-import-position
 # Check version
 if sys.version_info < (3, 8):
     print("Python 3.8 or newer is required", file=sys.stderr)
@@ -51,139 +42,6 @@
 __version__ = "2.3"
 
 AVAILABLE_LANGUAGES = ["en_US", "pt_BR"]
-DEFAULT_REGION = "us-east-1"
-DEFAULT_PARTITION_CODE = "aws"
-
-
-def str2bool(v):
-    if isinstance(v, bool):
-        return v
-    # pylint: disable=no-else-return
-    if v.lower() in ("yes", "true", "t", "y", "1"):
-        return True
-    elif v.lower() in ("no", "false", "f", "n", "0"):
-        return False
-    else:
-        raise argparse.ArgumentTypeError("Boolean value expected.")
-
-
-def generate_parser():
-    parser = argparse.ArgumentParser()
-
-    subparsers = parser.add_subparsers(help="commands", dest="command")
-
-    vpc_parser = subparsers.add_parser("aws-vpc", help="Analyze VPCs")
-    add_default_arguments(vpc_parser)
-    vpc_parser.add_argument(
-        "-v",
-        "--vpc-id",
-        required=False,
-        help="Inform VPC to analyze. If not informed, script will check all vpcs.",
-    )
-
-    iot_parser = subparsers.add_parser("aws-iot", help="Analyze IoTs")
-    add_default_arguments(iot_parser)
-    iot_parser.add_argument(
-        "-t",
-        "--thing-name",
-        required=False,
-        help="Inform Thing Name to analyze. If not informed, script will check all things inside a region.",
-    )
-
-    policy_parser = subparsers.add_parser("aws-policy", help="Analyze policies")
-    add_default_arguments(policy_parser, is_global=True)
-
-    all_parser = subparsers.add_parser("aws-all", help="Analyze all resources")
-    add_default_arguments(all_parser, diagram_enabled=False)
-    add_services_argument(all_parser)
-
-    limit_parser = subparsers.add_parser(
-        "aws-limit", help="Analyze aws limit resources."
-    )
-    add_default_arguments(limit_parser, diagram_enabled=False, filters_enabled=False)
-    add_services_argument(limit_parser)
-    limit_parser.add_argument(
-        "-t",
-        "--threshold",
-        required=False,
-        help="Select the %% of resource threshold between 0 and 100. \
-              For example: --threshold 50 will report all resources with more than 50%% threshold.",
-    )
-
-    security_parser = subparsers.add_parser(
-        "aws-security", help="Analyze aws several security checks."
-    )
-    add_default_arguments(security_parser, diagram_enabled=False, filters_enabled=False)
-    security_parser.add_argument(
-        "-c",
-        "--commands",
-        action="append",
-        required=False,
-        help='Select the security check command that you want to run. \
-              To see available commands, please type "-c list". \
-              If not passed, command will check all services.',
-    )
-
-    return parser
-
-
-def add_services_argument(limit_parser):
-    limit_parser.add_argument(
-        "-s",
-        "--services",
-        required=False,
-        help='Define services that you want to check, use "," (comma) to separate multiple names. \
-              If not passed, command will check all services.',
-    )
-
-
-def add_default_arguments(
-    parser, is_global=False, diagram_enabled=True, filters_enabled=True
-):
-    if not is_global:
-        parser.add_argument(
-            "-r",
-            "--region-name",
-            required=False,
-            help='Inform REGION NAME to analyze or "all" to check on all regions. \
-            If not informed, try to get from config file',
-        )
-    parser.add_argument(
-        "-p", "--profile-name", required=False, help="Profile to be used"
-    )
-    parser.add_argument(
-        "-l", "--language", required=False, help="Available languages: pt_BR, en_US"
-    )
-    parser.add_argument(
-        "--verbose",
-        "--verbose",
-        type=str2bool,
-        nargs="?",
-        const=True,
-        default=False,
-        help="Enable debug mode to sdk calls (default false)",
-    )
-    if filters_enabled:
-        parser.add_argument(
-            "-f",
-            "--filters",
-            action="append",
-            required=False,
-            help="filter resources (tags only for now, you must specify name and values); multiple filters "
-            "are possible to pass with -f <filter_1> -f <filter_2> approach, values can be separated by : sign; "
-            "example: Name=tags.costCenter;Value=20000:'20001:1'",
-        )
-    if diagram_enabled:
-        parser.add_argument(
-            "-d",
-            "--diagram",
-            type=str2bool,
-            nargs="?",
-            const=True,
-            default=True,
-            help="print diagram with resources (need Graphviz installed). Pass true/y[es] to "
-            "view image or false/n[o] not to generate image. Default true",
-        )
 
 
 # pylint: disable=too-many-branches,too-many-statements,too-many-locals
@@ -197,10 +55,6 @@ def main():
 
     args = parser.parse_args()
 
-    # Check if verbose mode is enabled
-    if args.verbose:
-        aws_verbose()
-
     if args.language is None or args.language not in AVAILABLE_LANGUAGES:
         language = "en_US"
     else:
@@ -228,83 +82,8 @@ def main():
         if args.filters is not None:
             filters = parse_filters(args.filters)
 
-    # aws profile check
-    if "region_name" not in args:
-        session = generate_session(profile_name=args.profile_name, region_name=None)
-    else:
-        session = generate_session(
-            profile_name=args.profile_name, region_name=args.region_name
-        )
-
-    session.get_credentials()
-    region_name = session.region_name
-
-    partition_code = get_partition(session, region_name)
-
-    if "region_name" not in args:
-        region_names = [DEFAULT_REGION]
-    else:
-        # checking region configuration
-        check_region_profile(
-            arg_region_name=args.region_name, profile_region_name=region_name
-        )
-
-        # assuming region parameter precedes region configuration
-        if args.region_name is not None:
-            region_name = args.region_name
-
-        # get regions
-        region_names = check_region(
-            region_parameter=args.region_name,
-            region_name=region_name,
-            session=session,
-            partition_code=partition_code,
-        )
-
-    if "threshold" in args:
-        if args.threshold is not None:
-            if args.threshold.isdigit() is False:
-                exit_critical(_("Threshold must be between 0 and 100"))
-            else:
-                if int(args.threshold) < 0 or int(args.threshold) > 100:
-                    exit_critical(_("Threshold must be between 0 and 100"))
-
-    if args.command == "aws-vpc":
-        command = Vpc(
-            vpc_id=args.vpc_id,
-            region_names=region_names,
-            session=session,
-            partition_code=partition_code,
-        )
-    elif args.command == "aws-policy":
-        command = Policy(
-            region_names=region_names, session=session, partition_code=partition_code
-        )
-    elif args.command == "aws-iot":
-        command = Iot(
-            thing_name=args.thing_name,
-            region_names=region_names,
-            session=session,
-            partition_code=partition_code,
-        )
-    elif args.command == "aws-all":
-        command = All(
-            region_names=region_names, session=session, partition_code=partition_code
-        )
-    elif args.command == "aws-limit":
-        command = Limit(
-            region_names=region_names,
-            session=session,
-            threshold=args.threshold,
-            partition_code=partition_code,
-        )
-    elif args.command == "aws-security":
-        command = Security(
-            region_names=region_names,
-            session=session,
-            commands=args.commands,
-            partition_code=partition_code,
-        )
+    if args.command.startswith("aws"):
+        command = aws_main(args)
     else:
         raise NotImplementedError("Unknown command")
 
@@ -316,28 +95,6 @@ def main():
     command.run(diagram, args.verbose, services, filters)
 
 
-def get_partition(session, region_name):
-    partition_code = DEFAULT_PARTITION_CODE  # assume it's always default partition, even if we can't find a region
-    partition_name = "AWS Standard"
-    # pylint: disable=protected-access
-    loader = session._session.get_component("data_loader")
-    endpoints = loader.load_data("endpoints")
-    for partition in endpoints["partitions"]:
-        for region, _ in partition["regions"].items():
-            if region == region_name:
-                partition_code = partition["partition"]
-                partition_name = partition["partitionName"]
-
-    if partition_code != DEFAULT_PARTITION_CODE:
-        message_handler(
-            "Found non-default partition: {} ({})".format(
-                partition_code, partition_name
-            ),
-            "HEADER",
-        )
-    return partition_code
-
-
 def check_diagram_version(diagram):
     if diagram:
         # Checking diagram version. Must be 0.13 or higher
@@ -348,37 +105,6 @@ def check_diagram_version(diagram):
             )
 
 
-def check_region_profile(arg_region_name, profile_region_name):
-    if arg_region_name is None and profile_region_name is None:
-        exit_critical("Neither region parameter nor region config were passed")
-
-
-def check_region(region_parameter, region_name, session, partition_code):
-    """
-    Region us-east-1 as a default region here, if not aws partition, just return asked region
-
-    This is just to list aws regions, doesn't matter default region
-    """
-    if partition_code != "aws":
-        return [region_name]
-
-    client = session.client("ec2", region_name=DEFAULT_REGION)
-
-    valid_region_names = [
-        region["RegionName"]
-        for region in client.describe_regions(AllRegions=True)["Regions"]
-    ]
-
-    if region_parameter != "all":
-        if region_name not in valid_region_names:
-            message = "There is no region named: {0}".format(region_name)
-            exit_critical(message)
-        else:
-            valid_region_names = [region_name]
-
-    return valid_region_names
-
-
 if __name__ == "__main__":
     try:
         main()

cloudiscovery/provider/all/command.py → cloudiscovery/provider/aws/all/command.py

@@ -1,7 +1,7 @@
 from typing import List
 
+from provider.aws.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner
 from shared.common import Filterable, BaseOptions
-from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner
 from shared.diagram import NoDiagram
 
 

cloudiscovery/provider/all/resource/all.py → cloudiscovery/provider/aws/all/resource/all.py

@@ -7,19 +7,19 @@
 from botocore.exceptions import UnknownServiceError
 from botocore.loaders import Loader
 
-from provider.all.command import AllOptions
-from provider.all.data.omitted_resources import OMITTED_RESOURCES
-from provider.all.data.on_top_policies import ON_TOP_POLICIES
-from provider.all.data.required_params_override import REQUIRED_PARAMS_OVERRIDE
-from provider.all.exception import all_exception
+from provider.aws.all.command import AllOptions
+from provider.aws.all.data.omitted_resources import OMITTED_RESOURCES
+from provider.aws.all.data.on_top_policies import ON_TOP_POLICIES
+from provider.aws.all.data.required_params_override import REQUIRED_PARAMS_OVERRIDE
+from provider.aws.all.exception import all_exception
+from provider.aws.common_aws import get_paginator, resource_tags
 from shared.common import (
     ResourceProvider,
     Resource,
     ResourceDigest,
     message_handler,
     ResourceAvailable,
 )
-from shared.common_aws import get_paginator, resource_tags
 
 SKIPPED_SERVICES = [
     "sagemaker"