Set up process #

export AWS_ACCESS_KEY_ID={Your AWS Access Key ID}
export AWS_SECRET_ACCESS_KEY={Your AWS secret access key}
export AWS_SESSION_TOKEN={Your AWS session token}

[default]
aws_access_key_id = YOUR_ACCESS_KEY_ID
aws_secret_access_key = YOUR_SECRET_ACCESS_KEY

[myprofile]
aws_access_key_id = YOUR_ACCESS_KEY_ID
aws_secret_access_key = YOUR_SECRET_ACCESS_KEY

export AWS_PROFILE=myprofile

accounts:
  id: <account_alias>
  local_profile: myprofile

aws sts get-session-token --serial-number <YOUR_MFA_SERIAL_NUMBER> --token-code <YOUR_MFA_TOKEN_CODE> --duration-seconds 3600

export AWS_ACCESS_KEY_ID=<YOUR_ACCESS_KEY_ID>
export AWS_SECRET_ACCESS_KEY=<YOUR_SECRET_ACCESS_KEY>
export AWS_SESSION_TOKEN=<YOUR_SESSION_TOKEN>

kind: source
spec:
  # Source spec section
  name: aws
  path: cloudquery/aws
  registry: cloudquery
  version: "v27.23.1"
  tables: ["aws_ec2_instances"]
  destinations: ["postgresql"]
  # Learn more about the configuration options at https://cql.ink/aws_source
  spec:
    # Optional parameters
    # regions: []
    # accounts: []
    # org: nil
    # concurrency: 50000
    # initialization_concurrency: 4
    # aws_debug: false
    # max_retries: 10
    # max_backoff: 30
    # custom_endpoint_url: ""
    # custom_endpoint_hostname_immutable: nil # required when custom_endpoint_url is set
    # custom_endpoint_partition_id: "" # required when custom_endpoint_url is set
    # custom_endpoint_signing_region: "" # required when custom_endpoint_url is set
    # use_paid_apis: false
    # table_options: nil
    # scheduler: shuffle # options are: dfs, round-robin or shuffle
    # use_nested_table_rate_limiting: false 
    # enable_api_level_tracing: false

kind: source
spec:
  name: aws
  path: cloudquery/aws
  registry: cloudquery
  version: "v27.23.1"
  tables: ['aws_s3_buckets']
  destinations: ["postgresql"]
  spec:
    aws_debug: false
    org:
      admin_account:
        local_profile: "<NAMED_PROFILE>"
      member_role_name: OrganizationAccountAccessRole
    regions:
      - '*'
    # Optional parameters
    # regions: []
    # accounts: []
    # org: nil
    # concurrency: 50000
    # initialization_concurrency: 4
    # aws_debug: false
    # max_retries: 10
    # max_backoff: 30
    # custom_endpoint_url: ""
    # custom_endpoint_hostname_immutable: nil # required when custom_endpoint_url is set
    # custom_endpoint_partition_id: "" # required when custom_endpoint_url is set
    # custom_endpoint_signing_region: "" # required when custom_endpoint_url is set
    # use_paid_apis: false
    # table_options: nil
    # scheduler: shuffle # options are: dfs, round-robin or shuffle

spec:
  regions: ["us-east-1","us-east-2"]
  skip_specific_apis:
    lambda:
      GetRuntimeManagementConfig: true
      GetFunction: true

lambda:
    GetRuntimeManagementConfig
    GetFunction
    GetFunctionCodeSigningConfig
    GetFunctionConcurrency
  kms:
    DescribeKey
    GetKeyRotationStatus
    ListResourceTags
  ssm:
    ListTagsForResource
  glacier:
    ListTagsForVault
  wafv2:
    ListResourcesForWebACL

kind: source
spec:
  # Source spec section
  name: aws
  path: cloudquery/aws
  registry: cloudquery
  version: "v27.23.1"
  tables: ["*"]
  skip_tables:
    - aws_cloudtrail_events
    - aws_docdb_cluster_parameter_groups
    - aws_docdb_engine_versions
    - aws_ec2_instance_types
    - aws_ec2_vpc_endpoint_services
    - aws_elasticache_engine_versions
    - aws_elasticache_parameter_groups
    - aws_elasticache_reserved_cache_nodes_offerings
    - aws_elasticache_service_updates
    - aws_iam_group_last_accessed_details
    - aws_iam_policy_last_accessed_details
    - aws_iam_role_last_accessed_details
    - aws_iam_user_last_accessed_details
    - aws_neptune_cluster_parameter_groups
    - aws_neptune_db_parameter_groups
    - aws_rds_cluster_parameter_groups
    - aws_rds_db_parameter_groups
    - aws_rds_engine_versions
    - aws_servicequotas_services
    - aws_stepfunctions_map_run_executions
    - aws_stepfunctions_map_runs
  destinations: ["postgresql"]
  spec:
    # AWS Spec section described below

AWSTemplateFormatVersion: 2010-09-09
Description: Configures Cloudtrail Events to be piped to a Kinesis Data stream via CloudWatch Logs.

Parameters:

  KinesisMessageDuration:
    Type: Number
    Description: Number of hours Kinesis will persist a record before it is purged.
    Default: 24
  ExistingS3BucketName:
    Type: String
    Description: Name of the S3 Bucket that CloudTrail will use to store logs.
    Default: ""

Conditions:
  CreateS3Bucket: !Equals [!Ref ExistingS3BucketName, ""]

Resources:
  # Stream that CQ will poll for changes
  CQSyncingKinesisStream:
    Type: AWS::Kinesis::Stream
    Properties:
      ShardCount: 1
      RetentionPeriodHours: !Ref KinesisMessageDuration

  # IAM Role for allowing CloudWatch Log to write to Kinesis Stream
  CloudWatchLogsToKinesisRole:
    Type: AWS::IAM::Role
    Properties:
      Policies:
      - PolicyName: CloudWatchLogsToKinesisPolicy
        PolicyDocument: 
          Version: 2012-10-17
          Statement:
            - Effect: Allow
              Action:
                - kinesis:PutRecord
              Resource: !GetAtt CQSyncingKinesisStream.Arn
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service: logs.amazonaws.com
            Action:
              - sts:AssumeRole


  CloudTrailS3Bucket:
    Type: AWS::S3::Bucket
    Condition: CreateS3Bucket
    Properties:
      LifecycleConfiguration:
        Rules:
          - ExpirationInDays: 30
            Status: Enabled

  CloudTrailS3BucketPolicy:
    Type: AWS::S3::BucketPolicy
    Properties:
      Bucket: !If [CreateS3Bucket,!Ref CloudTrailS3Bucket, !Ref ExistingS3BucketName]
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Sid: AWSCloudTrailAclCheck
            Effect: Allow
            Principal:
              Service: cloudtrail.amazonaws.com
            Action: s3:GetBucketAcl
            Resource: !Sub
                - arn:${AWS::Partition}:s3:::${Bucket}
                - { Bucket: !If [CreateS3Bucket,!Ref CloudTrailS3Bucket, !Ref ExistingS3BucketName] }
            Condition:
              StringEquals:
                'aws:SourceAccount': !Sub ${AWS::AccountId}
          - Sid: AWSCloudTrailWrite
            Effect: Allow
            Principal:
              Service: cloudtrail.amazonaws.com
            Action: s3:PutObject
            Resource: !Sub
                - arn:${AWS::Partition}:s3:::${Bucket}/*
                - { Bucket: !If [CreateS3Bucket,!Ref CloudTrailS3Bucket, !Ref ExistingS3BucketName] }
            Condition:
              StringEquals:
                's3:x-amz-acl': bucket-owner-full-control
                'aws:SourceAccount': !Sub ${AWS::AccountId}

  CloudWatchLogsGroup:
    Type: AWS::Logs::LogGroup
    UpdateReplacePolicy: Delete
    DeletionPolicy: Delete
    Properties:
      LogGroupName: "CloudTrailLogGroup"
      RetentionInDays: 1

  # Role for allowing CLoudTrail to write to CloudWatch Logs
  CloudWatchRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Sid: AssumeRole
          Effect: Allow
          Principal:
            Service: 'cloudtrail.amazonaws.com'
          Action: 'sts:AssumeRole'
      Policies:
      - PolicyName: 'cloudtrail-policy'
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action: 'logs:CreateLogStream'
            Resource: !GetAtt CloudWatchLogsGroup.Arn
          - Effect: Allow
            Action: 'logs:PutLogEvents'
            Resource: !GetAtt CloudWatchLogsGroup.Arn
  CloudTrailTrail:
    Type: AWS::CloudTrail::Trail
    DependsOn:
         - CloudTrailS3BucketPolicy
    Properties:
      CloudWatchLogsLogGroupArn: !GetAtt CloudWatchLogsGroup.Arn
      CloudWatchLogsRoleArn: !GetAtt CloudWatchRole.Arn
      EventSelectors:
        - IncludeManagementEvents: True
          ReadWriteType: WriteOnly
      IncludeGlobalServiceEvents: True
      IsLogging: True
      IsMultiRegionTrail: True
      S3BucketName: !If [CreateS3Bucket,!Ref CloudTrailS3Bucket, !Ref ExistingS3BucketName]

  SubscriptionFilter:
    Type: AWS::Logs::SubscriptionFilter
    Properties:
      LogGroupName: !Ref CloudWatchLogsGroup
      DestinationArn: !GetAtt CQSyncingKinesisStream.Arn
      RoleArn: !GetAtt CloudWatchLogsToKinesisRole.Arn
      FilterPattern: ""



Outputs:
  KinesisStreamArn:
    Description: The ARN of the Kinesis Data Stream that CloudQuery will use to listen for changes.
    Value: !GetAtt CQSyncingKinesisStream.Arn

kind: source
spec:
  name: aws
  path: cloudquery/aws
  registry: cloudquery
  version: "v27.23.1"
  tables: ['aws_s3_buckets']
  destinations: ["postgresql"]
  spec:
    aws_debug: false
    org:
      admin_account:
        local_profile: "<NAMED_PROFILE>"
      member_role_name: cloudquery-ro
    regions:
      - '*'

accounts:
  - account_name: <AccountName_1>
    role_arn: <YOUR_ROLE_ARN_1>
    # Optional. Local Profile is the named profile in your shared configuration file (usually `~/.aws/config`) that you want to use for this specific account
    local_profile: <NAMED_PROFILE>
    # Optional. Specify the Role Session name
    role_session_name: ""
  - account_name: <AccountName_2>
    local_profile: provider
    # Optional. Role ARN we want to assume when accessing this account
    role_arn: <YOUR_ROLE_ARN_2>

SELECT * FROM aws_elbv2_load_balancers WHERE scheme = 'internet-facing';

SELECT * FROM aws_rds_clusters WHERE storage_encrypted IS FALSE;

SELECT arn, region
FROM aws_s3_buckets
WHERE block_public_acls IS NOT TRUE
   OR block_public_policy IS NOT TRUE
   OR ignore_public_acls IS NOT TRUE
   OR restrict_public_buckets IS NOT TRUE

table_options:
  <table_name>:
    <api_method_name>:
      - <input_object>

table_options:
    aws_cloudtrail_events:
      lookup_events:
        - start_time: 2023-05-01T20:20:52Z
          end_time:   2023-05-03T20:20:52Z
          lookup_attributes:
            - attribute_key:   EventName
              attribute_value: RunInstances
        - start_time: 2023-05-01T20:20:52Z
          end_time:   2023-05-03T20:20:52Z
          lookup_attributes:
            - attribute_key:   EventName
              attribute_value: StartInstances
        - start_time: 2023-05-01T20:20:52Z
          end_time:   2023-05-03T20:20:52Z
          lookup_attributes:
            - attribute_key:   EventName
              attribute_value: StopInstances

table_options:
  aws_accessanalyzer_analyzer_findings:
    list_findings:
      - <AccessAnalyzer.ListFindings> # NextToken & AnalyzerArn are prohibited

  aws_accessanalyzer_analyzer_findings_v2:
    list_findings_v2:
      - <AccessAnalyzer.ListFindingsV2> # NextToken & AnalyzerArn are prohibited

  aws_cloudtrail_events:
    lookup_events:
      - <CloudTrail.LookupEvents> # NextToken is prohibited

  aws_cloudtrail_trails:
    describe_trails:
      - <CloudTrail.DescribeTrails>

  aws_cloudwatch_metrics:
    - list_metrics: <CloudWatch.ListMetrics> # NextToken is prohibited
      get_metric_statistics:
        - <CloudWatch.GetMetricStatistics>  # Namespace, MetricName and Dimensions are prohibited

  aws_costexplorer_cost_custom:
    get_cost_and_usage:
      - <CostExplorer.GetCostAndUsage> # NextPageToken is prohibited

  aws_ec2_images:
    describe_images:
      - <EC2.DescribeImages> # NextToken and ImageIds are prohibited. MaxResults should be in range [1-1000].

  aws_ec2_instances:
    describe_instances:
      - <EC2.DescribeInstances> # NextToken is prohibited. MaxResults should be in range [1-1000].

  aws_ec2_internet_gateways:
    describe_internet_gateways:
      - <EC2.DescribeInternetGateways> # NextToken is prohibited. MaxResults should be in range [5-1000].

  aws_ec2_network_interfaces:
    describe_network_interfaces:
      - <EC2.DescribeNetworkInterfaces> # NextToken is prohibited. MaxResults should be in range [5-1000].

  aws_ec2_route_tables:
    describe_route_tables:
      - <EC2.DescribeRouteTables> # NextToken is prohibited. MaxResults should be in range [5-100].

  aws_ec2_security_groups:
    describe_security_groups:
      - <EC2.DescribeSecurityGroups> # NextToken is prohibited. MaxResults should be in range [5-1000].

  aws_ec2_subnets:
    describe_subnets:
      - <EC2.DescribeSubnets> # NextToken is prohibited. MaxResults should be in range [5-1000].

  aws_ec2_vpcs:
    describe_vpcs:
      - <EC2.DescribeVpcs> # NextToken is prohibited. MaxResults should be in range [5-1000].

  aws_ecs_cluster_tasks:
    list_tasks:
      - <ECS.ListTasks> # Cluster and NextToken are prohibited. MaxResults should be in range [1-100].

  aws_guardduty_detectors:
    - list_detectors: <GuardDuty.ListDetectors> # NextToken is prohibited
      list_findings: <GuardDuty.ListFindings> # NextToken and DetectorID are prohibited

  aws_iam_groups:
    get_group:
      - <IAM.GetGroup> # Marker is prohibited. MaxItems should be in range [1-1000].

  aws_iam_policies:
    list_policies:
      - <IAM.ListPolicies> # Marker is prohibited. MaxItems should be in range [1-1000].

  aws_iam_roles:
    get_role:
      - <IAM.GetRole> # RoleName is required.

  aws_iam_users:
    get_user:
      - <IAM.GetUser> # UserName is required.

  aws_inspector_findings:
    list_findings:
      - <Inspector.ListFindings> # NextToken is prohibited. MaxResults should be in range [1-500].

  aws_inspector2_covered_resources:
    list_coverage:
      - <InspectorV2.ListCoverage> # NextToken is prohibited. MaxResults should be in range [1-200].

  aws_inspector2_findings:
    list_findings:
      - <InspectorV2.ListFindings> # NextToken is prohibited.

  aws_rds_clusters:
    describe_db_clusters:
      - <RDS.DescribeDBClusters> # Marker is prohibited. MaxRecords should be in range [20-100].

  aws_rds_engine_versions:
    describe_db_engine_versions:
      - <RDS.DescribeDBEngineVersions> # Marker is prohibited. MaxRecords should be in range [20-100].

  aws_rds_global_clusters:
    describe_global_clusters:
      - <RDS.DescribeGlobalClusters> # Marker is prohibited. MaxRecords should be in range [20-100].

  aws_rds_instances:
      # Marker is prohibited. MaxRecords should be in range [20-100].
    - describe_db_instances: <RDS.DescribeDBInstances>
      # NextToken, ServiceType and Identifier are prohibited.
      # StartTime, EndTime and MetricQueries are required.
      # MaxResults should be in range [1-25]. PeriodInSeconds should be in range [1-86400].
      get_resource_metrics: <PI.GetResourceMetrics>

  aws_route53_hosted_zones:
    list_hosted_zones: 
      # NextToken, DelegationSetId and HostedZoneType are prohibited. MaxResults should be in range [1-100].
      - <Route53.GetHostedZone>

  aws_securityhub_findings:
    get_findings:
      - <SecurityHub.GetFindings> # NextToken is prohibited. MaxResults should be in range [1-100].
  
  aws_servicequotas_services:
    - ListServices: <ServiceQuota.ListServices> # NextToken is prohibited. MaxResults should be in range [1-100].
      ListServiceQuotas:
        - <ServiceQuota.ListServiceQuotas>

  aws_ssm_sessions:
    describe_sessions:
      - <SSM.DescribeSessions> # NextToken is prohibited. MaxResults should be in range [1-200].

  aws_ssm_inventory_entries:
    list_inventory_entries:
      # NextToken is prohibited. MaxResults should be in range [1-50].
      # InstanceId and TypeName are required.
      - <SSM.ListInventoryEntries>