Merge branch 'feature/iam/alex-changes' into feature/iam.tf

6 files changed, 167 insertions, 180 deletions

diff --git a/terraform/events.tf b/terraform/events.tf
index ef01feb..263141f 100644
--- a/terraform/events.tf
+++ b/terraform/events.tf
@@ -1,41 +1,57 @@
-resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" {
-  rule      = aws_cloudwatch_event_rule.lambda_trigger.name
-  target_id = "TargetFunctionV1"
-  arn       = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder
-  force_destroy = true
+resource "random_string" "eventbridge_suffix" {
+  length  = 8
+  special = false
+  upper   = false
+}
+
+resource "random_string" "s3_ingestion_suffix" {
+  length  = 8
+  special = false
+  upper   = false
+}
+
+resource "random_string" "s3_transform_suffix" {
+  length  = 8
+  special = false
+  upper   = false
 }
 
 resource "aws_cloudwatch_event_rule" "lambda_trigger" {
   name                = "lambda-scheduled-trigger"
   description         = "Schedule to trigger the Lambda function"
-  schedule_expression = "rate(30 seconds)"
-  force_destroy = true
-  # depends_on = [ 
-  #   aws_cloudwatch_event_target.extract_lambda_cw_event]
-  
-#   event_pattern = jsonencode({
-#     detail-type = 
-#       "AWS Console Sign In via CloudTrail"
-#     ]
-#   })
+  schedule_expression = "rate(30 minutes)"
+}
+
+resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" {
+  rule       = aws_cloudwatch_event_rule.lambda_trigger.name
+  target_id  = "TargetFunctionV1"
+  arn        = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder
+  depends_on = [aws_lambda_permission.allow_eventbridge]
 }
 
 resource "aws_lambda_permission" "allow_eventbridge" {
-  statement_id  = "AllowExecutionFromEventBridge"
+  statement_id  = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder
+  function_name = aws_lambda_function.extract_lambda.function_name
   principal     = "events.amazonaws.com"
-  source_arn    = aws_cloudwatch_event_rule.lambda_trigger.arn  
-}
+  source_arn    = aws_cloudwatch_event_rule.lambda_trigger.arn
 
+  lifecycle {
+    replace_triggered_by = [random_string.eventbridge_suffix]
+  }
+}
 
 # below is step function 1
 resource "aws_lambda_permission" "allow_s3_ingestion" {
-  statement_id  = "AllowS3InvokeLambdaTransform"
+  statement_id  = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}"
   action        = "lambda:InvokeFunction"
   function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder
   principal     = "s3.amazonaws.com"
   source_arn    = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder
+
+  lifecycle {
+    replace_triggered_by = [random_string.s3_ingestion_suffix]
+  }
 }
 
 
@@ -43,21 +59,23 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" {
   bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder
 
   lambda_function {
-    events             = ["s3:ObjectCreated:*"]  
+    events              = ["s3:ObjectCreated:*"]
     lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder
   }
 
   depends_on = [aws_lambda_permission.allow_s3_ingestion]
 }
 
-######
-
 resource "aws_lambda_permission" "allow_s3_transform_bucket" {
-  statement_id  = "AllowS3InvokeLambdaTransform"
+  statement_id  = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}"
   action        = "lambda:InvokeFunction"
   function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder
   principal     = "s3.amazonaws.com"
   source_arn    = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder
+
+  lifecycle {
+    replace_triggered_by = [random_string.s3_transform_suffix]
+  }
 }
 
 
@@ -65,9 +83,9 @@ resource "aws_s3_bucket_notification" "transform_bucket_notification" {
   bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder
 
   lambda_function {
-    events             = ["s3:ObjectCreated:*"]  
+    events              = ["s3:ObjectCreated:*"]
     lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder
   }
 
   depends_on = [aws_lambda_permission.allow_s3_transform_bucket]
-}
\ No newline at end of file
+}
diff --git a/terraform/iam.tf b/terraform/iam.tf
index cf4902a..0e5fa6d 100644
--- a/terraform/iam.tf
+++ b/terraform/iam.tf
@@ -16,9 +16,7 @@ resource "aws_iam_role" "multi_service_role" {
         Principal = {
           Service = [
             "lambda.amazonaws.com",
-            "cloudwatch.amazonaws.com",
-            "events.amazonaws.com",
-            "s3.amazonaws.com"
+            "scheduler.amazonaws.com"
           ]
         }
       }
@@ -57,22 +55,22 @@ data "aws_iam_policy_document" "s3_data_policy_doc" {
 ########################################################################
 
 resource "aws_iam_policy" "lambda_execution_policy" {
-  name = "lambda_execution_policy"
-  path = "/"
+  name        = "lambda_execution_policy"
+  path        = "/"
   description = "IAM policy for Lambda execution"
 
   policy = jsonencode({
     Version = "2012-10-17"
     Statement = [
-        {
+      {
         Effect = "Allow"
         Action = [
           "lambda:InvokeFunction",
           "lambda:GetFunction"
         ]
         Resource = "*"
-        }
-      ]
+      }
+    ]
     }
   )
 }
@@ -87,7 +85,7 @@ data "aws_iam_policy_document" "cw_document" {
     actions = ["logs:CreateLogGroup"]
     resources = [
       "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*"
-      ]
+    ]
   }
 
   statement {
@@ -95,15 +93,15 @@ data "aws_iam_policy_document" "cw_document" {
       "logs:CreateLogStream",
       "logs:CreateLogGroup",
       "logs:PutLogEvents"
-      ]
-      resources = [
-        "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*"
-      ]
+    ]
+    resources = [
+      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*"
+    ]
   }
 }
 
 resource "aws_iam_policy" "cw_policy" {
-  name = "cw_policy"
+  name   = "cw_policy"
   policy = data.aws_iam_policy_document.cw_document.json
 }
 
@@ -116,32 +114,45 @@ resource "aws_iam_policy" "s3_write_policy" {
   policy = data.aws_iam_policy_document.s3_data_policy_doc.json
 }
 
-# S3 ATTACH POLICY
-# resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" {
-#   for_each = toset([
-#     aws_iam_policy.s3_write_policy.arn,
-#     aws_iam_policy.lambda_execution_policy.arn,
-#     aws_iam_policy.cw_policy.arn
-#   ])
-#   role       = aws_iam_role.multi_service_role.name
-#   policy_arn = each.value
-# }
-
 resource "aws_iam_role_policy_attachment" "s3_attachment" {
-  role = aws_iam_role.multi_service_role.name
+  role       = aws_iam_role.multi_service_role.name
   policy_arn = aws_iam_policy.s3_write_policy.arn
 }
 
 resource "aws_iam_role_policy_attachment" "lambda_attachment" {
-  role = aws_iam_role.multi_service_role.name
+  role       = aws_iam_role.multi_service_role.name
   policy_arn = aws_iam_policy.lambda_execution_policy.arn
 }
 
 resource "aws_iam_role_policy_attachment" "cw_attachment" {
-  role = aws_iam_role.multi_service_role.name
+  role       = aws_iam_role.multi_service_role.name
   policy_arn = aws_iam_policy.cw_policy.arn
 }
 
-################
-# RDS POLICIES #
-################
+###################
+# EVENTS POLICIES #
+###################
+
+data "aws_iam_policy_document" "cloudwatch_events_policy" {
+  statement {
+    actions = [
+      "events:PutRule",
+      "events:PutTargets",
+      "events:RemoveTargets",
+      "events:DeleteRule",
+      "events:PutEvents"
+    ]
+    resources = ["*"]
+    effect    = "Allow"
+  }
+}
+
+resource "aws_iam_policy" "cloudwatch_events_policy" {
+  name   = "cloudwatch_events_policy"
+  policy = data.aws_iam_policy_document.cloudwatch_events_policy.json
+}
+
+resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" {
+  role       = aws_iam_role.multi_service_role.name
+  policy_arn = aws_iam_policy.cloudwatch_events_policy.arn
+}
diff --git a/terraform/lambda.tf b/terraform/lambda.tf
index c190ea8..fb0a666 100644
--- a/terraform/lambda.tf
+++ b/terraform/lambda.tf
@@ -1,73 +1,68 @@
-### EXTRACT LAMBDA SET UP
-data "archive_file" "extract_lambda_zip" {
-  type        = "zip"
-  source_file = "${path.module}/../src/extract_lambda.py"
-  output_path = "${path.module}/../extract_function.zip"
+# Extract Lambda Function
+resource "aws_s3_object" "extract_lambda_code" {
+  bucket = aws_s3_bucket.lambda_code_bucket.bucket
+  key    = "${var.extract_lambda_name}/extract_function.zip"
+  source = "${path.module}/../extract_function.zip"
+  etag   = filemd5("${path.module}/../extract_function.zip")
 }
 
 resource "aws_lambda_function" "extract_lambda" {
-    function_name = "${var.extract_lambda_name}"
-    s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket
-    s3_key = "extract-lambda/extract_function.zip"
-    role = aws_iam_role.multi_service_role.arn #<< lambda role placehodler
-    handler = "extract_lambda.extract" # << check that the function is called lambda handler
-    runtime = "python3.11"
-    environment {
-        variables = {
-            output = aws_s3_bucket.extract_bucket.bucket
-        }
-    }
-}
+  function_name = var.extract_lambda_name
+  s3_bucket     = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key        = aws_s3_object.extract_lambda_code.key
+  role          = aws_iam_role.multi_service_role.arn
+  handler       = "extract_lambda.extract"
+  runtime       = "python3.11"
 
-resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" {
-  action = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.extract_lambda.function_name
-  principal = "s3.amazonaws.com"
-  source_arn = aws_s3_bucket.extract_bucket.arn
-}
+  lifecycle {
+    create_before_destroy = true
+  }
 
+  depends_on = [aws_s3_object.extract_lambda_code]
+}
 
-### TRANSFORM LAMBDA SET UP
-data "archive_file" "transform_lambda_zip" {
-  type        = "zip"
-  source_file = "${path.module}/../src/transform_lambda.py"
-  output_path = "${path.module}/../transform_function.zip"
+# Transform Lambda Function
+resource "aws_s3_object" "transform_lambda_code" {
+  bucket = aws_s3_bucket.lambda_code_bucket.bucket
+  key    = "${var.transform_lambda_name}/transform_function.zip"
+  source = "${path.module}/../transform_function.zip"
+  etag   = filemd5("${path.module}/../transform_function.zip")
 }
 
 resource "aws_lambda_function" "transform_lambda" {
-    function_name = "${var.transform_lambda_name}"
-    s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket
-    s3_key = "transform-lambda/transform_function.zip"
-    role = aws_iam_role.multi_service_role.arn # << lambda role placehodler
-    handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler
-    runtime = "python3.11"
-    environment {
-        variables = {
-            output = aws_s3_bucket.transform_bucket.bucket
-        }
-    }
-}
+  function_name = var.transform_lambda_name
+  s3_bucket     = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key        = aws_s3_object.transform_lambda_code.key
+  role          = aws_iam_role.multi_service_role.arn
+  handler       = "transform_lambda.transform"
+  runtime       = "python3.11"
 
-resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" {
-  action = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.transform_lambda.function_name
-  principal = "s3.amazonaws.com"
-  source_arn = aws_s3_bucket.transform_bucket.arn
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  depends_on = [aws_s3_object.transform_lambda_code]
 }
 
-### LOAD LAMBDA SET UP
-data "archive_file" "load_lambda_zip" {
-  type        = "zip"
-  source_file = "${path.module}/../src/load_lambda.py"
-  output_path = "${path.module}/../load_function.zip"
+# Load Lambda Function
+resource "aws_s3_object" "load_lambda_code" {
+  bucket = aws_s3_bucket.lambda_code_bucket.bucket
+  key    = "${var.load_lambda_name}/load_function.zip"
+  source = "${path.module}/../load_function.zip"
+  etag   = filemd5("${path.module}/../load_function.zip")
 }
 
 resource "aws_lambda_function" "load_lambda" {
-    function_name = "${var.load_lambda_name}"
-    s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket
-    s3_key = "load-lambda/load_function.zip"
-    role = aws_iam_role.multi_service_role.arn # << lambda role placehodler
-    handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler
-    runtime = "python3.11"
-}
+  function_name = var.load_lambda_name
+  s3_bucket     = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key        = aws_s3_object.load_lambda_code.key
+  role          = aws_iam_role.multi_service_role.arn
+  handler       = "load_lambda.load"
+  runtime       = "python3.11"
 
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  depends_on = [aws_s3_object.load_lambda_code]
+}
diff --git a/terraform/main.tf b/terraform/main.tf
index 2a048b9..3b06701 100644
--- a/terraform/main.tf
+++ b/terraform/main.tf
@@ -1,26 +1,26 @@
 terraform {
   required_providers {
     aws = {
-        source = "hashicorp/aws"
-        version = "~>5.0"
+      source  = "hashicorp/aws"
+      version = "~>5.0"
     }
   }
   backend "s3" {
     bucket = "bentley-project-secrets"
-    key = "bentley-project/terraform.tfstate"
+    key    = "bentley-project/terraform.tfstate"
     region = "eu-west-2"
   }
 }
 
 provider "aws" {
-    region =  "eu-west-2" 
-    default_tags {
-      tags = {
-        ProjectName = "Terrific-Totes"
-        Team = "Team-Bentley"
-        Environment = "Dev"
-        GitHubRepo = "de-project-bentley"
-        ManagedBy = "Terraform"
-      }
+  region = "eu-west-2"
+  default_tags {
+    tags = {
+      ProjectName = "Terrific-Totes"
+      Team        = "Team-Bentley"
+      Environment = "Dev"
+      GitHubRepo  = "de-project-bentley"
+      ManagedBy   = "Terraform"
     }
-}
\ No newline at end of file
+  }
+}
diff --git a/terraform/s3.tf b/terraform/s3.tf
index 8ab5622..d5cdee3 100644
--- a/terraform/s3.tf
+++ b/terraform/s3.tf
@@ -1,51 +1,14 @@
 ### EXTRACT BUCKET SET-UP
 resource "aws_s3_bucket" "extract_bucket" {
-    bucket_prefix = "${var.s3_extract_bucket_name}-"
+  bucket_prefix = "${var.s3_extract_bucket_name}-"
 }
 
-# resource "aws_s3_bucket_notification" "extract_bucket_notification" {
-#   bucket = aws_s3_bucket.extract_bucket.id
-#   lambda_function {
-#     lambda_function_arn = aws_lambda_function.extract_lambda.arn
-#     events              = ["s3:ObjectCreated:*"]
-#   }
-#   depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket]
-# }  # << is this the correct permission dependency?
-
 ### TRANSFORM BUCKET SET-UP
 resource "aws_s3_bucket" "transform_bucket" {
-    bucket_prefix = "${var.s3_transform_bucket_name}-"
+  bucket_prefix = "${var.s3_transform_bucket_name}-"
 }
 
-# resource "aws_s3_bucket_notification" "transform_bucket_notification" {
-#   bucket = aws_s3_bucket.transform_bucket.id
-#   lambda_function {
-#     lambda_function_arn = aws_lambda_function.transform_lambda.arn
-#     events              = ["s3:ObjectCreated:*"]
-#   }
-#   depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket]
-# }  # << is this the correct permission dependency?
-
-
 ### LAMBDA BUCKET
 resource "aws_s3_bucket" "lambda_code_bucket" {
-    bucket_prefix = "${var.s3_code_bucket_name}-"
+  bucket_prefix = "${var.s3_code_bucket_name}-"
 }
-
-resource "aws_s3_object" "extract_lambda_code" {
-  bucket = aws_s3_bucket.lambda_code_bucket.bucket
-  key = "${var.extract_lambda_name}/extract_function.zip"
-  source = "${path.module}/../extract_function.zip"
-} # << can't figure out how this is being used but we seem to need it
-
-resource "aws_s3_object" "transform_lambda_code" {
-  bucket = aws_s3_bucket.lambda_code_bucket.bucket
-  key = "${var.transform_lambda_name}/transform_function.zip"
-  source = "${path.module}/../transform_function.zip"
-} # << can't figure out how this is being used but we seem to need it
-
-resource "aws_s3_object" "load_lambda_code" {
-  bucket = aws_s3_bucket.lambda_code_bucket.bucket
-  key = "${var.load_lambda_name}/load_function.zip"
-  source = "${path.module}/../load_function.zip"
-}
\ No newline at end of file
diff --git a/terraform/vars.tf b/terraform/vars.tf
index cc9348a..84824ce 100644
--- a/terraform/vars.tf
+++ b/terraform/vars.tf
@@ -1,33 +1,33 @@
 variable "s3_extract_bucket_name" {
-    type = string
-    default = "extract-bucket"
+  type    = string
+  default = "extract-bucket"
 }
 
 variable "s3_transform_bucket_name" {
-    type = string
-    default = "transform-bucket"
+  type    = string
+  default = "transform-bucket"
 }
 
 variable "s3_code_bucket_name" {
-    type = string
-    default = "lambda-bucket"
+  type    = string
+  default = "lambda-bucket"
 }
 
 variable "extract_lambda_name" {
-    type = string
-    default = "extract-lambda"
+  type    = string
+  default = "extract-lambda"
 }
 
 variable "transform_lambda_name" {
-    type = string
-    default = "transform-lambda" 
+  type    = string
+  default = "transform-lambda"
 }
 
 variable "load_lambda_name" {
-    type = string
-    default = "load-lambda" 
+  type    = string
+  default = "load-lambda"
 }
 
 data "aws_caller_identity" "current" {}
 
-data "aws_region" "current" {}
\ No newline at end of file
+data "aws_region" "current" {}