Merge branch 'development' into feature/transform_lambda

7 files changed, 237 insertions, 150 deletions

diff --git a/terraform/events.tf b/terraform/events.tf
index 263141f..53ae10a 100644
--- a/terraform/events.tf
+++ b/terraform/events.tf
@@ -1,3 +1,7 @@
+#################
+# Random String #
+#################
+
 resource "random_string" "eventbridge_suffix" {
   length  = 8
   special = false
@@ -16,6 +20,10 @@ resource "random_string" "s3_transform_suffix" {
   upper   = false
 }
 
+#############################
+# EventBridge Configuration #
+#############################
+
 resource "aws_cloudwatch_event_rule" "lambda_trigger" {
   name                = "lambda-scheduled-trigger"
   description         = "Schedule to trigger the Lambda function"
@@ -25,7 +33,7 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" {
 resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" {
   rule       = aws_cloudwatch_event_rule.lambda_trigger.name
   target_id  = "TargetFunctionV1"
-  arn        = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder
+  arn        = aws_lambda_function.extract_lambda.arn
   depends_on = [aws_lambda_permission.allow_eventbridge]
 }
 
@@ -37,54 +45,64 @@ resource "aws_lambda_permission" "allow_eventbridge" {
   source_arn    = aws_cloudwatch_event_rule.lambda_trigger.arn
 
   lifecycle {
-    replace_triggered_by = [random_string.eventbridge_suffix]
+    create_before_destroy = true
+    replace_triggered_by  = [random_string.eventbridge_suffix]
   }
 }
 
-# below is step function 1
+########################################
+# S3 Extract Bucket Notification Setup #
+########################################
+
 resource "aws_lambda_permission" "allow_s3_ingestion" {
   statement_id  = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder
+  function_name = aws_lambda_function.transform_lambda.function_name
   principal     = "s3.amazonaws.com"
-  source_arn    = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder
+  source_arn    = aws_s3_bucket.extract_bucket.arn
 
   lifecycle {
-    replace_triggered_by = [random_string.s3_ingestion_suffix]
+    create_before_destroy = true
+    replace_triggered_by  = [random_string.s3_ingestion_suffix]
   }
 }
 
 
 resource "aws_s3_bucket_notification" "extract_bucket_notification" {
-  bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder
+  bucket = aws_s3_bucket.extract_bucket.id
 
   lambda_function {
     events              = ["s3:ObjectCreated:*"]
-    lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder
+    lambda_function_arn = aws_lambda_function.transform_lambda.arn
   }
 
   depends_on = [aws_lambda_permission.allow_s3_ingestion]
 }
 
+##########################################
+# S3 Transform Bucket Notification Setup #
+##########################################
+
 resource "aws_lambda_permission" "allow_s3_transform_bucket" {
   statement_id  = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder
+  function_name = aws_lambda_function.transform_lambda.function_name
   principal     = "s3.amazonaws.com"
-  source_arn    = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder
+  source_arn    = aws_s3_bucket.transform_bucket.arn
 
   lifecycle {
-    replace_triggered_by = [random_string.s3_transform_suffix]
+    create_before_destroy = true
+    replace_triggered_by  = [random_string.s3_transform_suffix]
   }
 }
 
 
 resource "aws_s3_bucket_notification" "transform_bucket_notification" {
-  bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder
+  bucket = aws_s3_bucket.transform_bucket.id
 
   lambda_function {
     events              = ["s3:ObjectCreated:*"]
-    lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder
+    lambda_function_arn = aws_lambda_function.transform_lambda.arn
   }
 
   depends_on = [aws_lambda_permission.allow_s3_transform_bucket]
diff --git a/terraform/iam.tf b/terraform/iam.tf
index 0e5fa6d..3d62b69 100644
--- a/terraform/iam.tf
+++ b/terraform/iam.tf
@@ -28,17 +28,21 @@ resource "aws_iam_role" "multi_service_role" {
 ########################################################################
 # S3 SETUP                                                        
 # Description: allows allows retention/tagging/access control settings
-# Lambda IAM Policy for S3 Write
+# Lambda IAM Policy for S3
 ########################################################################
 
 # S3 DEFINE POLICY
 data "aws_iam_policy_document" "s3_data_policy_doc" {
   statement {
+    effect = "Allow"
     actions = [
       "s3:PutObject",
       "s3:PutObjectRetention",
       "s3:PutObjectTagging",
-      "s3:PutObjectAcl"
+      "s3:PutObjectAcl",
+      "s3:ListObjects",
+      "s3:ListObjectsV2",
+      "s3:GetObject"
     ]
     resources = [
       "${aws_s3_bucket.extract_bucket.arn}/*",
@@ -46,6 +50,19 @@ data "aws_iam_policy_document" "s3_data_policy_doc" {
       "${aws_s3_bucket.lambda_code_bucket.arn}/*",
     ]
   }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:ListBucket",
+      "s3:ListAllMyBuckets",
+      "s3:ListObjectsV2",
+      "s3:ListObjects"
+    ]
+    resources = [
+      "arn:aws:s3:::*",
+    ]
+  }
 }
 
 
@@ -156,3 +173,30 @@ resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" {
   role       = aws_iam_role.multi_service_role.name
   policy_arn = aws_iam_policy.cloudwatch_events_policy.arn
 }
+
+#########################
+# SECRETS MANAGER SETUP #
+#########################
+
+# Policy Doc
+data "aws_iam_policy_document" "secrets_manager_policy_doc" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "secretsmanager:GetSecretValue"
+    ]
+    resources = ["arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:bentley-secrets-Na0yc8"]
+  }
+}
+
+# SM Policy Resource
+resource "aws_iam_policy" "secrets_manager_policy" {
+  name   = "secrets_manager_policy"
+  policy = data.aws_iam_policy_document.secrets_manager_policy_doc.json
+}
+
+# Attach SM Policy to Role
+resource "aws_iam_role_policy_attachment" "secrets_manager_attachment" {
+  role       = aws_iam_role.multi_service_role.name
+  policy_arn = aws_iam_policy.secrets_manager_policy.arn
+}
diff --git a/terraform/lambda.tf b/terraform/lambda.tf
index 67fd6eb..d33a6c9 100644
--- a/terraform/lambda.tf
+++ b/terraform/lambda.tf
@@ -1,4 +1,54 @@
-# Extract Lambda Function
+####################
+# Common Variables #
+####################
+
+locals {
+  layer_dir      = "../"
+  layer_zip      = "layer.zip"
+  layer_name     = "lambda_layer"
+  script_dir     = "../scripts"
+  layer_zip_path = "${local.layer_dir}/${local.layer_zip}"
+}
+
+######################
+# Lambda Layer Setup #
+######################
+
+resource "null_resource" "prepare_layer" {
+
+  # New change: only run the script if the layer zip does not exist
+
+  triggers = {
+    layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists"
+  }
+
+  provisioner "local-exec" {
+    command = "if [ ! -f ${local.layer_zip_path} ]; then bash ${local.script_dir}/make_layer_zip.sh; fi"
+  }
+}
+
+resource "aws_s3_object" "lambda_layer_zip" {
+  bucket     = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id
+  key        = "${local.layer_name}/${local.layer_zip}"
+  source     = "${local.layer_dir}/${local.layer_zip}"
+  depends_on = [null_resource.prepare_layer]
+  etag       = fileexists(local.layer_zip_path) ? filemd5(local.layer_zip_path) : null
+}
+
+resource "aws_lambda_layer_version" "lambda_layer" {
+  layer_name          = local.layer_name
+  compatible_runtimes = ["python3.11"]
+  s3_bucket           = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key              = aws_s3_object.lambda_layer_zip.key
+  source_code_hash    = fileexists(local.layer_zip_path) ? filebase64sha256(local.layer_zip_path) : null
+  skip_destroy        = true
+  depends_on          = [aws_s3_object.lambda_layer_zip]
+}
+
+###########################
+# Extract Lambda Function #
+###########################
+
 data "archive_file" "extract_lambda_zip" {
   type        = "zip"
   source_file = "${path.module}/../src/extract_lambda.py"
@@ -12,12 +62,15 @@ resource "aws_s3_object" "extract_lambda_code" {
 }
 
 resource "aws_lambda_function" "extract_lambda" {
-  function_name = var.extract_lambda_name
-  s3_bucket     = aws_s3_bucket.lambda_code_bucket.bucket
-  s3_key        = aws_s3_object.extract_lambda_code.key
-  role          = aws_iam_role.multi_service_role.arn
-  handler       = "extract_lambda.extract"
-  runtime       = "python3.11"
+  function_name    = var.extract_lambda_name
+  s3_bucket        = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key           = aws_s3_object.extract_lambda_code.key
+  layers           = [aws_lambda_layer_version.lambda_layer.arn]
+  role             = aws_iam_role.multi_service_role.arn
+  handler          = "extract_lambda.lambda_handler"
+  runtime          = "python3.11"
+  source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256
+  timeout          = 180
 
   lifecycle {
     create_before_destroy = true
@@ -26,7 +79,10 @@ resource "aws_lambda_function" "extract_lambda" {
   depends_on = [aws_s3_object.extract_lambda_code]
 }
 
-# Transform Lambda Function
+#############################
+# Transform Lambda Function #
+#############################
+
 data "archive_file" "transform_lambda_zip" {
   type        = "zip"
   source_file = "${path.module}/../src/transform_lambda.py"
@@ -40,12 +96,15 @@ resource "aws_s3_object" "transform_lambda_code" {
 }
 
 resource "aws_lambda_function" "transform_lambda" {
-  function_name = var.transform_lambda_name
-  s3_bucket     = aws_s3_bucket.lambda_code_bucket.bucket
-  s3_key        = aws_s3_object.transform_lambda_code.key
-  role          = aws_iam_role.multi_service_role.arn
-  handler       = "transform_lambda.transform"
-  runtime       = "python3.11"
+  function_name    = var.transform_lambda_name
+  s3_bucket        = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key           = aws_s3_object.transform_lambda_code.key
+  layers           = [aws_lambda_layer_version.lambda_layer.arn]
+  role             = aws_iam_role.multi_service_role.arn
+  handler          = "transform_lambda.lambda_handler"
+  runtime          = "python3.11"
+  source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256
+  timeout          = 180
 
   lifecycle {
     create_before_destroy = true
@@ -54,7 +113,10 @@ resource "aws_lambda_function" "transform_lambda" {
   depends_on = [aws_s3_object.transform_lambda_code]
 }
 
-# Load Lambda Function
+########################
+# Load Lambda Function #
+########################
+
 data "archive_file" "load_lambda_zip" {
   type        = "zip"
   source_file = "${path.module}/../src/load_lambda.py"
@@ -68,12 +130,15 @@ resource "aws_s3_object" "load_lambda_code" {
 }
 
 resource "aws_lambda_function" "load_lambda" {
-  function_name = var.load_lambda_name
-  s3_bucket     = aws_s3_bucket.lambda_code_bucket.bucket
-  s3_key        = aws_s3_object.load_lambda_code.key
-  role          = aws_iam_role.multi_service_role.arn
-  handler       = "load_lambda.load"
-  runtime       = "python3.11"
+  function_name    = var.load_lambda_name
+  s3_bucket        = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key           = aws_s3_object.load_lambda_code.key
+  layers           = [aws_lambda_layer_version.lambda_layer.arn]
+  role             = aws_iam_role.multi_service_role.arn
+  handler          = "load_lambda.lambda_handler"
+  runtime          = "python3.11"
+  source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256
+  timeout          = 180
 
   lifecycle {
     create_before_destroy = true
@@ -82,37 +147,3 @@ resource "aws_lambda_function" "load_lambda" {
   depends_on = [aws_s3_object.load_lambda_code]
 }
 
-locals {
-  layer_dir    = "${path.module}/.."
-  requirements = "${path.module}/../requirements.txt"
-  layer_zip    = "${path.module}/../layer.zip"
-}
-
-resource "null_resource" "prepare_layer" {
-  triggers = {
-    requirements_hash = filesha1(local.requirements)
-  }
-  provisioner "local-exec" {
-    command = <<EOT
-      mkdir -p ${local.layer_dir}/python/lib/python3.11/site-packages/
-      pip install -r ${local.requirements} -t ${local.layer_dir}/python/lib/python3.11/site-packages/
-      cd ${local.layer_dir} && zip -r ${local.layer_zip} .
-    EOT
-  }
-}
-
-resource "aws_s3_object" "layer_zip" {
-  bucket     = aws_s3_bucket.lambda_code_bucket.bucket
-  key        = "layer.zip"
-  source     = local.layer_zip
-  depends_on = [null_resource.prepare_layer]
-}
-
-resource "aws_lambda_layer_version" "lambda_layer" {
-  layer_name          = "lambda_layer"
-  compatible_runtimes = ["python3.11"]
-  s3_bucket           = aws_s3_bucket.lambda_code_bucket.bucket
-  s3_key              = aws_s3_object.layer_zip.key
-  skip_destroy        = true
-  depends_on          = [aws_s3_object.layer_zip]
-}
diff --git a/terraform/main.tf b/terraform/main.tf
index 310a251..6577b70 100644
--- a/terraform/main.tf
+++ b/terraform/main.tf
@@ -1,4 +1,5 @@
 terraform {
+  required_version = ">= 1.8.0"
   required_providers {
     aws = {
       source  = "hashicorp/aws"
@@ -12,11 +13,16 @@ terraform {
       source  = "hashicorp/archive"
       version = "~>2.5.0"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = "~>3.6.2"
+    }
   }
   backend "s3" {
-    bucket = "bentley-project-secrets"
-    key    = "bentley-project/terraform.tfstate"
-    region = "eu-west-2"
+    bucket  = "bentley-project-secrets"
+    key     = "bentley-project/terraform.tfstate"
+    region  = "eu-west-2"
+    encrypt = true
   }
 }
 
@@ -24,11 +30,11 @@ provider "aws" {
   region = "eu-west-2"
   default_tags {
     tags = {
-      ProjectName = "Terrific-Totes"
-      Team        = "Team-Bentley"
-      Environment = "Dev"
-      GitHubRepo  = "de-project-bentley"
+      ProjectName = var.project_name
+      Environment = var.environment
       ManagedBy   = "Terraform"
+      GitHubRepo  = var.github_repo
+      Team        = var.team_name
     }
   }
 }
diff --git a/terraform/rds.tf b/terraform/rds.tf
deleted file mode 100644
index a013fb3..0000000
--- a/terraform/rds.tf
+++ /dev/null
@@ -1,70 +0,0 @@
-# data "aws_availability_zones" "available" {}
-
-# module "vpc" {
-#   source  = "terraform-aws-modules/vpc/aws"
-#   version = "5.12.1"
-
-#   name                 = var.project_name
-#   cidr                 = "10.0.0.0/16"
-#   azs                  = data.aws_availability_zones.available.names
-#   public_subnets       = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
-#   enable_dns_hostnames = true
-#   enable_dns_support   = true
-# }
-
-# resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" {
-#   name       = "tt-db-subnet"
-#   subnet_ids = module.vpc.public_subnets
-
-#   tags = {
-#     Name = "${var.project_name}"
-#   }
-# }
-
-# resource "aws_security_group" "rds" {
-#   name   = "${var.project_name}-rds"
-#   vpc_id = module.vpc.vpc_id
-
-#   ingress {
-#     from_port   = 5432
-#     to_port     = 5432
-#     protocol    = "tcp"
-#     cidr_blocks = ["0.0.0.0/0"]
-#   }
-
-#   egress {
-#     from_port   = 5432
-#     to_port     = 5432
-#     protocol    = "tcp"
-#     cidr_blocks = ["0.0.0.0/0"]
-#   }
-
-#   tags = {
-#     Name = "${var.project_name}-rds"
-#   }
-# }
-
-# resource "aws_db_parameter_group" "Terrific-Totes-param-gr" {
-#   name   = "tt-db-param"
-#   family = "postgres14"
-
-#   parameter {
-#     name  = "log_connections"
-#     value = "1"
-#   }
-# }
-
-# resource "aws_db_instance" "terrific-totes-rds" {
-#   db_name           = var.project_name
-#   instance_class    = "db.t3.micro"
-#   allocated_storage = 5
-#   engine            = "postgres"
-#   engine_version    = "14.10"
-#   username          = ""
-#   password          = ""
-#   db_subnet_group_name   = aws_db_subnet_group.Terrific-Totes-sub-gr.name
-#   vpc_security_group_ids = [aws_security_group.rds.id]
-#   parameter_group_name   = aws_db_parameter_group.Terrific-Totes-param-gr.name
-#   publicly_accessible    = false
-#   skip_final_snapshot    = true
-# }
diff --git a/terraform/s3.tf b/terraform/s3.tf
index d5cdee3..14e8835 100644
--- a/terraform/s3.tf
+++ b/terraform/s3.tf
@@ -1,14 +1,57 @@
-### EXTRACT BUCKET SET-UP
+########################
+# EXTRACT BUCKET SETUP #
+########################
+
 resource "aws_s3_bucket" "extract_bucket" {
   bucket_prefix = "${var.s3_extract_bucket_name}-"
+  force_destroy = true
+  tags = {
+    Name = "Ingestion Bucket"
+  }
+}
+
+resource "aws_s3_bucket_versioning" "extract_bucket_versioning" {
+  bucket = aws_s3_bucket.extract_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
 }
 
-### TRANSFORM BUCKET SET-UP
+##########################
+# TRANSFORM BUCKET SETUP #
+##########################
+
 resource "aws_s3_bucket" "transform_bucket" {
   bucket_prefix = "${var.s3_transform_bucket_name}-"
+  force_destroy = true
+  tags = {
+    Name = "Transform Bucket"
+  }
+}
+
+
+resource "aws_s3_bucket_versioning" "transform_bucket_versioning" {
+  bucket = aws_s3_bucket.transform_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
 }
 
-### LAMBDA BUCKET
+#######################
+# LAMBDA BUCKET SETUP #
+#######################
+
 resource "aws_s3_bucket" "lambda_code_bucket" {
   bucket_prefix = "${var.s3_code_bucket_name}-"
+  force_destroy = true
+  tags = {
+    Name = "Lambda Bucket"
+  }
+}
+
+resource "aws_s3_bucket_versioning" "lambda_bucket_versioning" {
+  bucket = aws_s3_bucket.lambda_code_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
 }
diff --git a/terraform/vars.tf b/terraform/vars.tf
index 3c88731..b3e3e47 100644
--- a/terraform/vars.tf
+++ b/terraform/vars.tf
@@ -33,6 +33,21 @@ variable "project_name" {
   default = "tt"
 }
 
+variable "environment" {
+  type    = string
+  default = "dev"
+}
+
+variable "github_repo" {
+  type    = string
+  default = "de-project-bentley"
+}
+
+variable "team_name" {
+  type    = string
+  default = "Team-Bentley"
+}
+
 data "aws_caller_identity" "current" {}
 
 data "aws_region" "current" {}