Merge branch 'development' into test/tests_transform_lambda

2 files changed, 54 insertions, 35 deletions

diff --git a/terraform/events.tf b/terraform/events.tf
index 53ae10a..7f8f641 100644
--- a/terraform/events.tf
+++ b/terraform/events.tf
@@ -86,7 +86,7 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" {
 resource "aws_lambda_permission" "allow_s3_transform_bucket" {
   statement_id  = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.transform_lambda.function_name
+  function_name = aws_lambda_function.load_lambda.function_name
   principal     = "s3.amazonaws.com"
   source_arn    = aws_s3_bucket.transform_bucket.arn
 
@@ -102,7 +102,7 @@ resource "aws_s3_bucket_notification" "transform_bucket_notification" {
 
   lambda_function {
     events              = ["s3:ObjectCreated:*"]
-    lambda_function_arn = aws_lambda_function.transform_lambda.arn
+    lambda_function_arn = aws_lambda_function.load_lambda.arn
   }
 
   depends_on = [aws_lambda_permission.allow_s3_transform_bucket]
diff --git a/terraform/lambda.tf b/terraform/lambda.tf
index 5f4a58e..1e12180 100644
--- a/terraform/lambda.tf
+++ b/terraform/lambda.tf
@@ -3,46 +3,65 @@
 ####################
 
 locals {
-  layer_dir      = "../"
-  layer_zip      = "layer.zip"
-  layer_name     = "lambda_layer"
-  script_dir     = "../scripts"
-  layer_zip_path = "${local.layer_dir}/${local.layer_zip}"
+  layer_dir         = "../"
+  layer_zip_01      = "layer_01.zip"
+  layer_zip_02      = "layer_02.zip"
+  layer_name_01     = "lambda_layer_01"
+  layer_name_02     = "lambda_layer_02"
+  script_dir        = "../scripts"
+  layer_zip_01_path = "${local.layer_dir}${local.layer_zip_01}"
+  layer_zip_02_path = "${local.layer_dir}${local.layer_zip_02}"
 }
 
-######################
-# Lambda Layer Setup #
-######################
-
 resource "null_resource" "prepare_layer" {
-
-  # New change: only run the script if the layer zip does not exist
-
+  provisioner "local-exec" {
+    command = "bash ${local.script_dir}/make_layer_zip.sh"
+  }
   triggers = {
-    layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists"
+    always_run = timestamp()
   }
+}
 
-  provisioner "local-exec" {
-    command = "if [ ! -f ${local.layer_zip_path} ]; then bash ${local.script_dir}/make_layer_zip.sh; fi"
-  }
+################################
+# Lambda Layer (Extract) Setup #
+################################
+resource "aws_s3_object" "lambda_layer_zip_01" {
+  bucket        = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id
+  key           = "${local.layer_name_01}/${local.layer_zip_01}"
+  source        = "${local.layer_dir}${local.layer_zip_01}"
+  depends_on    = [null_resource.prepare_layer]
+  etag          = fileexists(local.layer_zip_01_path) ? filemd5(local.layer_zip_01_path) : null
+  force_destroy = true
+}
+
+resource "aws_lambda_layer_version" "lambda_layer_01" {
+  layer_name          = local.layer_name_01
+  compatible_runtimes = ["python3.11"]
+  s3_bucket           = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key              = aws_s3_object.lambda_layer_zip_01.key
+  source_code_hash    = fileexists(local.layer_zip_01_path) ? filebase64sha256(local.layer_zip_01_path) : null
+  depends_on          = [aws_s3_object.lambda_layer_zip_01]
 }
 
-resource "aws_s3_object" "lambda_layer_zip" {
-  bucket     = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id
-  key        = "${local.layer_name}/${local.layer_zip}"
-  source     = "${local.layer_dir}/${local.layer_zip}"
-  depends_on = [null_resource.prepare_layer]
-  etag       = fileexists(local.layer_zip_path) ? filemd5(local.layer_zip_path) : null
+#########################################
+# Lambda Layer (Load & Transform) Setup #
+#########################################
+resource "aws_s3_object" "lambda_layer_zip_02" {
+  bucket        = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id
+  key           = "${local.layer_name_02}/${local.layer_zip_02}"
+  source        = "${local.layer_dir}${local.layer_zip_02}"
+  depends_on    = [null_resource.prepare_layer]
+  etag          = fileexists(local.layer_zip_02_path) ? filemd5(local.layer_zip_02_path) : null
+  force_destroy = true
 }
 
-resource "aws_lambda_layer_version" "lambda_layer" {
-  layer_name          = local.layer_name
+resource "aws_lambda_layer_version" "lambda_layer_02" {
+  layer_name          = local.layer_name_02
   compatible_runtimes = ["python3.11"]
   s3_bucket           = aws_s3_bucket.lambda_code_bucket.bucket
-  s3_key              = aws_s3_object.lambda_layer_zip.key
-  source_code_hash    = fileexists(local.layer_zip_path) ? filebase64sha256(local.layer_zip_path) : null
-  skip_destroy        = true
-  depends_on          = [aws_s3_object.lambda_layer_zip]
+  s3_key              = aws_s3_object.lambda_layer_zip_02.key
+  source_code_hash    = fileexists(local.layer_zip_02_path) ? filebase64sha256(local.layer_zip_02_path) : null
+  depends_on          = [aws_s3_object.lambda_layer_zip_02]
 }
 
 ###########################
@@ -65,7 +84,7 @@ resource "aws_lambda_function" "extract_lambda" {
   function_name    = var.extract_lambda_name
   s3_bucket        = aws_s3_bucket.lambda_code_bucket.bucket
   s3_key           = aws_s3_object.extract_lambda_code.key
-  layers           = [aws_lambda_layer_version.lambda_layer.arn]
+  layers           = [aws_lambda_layer_version.lambda_layer_01.arn]
   role             = aws_iam_role.multi_service_role.arn
   handler          = "extract_lambda.lambda_handler"
   runtime          = "python3.11"
@@ -86,8 +105,8 @@ resource "aws_lambda_function" "extract_lambda" {
 
 data "archive_file" "transform_lambda_zip" {
   type        = "zip"
-  source_dir  = "${path.module}../src/transform_lambda" 
-  output_path = "${path.module}../transform_lambda.zip"
+  source_dir  = "${path.module}/../src/transform_lambda"
+  output_path = "${path.module}/../transform_lambda.zip"
 }
 
 resource "aws_s3_object" "transform_lambda_code" {
@@ -101,7 +120,7 @@ resource "aws_lambda_function" "transform_lambda" {
   function_name    = var.transform_lambda_name
   s3_bucket        = aws_s3_bucket.lambda_code_bucket.bucket
   s3_key           = aws_s3_object.transform_lambda_code.key
-  layers           = [aws_lambda_layer_version.lambda_layer.arn]
+  layers           = [aws_lambda_layer_version.lambda_layer_02.arn]
   role             = aws_iam_role.multi_service_role.arn
   handler          = "transform_lambda.lambda_handler"
   runtime          = "python3.11"
@@ -135,7 +154,7 @@ resource "aws_lambda_function" "load_lambda" {
   function_name    = var.load_lambda_name
   s3_bucket        = aws_s3_bucket.lambda_code_bucket.bucket
   s3_key           = aws_s3_object.load_lambda_code.key
-  layers           = [aws_lambda_layer_version.lambda_layer.arn]
+  layers           = [aws_lambda_layer_version.lambda_layer_02.arn]
   role             = aws_iam_role.multi_service_role.arn
   handler          = "load_lambda.lambda_handler"
   runtime          = "python3.11"