replacement for events.tf variable placeholders, s3 bucket notifications, lambda.tf set up (function, zip, permissions), code bucket to store zipped lambdas

5 files changed, 121 insertions, 20 deletions

diff --git a/events.tf b/events - TBD?.tf
index 25fb35b..25fb35b 100644
--- a/events.tf
+++ b/events - TBD?.tf
diff --git a/terraform/events.tf b/terraform/events.tf
index 4d68a23..0196dc3 100644
--- a/terraform/events.tf
+++ b/terraform/events.tf
@@ -11,18 +11,17 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" {
 }
 
 
-resource "aws_cloudwatch_event_target" "lambda" {
+resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" {
   rule      = aws_cloudwatch_event_rule.lambda_trigger.name
   target_id = "TargetFunctionV1"
-  arn       = aws_lambda_function.my_lambda_function.arn
+  arn       = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder
 }
 
 
-
 resource "aws_lambda_permission" "allow_eventbridge" {
   statement_id  = "AllowExecutionFromEventBridge"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.my_lambda_function.function_name 
+  function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder
   principal     = "events.amazonaws.com"
   source_arn    = aws_cloudwatch_event_rule.lambda_trigger.arn  
 }
@@ -32,18 +31,18 @@ resource "aws_lambda_permission" "allow_eventbridge" {
 resource "aws_lambda_permission" "allow_s3_ingestion" {
   statement_id  = "AllowS3InvokeLambdaTransform"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.lambda_transform.function_name
+  function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder
   principal     = "s3.amazonaws.com"
-  source_arn    = aws_s3_bucket.extract.arn
+  source_arn    = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder
 }
 
 
 resource "aws_s3_bucket_notification" "extract_bucket_notification" {
-  bucket = aws_s3_bucket.extract.id
+  bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder
 
   lambda_function {
     events             = ["s3:ObjectCreated:*"]  
-    lambda_function_arn = aws_lambda_function.lambda_transform.arn
+    lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder
   }
 
   depends_on = [aws_lambda_permission.allow_s3_ingestion]
@@ -54,18 +53,18 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" {
 resource "aws_lambda_permission" "allow_s3_transfrom_bucket" {
   statement_id  = "AllowS3InvokeLambdaTransform"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.lambda_transform.function_name
+  function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder
   principal     = "s3.amazonaws.com"
-  source_arn    = aws_s3_bucket.transform.arn
+  source_arn    = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder
 }
 
 
 resource "aws_s3_bucket_notification" "transform_bucket_notification" {
-  bucket = aws_s3_bucket.transform.id
+  bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder
 
   lambda_function {
     events             = ["s3:ObjectCreated:*"]  
-    lambda_function_arn = aws_lambda_function.lambda_transform.arn
+    lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder
   }
 
   depends_on = [aws_lambda_permission.allow_s3_transform]
diff --git a/terraform/lambda.tf b/terraform/lambda.tf
new file mode 100644
index 0000000..09d6697
--- /dev/null
+++ b/terraform/lambda.tf
@@ -0,0 +1,74 @@
+### EXTRACT LAMBDA SET UP
+data "archive_file" "extract_lambda_zip" {
+  type        = "zip"
+  source_file = "${path.module}/../src/extract_lambda.py"
+  output_path = "${path.module}/../extract_function.zip"
+}
+
+resource "aws_lambda_function" "extract_lambda" {
+    function_name = "${var.extract_lambda_name}"
+    s3_bucket = aws_s3_bucket.lambda_bucket.bucket
+    s3_key = "extract_lambda/extract_function.zip"
+    role = aws_iam_role.PLACEHOLDER_extract_lambda_role.arn # << lambda role placehodler
+    handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler
+    runtime = "python3.11"
+    environment {
+        variables = {
+            output = aws_s3_bucket.extract_bucket.bucket
+        }
+    }
+}
+
+resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" {
+  action = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.extract_lambda.function_name
+  principal = "s3.amazonaws.com"
+  source_arn = aws_s3_bucket.extract_bucket.arn
+}
+
+
+### TRANSFORM LAMBDA SET UP
+data "archive_file" "transform_lambda_zip" {
+  type        = "zip"
+  source_file = "${path.module}/../src/transform_lambda.py"
+  output_path = "${path.module}/../transform_function.zip"
+}
+
+resource "aws_lambda_function" "transform_lambda" {
+    function_name = "${var.transform_lambda_name}"
+    s3_bucket = aws_s3_bucket.lambda_bucket.bucket
+    s3_key = "transform_lambda/transform_function.zip"
+    role = aws_iam_role.PLACEHOLDER_transform_lambda_role.arn # << lambda role placehodler
+    handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler
+    runtime = "python3.11"
+    environment {
+        variables = {
+            output = aws_s3_bucket.transform_bucket.bucket
+        }
+    }
+}
+
+resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" {
+  action = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.transform_lambda.function_name
+  principal = "s3.amazonaws.com"
+  source_arn = aws_s3_bucket.transform_bucket.arn
+}
+
+
+### LOAD LAMBDA SET UP
+data "archive_file" "load_lambda_zip" {
+  type        = "zip"
+  source_file = "${path.module}/../src/load_lambda.py"
+  output_path = "${path.module}/../load_function.zip"
+}
+
+resource "aws_lambda_function" "load_lambda" {
+    function_name = "${var.load_lambda_name}"
+    s3_bucket = aws_s3_bucket.lambda_bucket.bucket
+    s3_key = "load_lambda/load_function.zip"
+    role = aws_iam_role.PLACEHOLDER_load_lambda_role.arn # << lambda role placehodler
+    handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler
+    runtime = "python3.11"
+}
+
diff --git a/terraform/s3.tf b/terraform/s3.tf
index bfe891e..8cb65ef 100644
--- a/terraform/s3.tf
+++ b/terraform/s3.tf
@@ -1,17 +1,40 @@
+### EXTRACT BUCKET SET-UP
 resource "aws_s3_bucket" "extract_bucket" {
     bucket = "${var.s3_extract_bucket_name}"
 }
 
+resource "aws_s3_object" "extract_lambda_code" {
+  bucket = aws_s3_bucket.s3_code_bucket_name.bucket
+  key = "${var.extract_lambda_name}/extract_function.zip"
+  source = "${path.module}/../extract_function.zip"
+} # << can't figure out how this is being used but we seem to need it
+
+resource "aws_s3_bucket_notification" "extract_bucket_notification" {
+  bucket = aws_s3_bucket.extract_bucket.id
+  lambda_function {
+    lambda_function_arn = aws_lambda_function.extract_lambda.arn
+    events              = ["s3:ObjectCreated:*"]
+  }
+  depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket]
+}  # << is this the correct permission dependency?
+
+
+### TRANSFORM BUCKET SET-UP
 resource "aws_s3_bucket" "transform_bucket" {
     bucket = "${var.s3_transform_bucket_name}"
 }
 
-resource "aws_s3_bucket" "lambda_bucket" {
-    bucket = "${var.s3_code_bucket_name}"
-}
-
-resource "aws_s3_object" "extract_lambda_code" {
+resource "aws_s3_object" "transform_lambda_code" {
   bucket = aws_s3_bucket.s3_code_bucket_name.bucket
-  key = "${var.extract_lambda_name}/function_e.zip"
-  source = "${path.module}/../function_e.zip"
-}
\ No newline at end of file
+  key = "${var.transform_lambda_name}/transform_function.zip"
+  source = "${path.module}/../transform_function.zip"
+} # << can't figure out how this is being used but we seem to need it
+
+resource "aws_s3_bucket_notification" "transform_bucket_notification" {
+  bucket = aws_s3_bucket.transform_bucket.id
+  lambda_function {
+    lambda_function_arn = aws_lambda_function.transform_lambda.arn
+    events              = ["s3:ObjectCreated:*"]
+  }
+  depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket]
+}  # << is this the correct permission dependency?
diff --git a/terraform/vars.tf b/terraform/vars.tf
index fa84222..cc9348a 100644
--- a/terraform/vars.tf
+++ b/terraform/vars.tf
@@ -23,6 +23,11 @@ variable "transform_lambda_name" {
     default = "transform-lambda" 
 }
 
+variable "load_lambda_name" {
+    type = string
+    default = "load-lambda" 
+}
+
 data "aws_caller_identity" "current" {}
 
 data "aws_region" "current" {}
\ No newline at end of file