From 1e27974ecc48d8611b87af1b9cd51e29afa8c792 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Fri, 16 Aug 2024 17:15:59 +0100 Subject: test(fx): fix prepare_layer - broken --- terraform/lambda.tf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 67fd6eb..27e6266 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -89,14 +89,13 @@ locals { } resource "null_resource" "prepare_layer" { - triggers = { - requirements_hash = filesha1(local.requirements) - } provisioner "local-exec" { command = < Date: Mon, 19 Aug 2024 11:21:58 +0100 Subject: layers block update, function resources to inlcude attributes: layers, correct handler and source_code_hash --- terraform/lambda.tf | 70 +++++++++++++++++++++++++++++------------------------ terraform/s3.tf | 5 ++++ 2 files changed, 44 insertions(+), 31 deletions(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 27e6266..e33bc79 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -12,12 +12,14 @@ resource "aws_s3_object" "extract_lambda_code" { } resource "aws_lambda_function" "extract_lambda" { - function_name = var.extract_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.extract_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "extract_lambda.extract" - runtime = "python3.11" + function_name = var.extract_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.extract_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "extract_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -40,12 +42,14 @@ resource "aws_s3_object" "transform_lambda_code" { } resource "aws_lambda_function" "transform_lambda" { - function_name = var.transform_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.transform_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "transform_lambda.transform" - runtime = "python3.11" + function_name = var.transform_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.transform_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "transform_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -68,12 +72,14 @@ resource "aws_s3_object" "load_lambda_code" { } resource "aws_lambda_function" "load_lambda" { - function_name = var.load_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.load_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "load_lambda.load" - runtime = "python3.11" + function_name = var.load_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.load_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "load_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -82,10 +88,12 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } +# Lambda Layer Specification locals { - layer_dir = "${path.module}/.." - requirements = "${path.module}/../requirements.txt" - layer_zip = "${path.module}/../layer.zip" + layer_dir = "lambda_layer" + requirements = "requirements.txt" + layer_zip = "layer.zip" + layer_name = "lambda_layer_dev" } resource "null_resource" "prepare_layer" { @@ -95,23 +103,23 @@ resource "null_resource" "prepare_layer" { rm -rf python mkdir python pip3 install -r ${local.requirements} -t python/ - zip -r ${local.layer_zip} python/ - EOT - } + zip -r ${local.layer_zip} python + EOT + } #removed / at the end of python in line 99 } -resource "aws_s3_object" "layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "layer.zip" +resource "aws_s3_object" "lambda_layer_zip" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "lambda_layer/${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = "lambda_layer" + layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.layer_zip.key + s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true - depends_on = [aws_s3_object.layer_zip] + depends_on = [aws_s3_object.lambda_layer_zip] } diff --git a/terraform/s3.tf b/terraform/s3.tf index d5cdee3..b3a863c 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -12,3 +12,8 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } + +### LAMBDA LAYER BUCKET +resource "aws_s3_bucket" "lambda_layer_bucket" { + bucket_prefix = "lambda-layer-dev-" +} \ No newline at end of file -- cgit v1.2.3 From 57d1e1ee5a13269f1bef6c3b754cb8374a657202 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 14:55:39 +0100 Subject: style: remove redundant comment --- terraform/lambda.tf | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index e33bc79..714ffa5 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -99,13 +99,9 @@ locals { resource "null_resource" "prepare_layer" { provisioner "local-exec" { command = < Date: Mon, 19 Aug 2024 15:02:39 +0100 Subject: infra(tf): modify variables & remove past zip creation --- terraform/lambda.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 714ffa5..986170f 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -90,17 +90,16 @@ resource "aws_lambda_function" "load_lambda" { # Lambda Layer Specification locals { - layer_dir = "lambda_layer" + layer_dir = "../" requirements = "requirements.txt" layer_zip = "layer.zip" layer_name = "lambda_layer_dev" + script_dir = "../scripts" } resource "null_resource" "prepare_layer" { provisioner "local-exec" { - command = < Date: Mon, 19 Aug 2024 15:12:56 +0100 Subject: fix(tf): correct layer.zip output path --- terraform/lambda.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 986170f..8a4207d 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -93,7 +93,7 @@ locals { layer_dir = "../" requirements = "requirements.txt" layer_zip = "layer.zip" - layer_name = "lambda_layer_dev" + layer_name = "lambda_layer" script_dir = "../scripts" } @@ -105,7 +105,7 @@ resource "null_resource" "prepare_layer" { resource "aws_s3_object" "lambda_layer_zip" { bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "lambda_layer/${local.layer_name}/${local.layer_zip}" + key = "${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } @@ -113,7 +113,7 @@ resource "aws_s3_object" "lambda_layer_zip" { resource "aws_lambda_layer_version" "lambda_layer" { layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_bucket = aws_s3_bucket.lambda_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] -- cgit v1.2.3 From cbf1d083dc0bf4d78da83cb169da49731f8ace65 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:18:22 +0100 Subject: fix(tf): correct s3_bucket value for lambda_layer --- terraform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 8a4207d..bf96747 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -113,7 +113,7 @@ resource "aws_s3_object" "lambda_layer_zip" { resource "aws_lambda_layer_version" "lambda_layer" { layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] -- cgit v1.2.3 From 4b3b80a2f2177456ed6c2857a7ae0987d7304360 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:40:01 +0100 Subject: chore(tf): remove unused requirements variable --- terraform/lambda.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index bf96747..72aae04 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -90,11 +90,10 @@ resource "aws_lambda_function" "load_lambda" { # Lambda Layer Specification locals { - layer_dir = "../" - requirements = "requirements.txt" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" } resource "null_resource" "prepare_layer" { -- cgit v1.2.3 From 57e855a797f225cd77401e85a671cde95e07ee70 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:55:19 +0100 Subject: style(tf): improve legibility of lambda.tf sections --- terraform/lambda.tf | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 72aae04..b7b362b 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,4 +1,7 @@ -# Extract Lambda Function +########################### +# Extract Lambda Function # +########################### + data "archive_file" "extract_lambda_zip" { type = "zip" source_file = "${path.module}/../src/extract_lambda.py" @@ -28,7 +31,10 @@ resource "aws_lambda_function" "extract_lambda" { depends_on = [aws_s3_object.extract_lambda_code] } -# Transform Lambda Function +############################# +# Transform Lambda Function # +############################# + data "archive_file" "transform_lambda_zip" { type = "zip" source_file = "${path.module}/../src/transform_lambda.py" @@ -58,7 +64,10 @@ resource "aws_lambda_function" "transform_lambda" { depends_on = [aws_s3_object.transform_lambda_code] } -# Load Lambda Function +######################## +# Load Lambda Function # +######################## + data "archive_file" "load_lambda_zip" { type = "zip" source_file = "${path.module}/../src/load_lambda.py" @@ -88,7 +97,10 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } -# Lambda Layer Specification +###################### +# Lambda Layer Setup # +###################### + locals { layer_dir = "../" layer_zip = "layer.zip" -- cgit v1.2.3 From d0e7b1304efe4ab6de2dc5bef1691b389a5bc449 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:58:46 +0100 Subject: refactor(tf): move sections in lambda.tf for better readability --- terraform/lambda.tf | 69 ++++++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 32 deletions(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index b7b362b..aa730c1 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,3 +1,40 @@ +#################### +# Common Variables # +#################### + +locals { + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" +} + +###################### +# Lambda Layer Setup # +###################### + +resource "null_resource" "prepare_layer" { + provisioner "local-exec" { + command = "bash ${local.script_dir}/make_layer_zip.sh" + } +} + +resource "aws_s3_object" "lambda_layer_zip" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "${local.layer_name}/${local.layer_zip}" + source = "${local.layer_dir}/${local.layer_zip}" + depends_on = [null_resource.prepare_layer] +} + +resource "aws_lambda_layer_version" "lambda_layer" { + layer_name = local.layer_name + compatible_runtimes = ["python3.11"] + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.lambda_layer_zip.key + skip_destroy = true + depends_on = [aws_s3_object.lambda_layer_zip] +} + ########################### # Extract Lambda Function # ########################### @@ -97,35 +134,3 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } -###################### -# Lambda Layer Setup # -###################### - -locals { - layer_dir = "../" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" -} - -resource "null_resource" "prepare_layer" { - provisioner "local-exec" { - command = "bash ${local.script_dir}/make_layer_zip.sh" - } -} - -resource "aws_s3_object" "lambda_layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "${local.layer_name}/${local.layer_zip}" - source = "${local.layer_dir}/${local.layer_zip}" - depends_on = [null_resource.prepare_layer] -} - -resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = local.layer_name - compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.lambda_layer_zip.key - skip_destroy = true - depends_on = [aws_s3_object.lambda_layer_zip] -} -- cgit v1.2.3 From cfd6b462a874da77ada8facb3b2a3c0e85059fa4 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:07:10 +0100 Subject: infra(tf): only create layer zip if doesn't exist --- terraform/lambda.tf | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index aa730c1..b1e0d8e 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -3,10 +3,11 @@ #################### locals { - layer_dir = "../" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" + layer_zip_path = "${local.layer_dir}/${local.layer_zip}" } ###################### @@ -14,8 +15,13 @@ locals { ###################### resource "null_resource" "prepare_layer" { + + triggers = { + layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists" + } + provisioner "local-exec" { - command = "bash ${local.script_dir}/make_layer_zip.sh" + command = "if [ ! -f ${local.layer_zip_path} ]; then bash ${local.script_dir}/make_layer_zip.sh; fi" } } -- cgit v1.2.3 From f035f60c7ece05b70275760238c5513b8f113310 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:09:26 +0100 Subject: docs(tf): add information about layer zip creation --- terraform/lambda.tf | 2 ++ 1 file changed, 2 insertions(+) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index b1e0d8e..fc10431 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -16,6 +16,8 @@ locals { resource "null_resource" "prepare_layer" { + # New change: only run the script if the layer zip does not exist + triggers = { layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists" } -- cgit v1.2.3 From 40c2952e628a92e63b3468be4d49f44a234cacce Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:13:03 +0100 Subject: infra(tf): add md5/source_code_hash checks for lambda layer --- terraform/lambda.tf | 2 ++ 1 file changed, 2 insertions(+) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index fc10431..f8e7515 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -32,6 +32,7 @@ resource "aws_s3_object" "lambda_layer_zip" { key = "${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] + etag = fileexists(local.layer_zip_path) ? filemd5(local.layer_zip_path) : null } resource "aws_lambda_layer_version" "lambda_layer" { @@ -39,6 +40,7 @@ resource "aws_lambda_layer_version" "lambda_layer" { compatible_runtimes = ["python3.11"] s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key + source_code_hash = fileexists(local.layer_zip_path) ? filebase64sha256(local.layer_zip_path) : null skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] } -- cgit v1.2.3 From d1f3fd505c4deea8cc811a851cda15d00419ade3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 15:50:23 +0100 Subject: hotfix: increase lambda timeout --- terraform/lambda.tf | 3 +++ 1 file changed, 3 insertions(+) (limited to 'terraform/lambda.tf') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index f8e7515..d33a6c9 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -70,6 +70,7 @@ resource "aws_lambda_function" "extract_lambda" { handler = "extract_lambda.lambda_handler" runtime = "python3.11" source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256 + timeout = 180 lifecycle { create_before_destroy = true @@ -103,6 +104,7 @@ resource "aws_lambda_function" "transform_lambda" { handler = "transform_lambda.lambda_handler" runtime = "python3.11" source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256 + timeout = 180 lifecycle { create_before_destroy = true @@ -136,6 +138,7 @@ resource "aws_lambda_function" "load_lambda" { handler = "load_lambda.lambda_handler" runtime = "python3.11" source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256 + timeout = 180 lifecycle { create_before_destroy = true -- cgit v1.2.3