From 54aca43830f7c4e6962eabc2aace63256e81eb96 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 29 Aug 2024 11:12:43 +0100 Subject: infra(tf): update load function_name & lambda_function_arn --- terraform/events.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/events.tf b/terraform/events.tf index 53ae10a..7f8f641 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -86,7 +86,7 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { resource "aws_lambda_permission" "allow_s3_transform_bucket" { statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.transform_lambda.function_name + function_name = aws_lambda_function.load_lambda.function_name principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.transform_bucket.arn @@ -102,7 +102,7 @@ resource "aws_s3_bucket_notification" "transform_bucket_notification" { lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.transform_lambda.arn + lambda_function_arn = aws_lambda_function.load_lambda.arn } depends_on = [aws_lambda_permission.allow_s3_transform_bucket] -- cgit v1.2.3 From cd7a8f83dc20c045f607aba9ea0e83e0ef22b19e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 29 Aug 2024 11:16:57 +0100 Subject: chore: remove car_data.parquet --- car_data.parquet | Bin 2827 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 car_data.parquet diff --git a/car_data.parquet b/car_data.parquet deleted file mode 100644 index 1853af6..0000000 Binary files a/car_data.parquet and /dev/null differ -- cgit v1.2.3 From e3f6e9b7477ac633d272d2fcacb13e42bb7eb572 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 29 Aug 2024 12:19:21 +0100 Subject: feat: separate layer zip generation for lambda functions --- scripts/make_layer_zip_01.sh | 8 ++++++++ scripts/make_layer_zip_02.sh | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100755 scripts/make_layer_zip_01.sh create mode 100755 scripts/make_layer_zip_02.sh diff --git a/scripts/make_layer_zip_01.sh b/scripts/make_layer_zip_01.sh new file mode 100755 index 0000000..145ea82 --- /dev/null +++ b/scripts/make_layer_zip_01.sh @@ -0,0 +1,8 @@ +# Description: Make the zip file for the layer for the extract lambda function + +cd "$(dirname "$0")/.." +mkdir -p python/lib/python3.11/site-packages +pip3 install --upgrade -r requirements.txt -t python/lib/python3.11/site-packages +rm layer.zip +zip -r layer.zip python +rm -r python/ diff --git a/scripts/make_layer_zip_02.sh b/scripts/make_layer_zip_02.sh new file mode 100755 index 0000000..53e9099 --- /dev/null +++ b/scripts/make_layer_zip_02.sh @@ -0,0 +1,9 @@ +# Description: Make the zip file for the layer for the transform & load +# lambda functions + +cd "$(dirname "$0")/.." +mkdir -p python/lib/python3.11/site-packages +pip3 install --upgrade -r requirements.txt -t python/lib/python3.11/site-packages +rm layer.zip +zip -r layer.zip python +rm -r python/ -- cgit v1.2.3 From 4838efc2cdff9966d4aa2042068be744e4ae1f84 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 29 Aug 2024 12:19:45 +0100 Subject: chore: rm old make_layer_zip.sh script --- scripts/make_layer_zip.sh | 8 -------- 1 file changed, 8 deletions(-) delete mode 100755 scripts/make_layer_zip.sh diff --git a/scripts/make_layer_zip.sh b/scripts/make_layer_zip.sh deleted file mode 100755 index eabe301..0000000 --- a/scripts/make_layer_zip.sh +++ /dev/null @@ -1,8 +0,0 @@ -# Description: Make the zip file for the layer - -cd "$(dirname "$0")/.." -mkdir -p python/lib/python3.11/site-packages -pip3 install --upgrade -r requirements.txt -t python/lib/python3.11/site-packages -rm layer.zip -zip -r layer.zip python -rm -r python/ -- cgit v1.2.3 From 107619511b8d1668707048d9caf38d4f6da96426 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 29 Aug 2024 12:22:24 +0100 Subject: refactor: separate layer zip prep --- scripts/make_layer_zip_01.sh | 10 +++++----- scripts/make_layer_zip_02.sh | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/make_layer_zip_01.sh b/scripts/make_layer_zip_01.sh index 145ea82..c8b4d2a 100755 --- a/scripts/make_layer_zip_01.sh +++ b/scripts/make_layer_zip_01.sh @@ -1,8 +1,8 @@ # Description: Make the zip file for the layer for the extract lambda function cd "$(dirname "$0")/.." -mkdir -p python/lib/python3.11/site-packages -pip3 install --upgrade -r requirements.txt -t python/lib/python3.11/site-packages -rm layer.zip -zip -r layer.zip python -rm -r python/ +mkdir -p python_01/lib/python3.11/site-packages +pip3 install --upgrade -r requirements_01.txt -t python_01/lib/python3.11/site-packages +rm layer_01.zip +zip -r layer_01.zip python_01 +rm -r python_01/ diff --git a/scripts/make_layer_zip_02.sh b/scripts/make_layer_zip_02.sh index 53e9099..8b5bb33 100755 --- a/scripts/make_layer_zip_02.sh +++ b/scripts/make_layer_zip_02.sh @@ -2,8 +2,8 @@ # lambda functions cd "$(dirname "$0")/.." -mkdir -p python/lib/python3.11/site-packages -pip3 install --upgrade -r requirements.txt -t python/lib/python3.11/site-packages -rm layer.zip -zip -r layer.zip python -rm -r python/ +mkdir -p python_02/lib/python3.11/site-packages +pip3 install --upgrade -r requirements_02.txt -t python_02/lib/python3.11/site-packages +rm layer_02.zip +zip -r layer_02.zip python_02 +rm -r python_02/ -- cgit v1.2.3 From ecf330af27b4b32535de61fd3dd958bd5be6d278 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 29 Aug 2024 12:33:05 +0100 Subject: chore: add new requirements.txt files for extract/transform/lambda layers --- requirements_lambda_01.txt | 3 +++ requirements_lambda_02.txt | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 requirements_lambda_01.txt create mode 100644 requirements_lambda_02.txt diff --git a/requirements_lambda_01.txt b/requirements_lambda_01.txt new file mode 100644 index 0000000..10f56be --- /dev/null +++ b/requirements_lambda_01.txt @@ -0,0 +1,3 @@ +boto3 +botocore +pg8000 \ No newline at end of file diff --git a/requirements_lambda_02.txt b/requirements_lambda_02.txt new file mode 100644 index 0000000..20c88d7 --- /dev/null +++ b/requirements_lambda_02.txt @@ -0,0 +1,9 @@ +pandas +pyarrow +SQLAlchemy +auto_mix_prep +beautifulsoup4 +boto3 +botocore +pg8000 +Requests \ No newline at end of file -- cgit v1.2.3 From abcf3df1913507517f8e2401eb9a79033b755287 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 29 Aug 2024 12:34:21 +0100 Subject: chore: update requirements file names in bash scripts --- scripts/make_layer_zip_01.sh | 2 +- scripts/make_layer_zip_02.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/make_layer_zip_01.sh b/scripts/make_layer_zip_01.sh index c8b4d2a..c2d4534 100755 --- a/scripts/make_layer_zip_01.sh +++ b/scripts/make_layer_zip_01.sh @@ -2,7 +2,7 @@ cd "$(dirname "$0")/.." mkdir -p python_01/lib/python3.11/site-packages -pip3 install --upgrade -r requirements_01.txt -t python_01/lib/python3.11/site-packages +pip3 install --upgrade -r requirements_lambda_01.txt -t python_01/lib/python3.11/site-packages rm layer_01.zip zip -r layer_01.zip python_01 rm -r python_01/ diff --git a/scripts/make_layer_zip_02.sh b/scripts/make_layer_zip_02.sh index 8b5bb33..c788acf 100755 --- a/scripts/make_layer_zip_02.sh +++ b/scripts/make_layer_zip_02.sh @@ -3,7 +3,7 @@ cd "$(dirname "$0")/.." mkdir -p python_02/lib/python3.11/site-packages -pip3 install --upgrade -r requirements_02.txt -t python_02/lib/python3.11/site-packages +pip3 install --upgrade -r requirements_lambda_02.txt -t python_02/lib/python3.11/site-packages rm layer_02.zip zip -r layer_02.zip python_02 rm -r python_02/ -- cgit v1.2.3 From 62417b57133cc1c6df3efd89c247706a93e3f182 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 29 Aug 2024 17:46:33 +0100 Subject: infra(tf): restore make_layer_zip & add layer_01/layer_02 creation in same script --- scripts/make_layer_zip.sh | 17 +++++++++++++++++ scripts/make_layer_zip_01.sh | 8 -------- scripts/make_layer_zip_02.sh | 9 --------- 3 files changed, 17 insertions(+), 17 deletions(-) create mode 100755 scripts/make_layer_zip.sh delete mode 100755 scripts/make_layer_zip_01.sh delete mode 100755 scripts/make_layer_zip_02.sh diff --git a/scripts/make_layer_zip.sh b/scripts/make_layer_zip.sh new file mode 100755 index 0000000..7f64873 --- /dev/null +++ b/scripts/make_layer_zip.sh @@ -0,0 +1,17 @@ +# Description: Make the zip file for the layer for the extract lambda function + +cd "$(dirname "$0")/.." + +# Layer 01 +mkdir -p python/lib/python3.11/site-packages +pip3 install --upgrade -r requirements_lambda_01.txt -t python/lib/python3.11/site-packages +rm layer_01.zip +zip -r layer_01.zip python +rm -r python/ + +# Layer 02 +mkdir -p python/lib/python3.11/site-packages +pip3 install --upgrade -r requirements_lambda_02.txt -t python/lib/python3.11/site-packages +rm layer_02.zip +zip -r layer_02.zip python +rm -r python/ diff --git a/scripts/make_layer_zip_01.sh b/scripts/make_layer_zip_01.sh deleted file mode 100755 index c2d4534..0000000 --- a/scripts/make_layer_zip_01.sh +++ /dev/null @@ -1,8 +0,0 @@ -# Description: Make the zip file for the layer for the extract lambda function - -cd "$(dirname "$0")/.." -mkdir -p python_01/lib/python3.11/site-packages -pip3 install --upgrade -r requirements_lambda_01.txt -t python_01/lib/python3.11/site-packages -rm layer_01.zip -zip -r layer_01.zip python_01 -rm -r python_01/ diff --git a/scripts/make_layer_zip_02.sh b/scripts/make_layer_zip_02.sh deleted file mode 100755 index c788acf..0000000 --- a/scripts/make_layer_zip_02.sh +++ /dev/null @@ -1,9 +0,0 @@ -# Description: Make the zip file for the layer for the transform & load -# lambda functions - -cd "$(dirname "$0")/.." -mkdir -p python_02/lib/python3.11/site-packages -pip3 install --upgrade -r requirements_lambda_02.txt -t python_02/lib/python3.11/site-packages -rm layer_02.zip -zip -r layer_02.zip python_02 -rm -r python_02/ -- cgit v1.2.3 From 904c75c88b9eb2a5f28b5fc737d253f70196a17b Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 29 Aug 2024 17:47:25 +0100 Subject: infra(tf)/wip: update lambda layer creation scripting --- terraform/lambda.tf | 81 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index b6f36fb..1e12180 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -3,46 +3,65 @@ #################### locals { - layer_dir = "../" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" - layer_zip_path = "${local.layer_dir}/${local.layer_zip}" + layer_dir = "../" + layer_zip_01 = "layer_01.zip" + layer_zip_02 = "layer_02.zip" + layer_name_01 = "lambda_layer_01" + layer_name_02 = "lambda_layer_02" + script_dir = "../scripts" + layer_zip_01_path = "${local.layer_dir}${local.layer_zip_01}" + layer_zip_02_path = "${local.layer_dir}${local.layer_zip_02}" } -###################### -# Lambda Layer Setup # -###################### - resource "null_resource" "prepare_layer" { - - # New change: only run the script if the layer zip does not exist - + provisioner "local-exec" { + command = "bash ${local.script_dir}/make_layer_zip.sh" + } triggers = { - layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists" + always_run = timestamp() } +} - provisioner "local-exec" { - command = "if [ ! -f ${local.layer_zip_path} ]; then bash ${local.script_dir}/make_layer_zip.sh; fi" - } +################################ +# Lambda Layer (Extract) Setup # +################################ +resource "aws_s3_object" "lambda_layer_zip_01" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "${local.layer_name_01}/${local.layer_zip_01}" + source = "${local.layer_dir}${local.layer_zip_01}" + depends_on = [null_resource.prepare_layer] + etag = fileexists(local.layer_zip_01_path) ? filemd5(local.layer_zip_01_path) : null + force_destroy = true +} + +resource "aws_lambda_layer_version" "lambda_layer_01" { + layer_name = local.layer_name_01 + compatible_runtimes = ["python3.11"] + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.lambda_layer_zip_01.key + source_code_hash = fileexists(local.layer_zip_01_path) ? filebase64sha256(local.layer_zip_01_path) : null + depends_on = [aws_s3_object.lambda_layer_zip_01] } -resource "aws_s3_object" "lambda_layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "${local.layer_name}/${local.layer_zip}" - source = "${local.layer_dir}/${local.layer_zip}" - depends_on = [null_resource.prepare_layer] - etag = fileexists(local.layer_zip_path) ? filemd5(local.layer_zip_path) : null +######################################### +# Lambda Layer (Load & Transform) Setup # +######################################### +resource "aws_s3_object" "lambda_layer_zip_02" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "${local.layer_name_02}/${local.layer_zip_02}" + source = "${local.layer_dir}${local.layer_zip_02}" + depends_on = [null_resource.prepare_layer] + etag = fileexists(local.layer_zip_02_path) ? filemd5(local.layer_zip_02_path) : null + force_destroy = true } -resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = local.layer_name +resource "aws_lambda_layer_version" "lambda_layer_02" { + layer_name = local.layer_name_02 compatible_runtimes = ["python3.11"] s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.lambda_layer_zip.key - source_code_hash = fileexists(local.layer_zip_path) ? filebase64sha256(local.layer_zip_path) : null - skip_destroy = true - depends_on = [aws_s3_object.lambda_layer_zip] + s3_key = aws_s3_object.lambda_layer_zip_02.key + source_code_hash = fileexists(local.layer_zip_02_path) ? filebase64sha256(local.layer_zip_02_path) : null + depends_on = [aws_s3_object.lambda_layer_zip_02] } ########################### @@ -65,7 +84,7 @@ resource "aws_lambda_function" "extract_lambda" { function_name = var.extract_lambda_name s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.extract_lambda_code.key - layers = [aws_lambda_layer_version.lambda_layer.arn] + layers = [aws_lambda_layer_version.lambda_layer_01.arn] role = aws_iam_role.multi_service_role.arn handler = "extract_lambda.lambda_handler" runtime = "python3.11" @@ -101,7 +120,7 @@ resource "aws_lambda_function" "transform_lambda" { function_name = var.transform_lambda_name s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.transform_lambda_code.key - layers = [aws_lambda_layer_version.lambda_layer.arn] + layers = [aws_lambda_layer_version.lambda_layer_02.arn] role = aws_iam_role.multi_service_role.arn handler = "transform_lambda.lambda_handler" runtime = "python3.11" @@ -135,7 +154,7 @@ resource "aws_lambda_function" "load_lambda" { function_name = var.load_lambda_name s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.load_lambda_code.key - layers = [aws_lambda_layer_version.lambda_layer.arn] + layers = [aws_lambda_layer_version.lambda_layer_02.arn] role = aws_iam_role.multi_service_role.arn handler = "load_lambda.lambda_handler" runtime = "python3.11" -- cgit v1.2.3