From 0c42e8f165e0f98a6c16252e841432922467ef94 Mon Sep 17 00:00:00 2001 From: Ellie Date: Fri, 16 Aug 2024 10:51:16 +0100 Subject: add lambda layer --- terraform/lambda.tf | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 72d1306..658b8c8 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -81,3 +81,38 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } + +locals { + layer_dir = "${path.module}/../python" + requirements = "${path.module}/../requirements.txt" + layer_zip = "${path.module}/../layer.zip" +} + +resource "null_resource" "prepare_layer" { + triggers = { + requirements_hash = filesha1(local.requirements) + } + provisioner "local-exec" { + command = < Date: Fri, 16 Aug 2024 12:19:54 +0100 Subject: comment out rds.tf to increases tf speed --- terraform/rds.tf | 138 +++++++++++++++++++++++++++---------------------------- 1 file changed, 69 insertions(+), 69 deletions(-) (limited to 'terraform') diff --git a/terraform/rds.tf b/terraform/rds.tf index 88783b7..d1b4959 100644 --- a/terraform/rds.tf +++ b/terraform/rds.tf @@ -1,80 +1,80 @@ -data "aws_availability_zones" "available" {} +# data "aws_availability_zones" "available" {} -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "5.12.1" +# module "vpc" { +# source = "terraform-aws-modules/vpc/aws" +# version = "5.12.1" - name = var.project_name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_dns_hostnames = true - enable_dns_support = true -} +# name = var.project_name +# cidr = "10.0.0.0/16" +# azs = data.aws_availability_zones.available.names +# public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] +# enable_dns_hostnames = true +# enable_dns_support = true +# } -resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { - name = "tt-db-subnet" - subnet_ids = module.vpc.public_subnets +# resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { +# name = "tt-db-subnet" +# subnet_ids = module.vpc.public_subnets - tags = { - Name = "${var.project_name}" - } -} +# tags = { +# Name = "${var.project_name}" +# } +# } -resource "aws_security_group" "rds" { - name = "${var.project_name}-rds" - vpc_id = module.vpc.vpc_id +# resource "aws_security_group" "rds" { +# name = "${var.project_name}-rds" +# vpc_id = module.vpc.vpc_id - ingress { - from_port = 5432 - to_port = 5432 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } +# ingress { +# from_port = 5432 +# to_port = 5432 +# protocol = "tcp" +# cidr_blocks = ["0.0.0.0/0"] +# } - egress { - from_port = 5432 - to_port = 5432 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } +# egress { +# from_port = 5432 +# to_port = 5432 +# protocol = "tcp" +# cidr_blocks = ["0.0.0.0/0"] +# } - tags = { - Name = "${var.project_name}-rds" - } -} +# tags = { +# Name = "${var.project_name}-rds" +# } +# } -resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { - name = "tt-db-param" - family = "postgres14" +# resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { +# name = "tt-db-param" +# family = "postgres14" - parameter { - name = "log_connections" - value = "1" - } -} +# parameter { +# name = "log_connections" +# value = "1" +# } +# } -resource "aws_db_instance" "terrific-totes-rds" { - db_name = var.project_name - instance_class = "db.t3.micro" - allocated_storage = 5 - engine = "postgres" - engine_version = "14.10" - username = "totes" - password = "totes123" - # username = "user credentials for the root user" # we could use .env here - # password = "user password for the root user" # we could use .env here - ### alternatively to providing username nad password we can specify: - # resource "aws_kms_key" "example_key" { - # description = "Example KMS Key" - # } - # within the resource: - # manage_master_user_password = true - # master_user_secret_kms_key_id = aws_kms_key.example.key_id - # } - db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name - vpc_security_group_ids = [aws_security_group.rds.id] - parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name - publicly_accessible = false - skip_final_snapshot = true -} +# resource "aws_db_instance" "terrific-totes-rds" { +# db_name = var.project_name +# instance_class = "db.t3.micro" +# allocated_storage = 5 +# engine = "postgres" +# engine_version = "14.10" +# username = "totes" +# password = "totes123" +# # username = "user credentials for the root user" # we could use .env here +# # password = "user password for the root user" # we could use .env here +# ### alternatively to providing username nad password we can specify: +# # resource "aws_kms_key" "example_key" { +# # description = "Example KMS Key" +# # } +# # within the resource: +# # manage_master_user_password = true +# # master_user_secret_kms_key_id = aws_kms_key.example.key_id +# # } +# db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name +# vpc_security_group_ids = [aws_security_group.rds.id] +# parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name +# publicly_accessible = false +# skip_final_snapshot = true +# } -- cgit v1.2.3 From 39a33cecb5e19f15bed4a099b02bdba56c80c073 Mon Sep 17 00:00:00 2001 From: HastarTara Date: Fri, 16 Aug 2024 16:05:03 +0100 Subject: infra[tf] update lambda extract doesnt work yet --- terraform/lambda.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 658b8c8..71ddd11 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -83,7 +83,7 @@ resource "aws_lambda_function" "load_lambda" { } locals { - layer_dir = "${path.module}/../python" + layer_dir = "${path.module}/.." requirements = "${path.module}/../requirements.txt" layer_zip = "${path.module}/../layer.zip" } @@ -94,9 +94,9 @@ resource "null_resource" "prepare_layer" { } provisioner "local-exec" { command = < Date: Fri, 16 Aug 2024 16:23:56 +0100 Subject: infra(tf): add version constraints for null and archive --- terraform/main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 3b06701..310a251 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -4,6 +4,14 @@ terraform { source = "hashicorp/aws" version = "~>5.0" } + null = { + source = "hashicorp/null" + version = "~>3.2.2" + } + archive = { + source = "hashicorp/archive" + version = "~>2.5.0" + } } backend "s3" { bucket = "bentley-project-secrets" -- cgit v1.2.3 From 303725f83cf5551b3d165aa02ce81562de488a01 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Fri, 16 Aug 2024 16:24:44 +0100 Subject: infra(tf): re-add code that creates layer zip --- terraform/lambda.tf | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 71ddd11..67fd6eb 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -83,9 +83,9 @@ resource "aws_lambda_function" "load_lambda" { } locals { - layer_dir = "${path.module}/.." - requirements = "${path.module}/../requirements.txt" - layer_zip = "${path.module}/../layer.zip" + layer_dir = "${path.module}/.." + requirements = "${path.module}/../requirements.txt" + layer_zip = "${path.module}/../layer.zip" } resource "null_resource" "prepare_layer" { @@ -96,23 +96,23 @@ resource "null_resource" "prepare_layer" { command = < Date: Fri, 16 Aug 2024 16:28:04 +0100 Subject: chore(tf): remove dummy username/password sorry hackers! --- terraform/rds.tf | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) (limited to 'terraform') diff --git a/terraform/rds.tf b/terraform/rds.tf index d1b4959..a013fb3 100644 --- a/terraform/rds.tf +++ b/terraform/rds.tf @@ -60,18 +60,8 @@ # allocated_storage = 5 # engine = "postgres" # engine_version = "14.10" -# username = "totes" -# password = "totes123" -# # username = "user credentials for the root user" # we could use .env here -# # password = "user password for the root user" # we could use .env here -# ### alternatively to providing username nad password we can specify: -# # resource "aws_kms_key" "example_key" { -# # description = "Example KMS Key" -# # } -# # within the resource: -# # manage_master_user_password = true -# # master_user_secret_kms_key_id = aws_kms_key.example.key_id -# # } +# username = "" +# password = "" # db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name # vpc_security_group_ids = [aws_security_group.rds.id] # parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name -- cgit v1.2.3 From 1e27974ecc48d8611b87af1b9cd51e29afa8c792 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Fri, 16 Aug 2024 17:15:59 +0100 Subject: test(fx): fix prepare_layer - broken --- terraform/lambda.tf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 67fd6eb..27e6266 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -89,14 +89,13 @@ locals { } resource "null_resource" "prepare_layer" { - triggers = { - requirements_hash = filesha1(local.requirements) - } provisioner "local-exec" { command = < Date: Mon, 19 Aug 2024 11:21:58 +0100 Subject: layers block update, function resources to inlcude attributes: layers, correct handler and source_code_hash --- terraform/lambda.tf | 70 +++++++++++++++++++++++++++++------------------------ terraform/s3.tf | 5 ++++ 2 files changed, 44 insertions(+), 31 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 27e6266..e33bc79 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -12,12 +12,14 @@ resource "aws_s3_object" "extract_lambda_code" { } resource "aws_lambda_function" "extract_lambda" { - function_name = var.extract_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.extract_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "extract_lambda.extract" - runtime = "python3.11" + function_name = var.extract_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.extract_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "extract_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -40,12 +42,14 @@ resource "aws_s3_object" "transform_lambda_code" { } resource "aws_lambda_function" "transform_lambda" { - function_name = var.transform_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.transform_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "transform_lambda.transform" - runtime = "python3.11" + function_name = var.transform_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.transform_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "transform_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -68,12 +72,14 @@ resource "aws_s3_object" "load_lambda_code" { } resource "aws_lambda_function" "load_lambda" { - function_name = var.load_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.load_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "load_lambda.load" - runtime = "python3.11" + function_name = var.load_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.load_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "load_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -82,10 +88,12 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } +# Lambda Layer Specification locals { - layer_dir = "${path.module}/.." - requirements = "${path.module}/../requirements.txt" - layer_zip = "${path.module}/../layer.zip" + layer_dir = "lambda_layer" + requirements = "requirements.txt" + layer_zip = "layer.zip" + layer_name = "lambda_layer_dev" } resource "null_resource" "prepare_layer" { @@ -95,23 +103,23 @@ resource "null_resource" "prepare_layer" { rm -rf python mkdir python pip3 install -r ${local.requirements} -t python/ - zip -r ${local.layer_zip} python/ - EOT - } + zip -r ${local.layer_zip} python + EOT + } #removed / at the end of python in line 99 } -resource "aws_s3_object" "layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "layer.zip" +resource "aws_s3_object" "lambda_layer_zip" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "lambda_layer/${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = "lambda_layer" + layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.layer_zip.key + s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true - depends_on = [aws_s3_object.layer_zip] + depends_on = [aws_s3_object.lambda_layer_zip] } diff --git a/terraform/s3.tf b/terraform/s3.tf index d5cdee3..b3a863c 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -12,3 +12,8 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } + +### LAMBDA LAYER BUCKET +resource "aws_s3_bucket" "lambda_layer_bucket" { + bucket_prefix = "lambda-layer-dev-" +} \ No newline at end of file -- cgit v1.2.3