From 151429859bca904cbacf18f4b169f1f768fa212a Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 27 Aug 2024 12:01:53 +0100 Subject: remove import as not required --- src/load_lambda.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/load_lambda.py') diff --git a/src/load_lambda.py b/src/load_lambda.py index 6e6bc80..685c562 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -5,7 +5,6 @@ import pyarrow.parquet as pq from io import BytesIO import logging import json -from src.extract_lambda import retrieve_secrets from sqlalchemy import create_engine @@ -169,7 +168,7 @@ def upload_dfs_to_database(): table_name, con=db_engine, schema="project_team_2", - if_exists="overwrite", + if_exists="append", index=False, ) upload_status["uploaded"].append(table_name) @@ -183,7 +182,7 @@ def upload_dfs_to_database(): table_name, con=db_engine, schema="project_team_2", - if_exists="overwrite", + if_exists="append", index=False, ) upload_status["uploaded"].append(table_name) @@ -195,3 +194,6 @@ def upload_dfs_to_database(): logger.error(f"{file_name} does not correspond with table in database") db_engine.dispose() return upload_status + +if __name__ == "__main__": + lambda_handler(None, None) -- cgit v1.2.3 From 8cd9edde84f4ca706ad93b143c5ff7e3397ce981 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 27 Aug 2024 12:28:58 +0100 Subject: add json.loads to retrieve secrests function --- src/load_lambda.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'src/load_lambda.py') diff --git a/src/load_lambda.py b/src/load_lambda.py index 685c562..f08e335 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -40,16 +40,19 @@ def lambda_handler(event, context): return {"statusCode": 500, "body": json.dumps("Internal server error.")} -def retrieve_secrets(): - secret_name = "bentley-RDS-credentials" +def retrieve_secrets(client=None, secret_name=None): + session = boto3.session.Session() region_name = "eu-west-2" - # Create a Secrets Manager client - session = boto3.session.Session() - client = session.client(service_name="secretsmanager", region_name=region_name) + if secret_name == None: + secret_name = "bentley-RDS-credentials" + if client == None: + client = session.client(service_name="secretsmanager", region_name=region_name) + try: get_secret_value_response = client.get_secret_value(SecretId=secret_name) + print(get_secret_value_response) except ClientError as e: logger.error(f"Failed to retrieve secret {secret_name}: {str(e)}") raise e @@ -57,7 +60,7 @@ def retrieve_secrets(): logger.error(f"Secret {secret_name} does not contain a SecretString") raise ValueError(f"Secret {secret_name} does not contain a SecretString") - return get_secret_value_response["SecretString"] + return json.loads(get_secret_value_response["SecretString"]) # connect to database, slightly different way of doing it, to allow manipulation through pandas -- cgit v1.2.3 From d623c42a891f2fe8a26493354af0d9e299f3c526 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 27 Aug 2024 15:19:14 +0100 Subject: refactor: add parameter for sm_secret --- src/load_lambda.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'src/load_lambda.py') diff --git a/src/load_lambda.py b/src/load_lambda.py index f08e335..11d1d70 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -49,7 +49,6 @@ def retrieve_secrets(client=None, secret_name=None): if client == None: client = session.client(service_name="secretsmanager", region_name=region_name) - try: get_secret_value_response = client.get_secret_value(SecretId=secret_name) print(get_secret_value_response) @@ -66,9 +65,12 @@ def retrieve_secrets(client=None, secret_name=None): # connect to database, slightly different way of doing it, to allow manipulation through pandas -def connect_to_db_and_return_engine(): +def connect_to_db_and_return_engine(sm_secret=None): + if sm_secret is None: + sm_secret = retrieve_secrets() + try: - secrets = json.loads(retrieve_secrets()) + secrets = json.loads(sm_secret) host = secrets["host"] port = secrets["port"] user = secrets["user"] @@ -198,5 +200,6 @@ def upload_dfs_to_database(): db_engine.dispose() return upload_status + if __name__ == "__main__": lambda_handler(None, None) -- cgit v1.2.3 From cbfc98a9f43b5a0dae95337057c18c9dc2a298e3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 27 Aug 2024 16:00:29 +0100 Subject: wip: update TestLambdaHandler & lambda_handler function --- src/load_lambda.py | 19 +++++++++++-------- tests/test_load_lambda.py | 12 +++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) (limited to 'src/load_lambda.py') diff --git a/src/load_lambda.py b/src/load_lambda.py index 11d1d70..39fa27d 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -23,18 +23,21 @@ logging.getLogger("botocore").setLevel(logging.INFO) def lambda_handler(event, context): try: uploaded_tables = upload_dfs_to_database() - if not uploaded_tables["uploaded"]: + if uploaded_tables["not_uploaded"]: return { "statusCode": 200, "body": json.dumps("No dataframes were uploaded."), } - return { - "statusCode": 200, - "body": json.dumps( - f"""The following dataframes were uploaded successfully: - {uploaded_tables["uploaded"]} .""" - ), - } + + if uploaded_tables["uploaded"]: + return { + "statusCode": 200, + "body": json.dumps( + f"""The following dataframes were uploaded successfully: + {uploaded_tables["uploaded"]} .""" + ), + } + except Exception as e: logger.error(f"Error: {e}", exc_info=True) return {"statusCode": 500, "body": json.dumps("Internal server error.")} diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index a29b75a..9286e48 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -35,7 +35,7 @@ class TestLambdaHandler: def test_lambda_handler_returns_success(self, mocker): mocker.patch( "src.load_lambda.upload_dfs_to_database", - return_value={"uploaded": ["table_one", "table_two"]}, + return_value={"uploaded": ["table_one", "table_two"], "not_uploaded": []}, ) result = lambda_handler(None, None) assert result["statusCode"] == 200 @@ -45,14 +45,20 @@ class TestLambdaHandler: def test_lambda_handler_does_not_upload_anything(self, mocker): mocker.patch( "src.load_lambda.upload_dfs_to_database", - return_value={"uploaded": []}, + return_value={"uploaded": [], "not_uploaded": []}, ) result = lambda_handler(None, None) assert result["statusCode"] == 200 assert "No dataframes were uploaded" in result["body"] def test_lambda_handler_returns_exception(self, mocker): - pass + mocker.patch( + "src.load_lambda.upload_dfs_to_database", + return_value={"test": []}, + ) + + with pytest.raises(Exception): + lambda_handler(None, None) class TestRetrieveSecrets: -- cgit v1.2.3 From 27f89b78775f9b6fd8d3d560689c53db2beb1b64 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 27 Aug 2024 16:39:38 +0100 Subject: add logger error to lambda handler --- src/load_lambda.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'src/load_lambda.py') diff --git a/src/load_lambda.py b/src/load_lambda.py index 39fa27d..9e15af3 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -5,6 +5,7 @@ import pyarrow.parquet as pq from io import BytesIO import logging import json +import traceback from sqlalchemy import create_engine @@ -28,8 +29,7 @@ def lambda_handler(event, context): "statusCode": 200, "body": json.dumps("No dataframes were uploaded."), } - - if uploaded_tables["uploaded"]: + elif uploaded_tables["uploaded"]: return { "statusCode": 200, "body": json.dumps( @@ -37,10 +37,12 @@ def lambda_handler(event, context): {uploaded_tables["uploaded"]} .""" ), } - + else: + logger.error(f"error") + return {"error"} except Exception as e: - logger.error(f"Error: {e}", exc_info=True) - return {"statusCode": 500, "body": json.dumps("Internal server error.")} + logger.error({e}) + return {"statusCode": 500, "body": {e}} def retrieve_secrets(client=None, secret_name=None): -- cgit v1.2.3 From 0915d4fe4e151d6b593467129b51a1322398fc04 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 27 Aug 2024 17:27:21 +0100 Subject: add json.loads --- src/load_lambda.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src/load_lambda.py') diff --git a/src/load_lambda.py b/src/load_lambda.py index 9e15af3..7339ab9 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -64,7 +64,7 @@ def retrieve_secrets(client=None, secret_name=None): logger.error(f"Secret {secret_name} does not contain a SecretString") raise ValueError(f"Secret {secret_name} does not contain a SecretString") - return json.loads(get_secret_value_response["SecretString"]) + return get_secret_value_response["SecretString"] # connect to database, slightly different way of doing it, to allow manipulation through pandas @@ -72,10 +72,10 @@ def retrieve_secrets(client=None, secret_name=None): def connect_to_db_and_return_engine(sm_secret=None): if sm_secret is None: - sm_secret = retrieve_secrets() + sm_secret = json.loads(retrieve_secrets()) try: - secrets = json.loads(sm_secret) + secrets = sm_secret host = secrets["host"] port = secrets["port"] user = secrets["user"] @@ -171,13 +171,14 @@ def upload_dfs_to_database(): ] for file_name, df in dict_of_dfs.items(): + print(df) if file_name in immutable_df_dict: table_name = file_name.split(".")[0] + print(table_name, "<<<<<") try: df.to_sql( table_name, con=db_engine, - schema="project_team_2", if_exists="append", index=False, ) -- cgit v1.2.3