From 8255a66c86dd932b8a453298067e0abb11fe639f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 28 Aug 2024 11:14:09 +0100 Subject: test: remove aws_credentials parameter from fixtures --- tests/test_load_lambda.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index 65106f7..da11c1e 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -20,13 +20,13 @@ def aws_credentials(): @pytest.fixture(scope="class") -def mock_s3_client(aws_credentials): +def mock_s3_client(): with mock_aws(): yield boto3.client("s3") @pytest.fixture(scope="class") -def mock_sm_client(aws_credentials): +def mock_sm_client(): with mock_aws(): yield boto3.client("secretsmanager") -- cgit v1.2.3 From 75d43f52a8ae7dd087f389de65b19501f2db4b3e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 28 Aug 2024 11:14:57 +0100 Subject: test: remove unnecessary imports --- tests/test_load_lambda.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'tests') diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index da11c1e..830bb00 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -1,6 +1,4 @@ import pandas as pd -import pyarrow.parquet as pq -from io import BytesIO from moto import mock_aws import boto3 import botocore.exceptions -- cgit v1.2.3 From 23e20341e0b61c75ea81ab8a8d7a55bdd06e1b16 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 28 Aug 2024 11:16:09 +0100 Subject: test: import load_lambda functions individually --- tests/test_load_lambda.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index 830bb00..3ad7158 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -4,7 +4,14 @@ import boto3 import botocore.exceptions import os import pytest -from src.load_lambda import * +from src.load_lambda import ( + lambda_handler, + retrieve_secrets, + connect_to_db_and_return_engine, + convert_parquet_files_to_dfs, + get_transform_bucket, + upload_dfs_to_database, +) import tempfile -- cgit v1.2.3 From d4c20088cdfc602faec21bd55df703a41dc0a343 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 28 Aug 2024 11:17:24 +0100 Subject: test: re-add json module import --- tests/test_load_lambda.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tests') diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index 3ad7158..02fd461 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -13,6 +13,7 @@ from src.load_lambda import ( upload_dfs_to_database, ) import tempfile +import json @pytest.fixture(scope="class") -- cgit v1.2.3 From b94cbcb355ad2bde408e5e714252d06621cbbc8e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 28 Aug 2024 11:19:53 +0100 Subject: test: add staticmethod decorator to methods --- tests/test_load_lambda.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index 02fd461..25a176d 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -38,7 +38,8 @@ def mock_sm_client(): class TestLambdaHandler: - def test_lambda_handler_returns_200_and_table_name_if_uploaded(self, mocker): + @staticmethod + def test_lambda_handler_returns_200_and_table_name_if_uploaded(mocker): mocker.patch( "src.load_lambda.upload_dfs_to_database", return_value={"uploaded": ["table_one", "table_two"], "not_uploaded": []}, @@ -48,7 +49,8 @@ class TestLambdaHandler: assert "table_one" in result["body"] assert "table_two" in result["body"] - def test_lambda_handler_returns_200_and_table_name_if_not_uploaded(self, mocker): + @staticmethod + def test_lambda_handler_returns_200_and_table_name_if_not_uploaded(mocker): mocker.patch( "src.load_lambda.upload_dfs_to_database", return_value={"uploaded": [], "not_uploaded": ["table_one"]}, @@ -57,7 +59,8 @@ class TestLambdaHandler: assert result["statusCode"] == 200 assert "No dataframes were uploaded" in result["body"] - def test_lambda_handler_returns_error_if_both_lists_empty(self, mocker): + @staticmethod + def test_lambda_handler_returns_error_if_both_lists_empty(mocker): mocker.patch( "src.load_lambda.upload_dfs_to_database", return_value={"uploaded": [], "not_uploaded": []}, -- cgit v1.2.3 From 5ae0d01c0429833e61e36c73d2fad4aef565ddd2 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 28 Aug 2024 12:02:21 +0100 Subject: wip: add tests for upload_dfs_to_database --- tests/test_load_lambda.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index 25a176d..cb27178 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -199,7 +199,34 @@ class TestConvertParquetToDfs: class TestUploadDfsToDatabase: - # Full success test - # Partial success test - # Failure test - pass + def test_function_returns_dictionary_with_uploaded_and_not_uploaded_keys( + self, mocker + ): + mocker.patch( + "src.load_lambda.convert_parquet_files_to_dfs", + return_value={"dim_counterparty.parquet": pd.DataFrame()}, + ) + mocker.patch( + "src.load_lambda.connect_to_db_and_return_engine", + return_value="test_engine", + ) + + result = upload_dfs_to_database() + + assert "uploaded" in result + assert "not_uploaded" in result + + def test_function_returns_uploaded_and_not_uploaded_tables(self, mocker): + mocker.patch( + "src.load_lambda.convert_parquet_files_to_dfs", + return_value={"dim_counterparty.parquet": pd.DataFrame()}, + ) + mocker.patch( + "src.load_lambda.connect_to_db_and_return_engine", + return_value="test_engine", + ) + + result = upload_dfs_to_database() + + assert result["uploaded"] == ["dim_counterparty"] + assert result["not_uploaded"] == [] -- cgit v1.2.3 From f109d5b2856efe6b2822e4376fd06b773fa5c4a1 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 28 Aug 2024 12:02:54 +0100 Subject: chore: remove commented code --- tests/test_load_lambda.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'tests') diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index cb27178..290576f 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -160,11 +160,6 @@ class TestConvertParquetToDfs: ) assert result == {} - # def test_function_returns_dictionary_with_table_with_file_key(): - # # need to mock parquet file and upload to mock bucket - # result = convert_parquet_files_to_dfs(bucket_name="transform_bucket", client=mock_s3_client) - # assert "dim_staff" in result - def test_function_returns_dictionary_with_file_key_and_dataframe( self, mock_s3_client ): -- cgit v1.2.3 From c05fd0ed48b84b35b030c96d1e067496ebe71e84 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 28 Aug 2024 12:05:11 +0100 Subject: test: add staticmethod decorator to remaining methods --- tests/test_load_lambda.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) (limited to 'tests') diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index 290576f..cc6a85e 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -72,7 +72,8 @@ class TestLambdaHandler: class TestRetrieveSecrets: - def test_retrieve_secrets_returns_dictionary(self, mock_sm_client): + @staticmethod + def test_retrieve_secrets_returns_dictionary(mock_sm_client): secret = { "cohort_id": "test_cohort_id", "user": "test_user_id", @@ -90,7 +91,8 @@ class TestRetrieveSecrets: assert isinstance(result, dict) - def test_retrieve_secrets_returns_correct_keys_and_values(self, mock_sm_client): + @staticmethod + def test_retrieve_secrets_returns_correct_keys_and_values(mock_sm_client): secret_name = "test_secret" result = json.loads(retrieve_secrets(mock_sm_client, secret_name)) @@ -98,7 +100,8 @@ class TestRetrieveSecrets: assert result["user"] == "test_user_id" assert result["password"] == "test_password" - def test_retrieve_secrets_returns_client_error_if_no_secret(self, mock_sm_client): + @staticmethod + def test_retrieve_secrets_returns_client_error_if_no_secret(mock_sm_client): secret_name = "another_test_secret" with pytest.raises(botocore.exceptions.ClientError) as error: @@ -106,7 +109,8 @@ class TestRetrieveSecrets: class TestConnectToDBAndReturnEngine: - def test_returns_unsuccessful_connection_when_wrong_credentials(self): + @staticmethod + def test_returns_unsuccessful_connection_when_wrong_credentials(): sm_secret = { "host": "host", "port": "port", @@ -120,11 +124,13 @@ class TestConnectToDBAndReturnEngine: class TestGetTransformBucket: - def test_raises_value_error_if_no_buckets(self, mock_s3_client): + @staticmethod + def test_raises_value_error_if_no_buckets(mock_s3_client): with pytest.raises(ValueError, match="No transform bucket found"): get_transform_bucket(mock_s3_client) - def test_raises_value_error_if_no_transform_bucket(self, mock_s3_client): + @staticmethod + def test_raises_value_error_if_no_transform_bucket(mock_s3_client): mock_s3_client.create_bucket( Bucket="extract_bucket", CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, @@ -132,7 +138,8 @@ class TestGetTransformBucket: with pytest.raises(ValueError, match="No transform bucket found"): get_transform_bucket(mock_s3_client) - def test_returns_transform_bucket_if_one_bucket(self, mock_s3_client): + @staticmethod + def test_returns_transform_bucket_if_one_bucket(mock_s3_client): mock_s3_client.create_bucket( Bucket="transform_bucket", CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, @@ -140,7 +147,8 @@ class TestGetTransformBucket: result = get_transform_bucket(mock_s3_client) assert result == "transform_bucket" - def test_only_returns_transform_bucket_if_several_buckets(self, mock_s3_client): + @staticmethod + def test_only_returns_transform_bucket_if_several_buckets(mock_s3_client): mock_s3_client.create_bucket( Bucket="another_test_bucket", CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, @@ -150,7 +158,8 @@ class TestGetTransformBucket: class TestConvertParquetToDfs: - def test_function_returns_empty_dictionary_if_no_files(self, mock_s3_client): + @staticmethod + def test_function_returns_empty_dictionary_if_no_files(mock_s3_client): mock_s3_client.create_bucket( Bucket="transform_bucket", CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, @@ -160,9 +169,8 @@ class TestConvertParquetToDfs: ) assert result == {} - def test_function_returns_dictionary_with_file_key_and_dataframe( - self, mock_s3_client - ): + @staticmethod + def test_function_returns_dictionary_with_file_key_and_dataframe(mock_s3_client): with tempfile.TemporaryDirectory() as tmp: d = { "test": ["Hello", "Bye"], @@ -194,9 +202,8 @@ class TestConvertParquetToDfs: class TestUploadDfsToDatabase: - def test_function_returns_dictionary_with_uploaded_and_not_uploaded_keys( - self, mocker - ): + @staticmethod + def test_function_returns_dictionary_with_uploaded_and_not_uploaded_keys(mocker): mocker.patch( "src.load_lambda.convert_parquet_files_to_dfs", return_value={"dim_counterparty.parquet": pd.DataFrame()}, @@ -211,7 +218,8 @@ class TestUploadDfsToDatabase: assert "uploaded" in result assert "not_uploaded" in result - def test_function_returns_uploaded_and_not_uploaded_tables(self, mocker): + @staticmethod + def test_function_returns_uploaded_and_not_uploaded_tables(mocker): mocker.patch( "src.load_lambda.convert_parquet_files_to_dfs", return_value={"dim_counterparty.parquet": pd.DataFrame()}, -- cgit v1.2.3 From c89de263ec3f79c68d9b7a35d2bffcd4fb62a8c4 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 28 Aug 2024 12:34:02 +0100 Subject: test: update upload_dfs_to_database tests --- tests/test_load_lambda.py | 52 ++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) (limited to 'tests') diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index cc6a85e..b284588 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -14,6 +14,7 @@ from src.load_lambda import ( ) import tempfile import json +from unittest.mock import MagicMock, patch @pytest.fixture(scope="class") @@ -202,34 +203,29 @@ class TestConvertParquetToDfs: class TestUploadDfsToDatabase: - @staticmethod - def test_function_returns_dictionary_with_uploaded_and_not_uploaded_keys(mocker): - mocker.patch( - "src.load_lambda.convert_parquet_files_to_dfs", - return_value={"dim_counterparty.parquet": pd.DataFrame()}, - ) - mocker.patch( - "src.load_lambda.connect_to_db_and_return_engine", - return_value="test_engine", - ) - - result = upload_dfs_to_database() - - assert "uploaded" in result - assert "not_uploaded" in result + @pytest.fixture + def mock_engine(self): + engine = MagicMock() + engine.dispose = MagicMock() + return engine + + @pytest.fixture + def mock_df(self): + df = MagicMock(spec=pd.DataFrame) + df.to_sql = MagicMock() + return df @staticmethod - def test_function_returns_uploaded_and_not_uploaded_tables(mocker): - mocker.patch( + def test_function_returns_dictionary_with_uploaded_and_not_uploaded_keys( + mock_engine, mock_df + ): + with patch( "src.load_lambda.convert_parquet_files_to_dfs", - return_value={"dim_counterparty.parquet": pd.DataFrame()}, - ) - mocker.patch( - "src.load_lambda.connect_to_db_and_return_engine", - return_value="test_engine", - ) - - result = upload_dfs_to_database() - - assert result["uploaded"] == ["dim_counterparty"] - assert result["not_uploaded"] == [] + return_value={"dim_counterparty.parquet": mock_df}, + ), patch( + "src.load_lambda.connect_to_db_and_return_engine", return_value=mock_engine + ): + result = upload_dfs_to_database() + + assert "uploaded" in result + assert "not_uploaded" in result -- cgit v1.2.3