diff options
| -rw-r--r-- | tests/test_transform_lambda.py | 66 |
1 files changed, 50 insertions, 16 deletions
diff --git a/tests/test_transform_lambda.py b/tests/test_transform_lambda.py index cf0723a..bc070fe 100644 --- a/tests/test_transform_lambda.py +++ b/tests/test_transform_lambda.py @@ -1,17 +1,15 @@ -from src.transform_lambda.transform_lambda import read_from_s3_subfolder_to_df, list_existing_s3_files, bucket_name, process_to_parquet_and_upload_to_s3 -import pytest -import pandas as pd -from moto import mock_aws +from datetime import datetime +import logging +import io import os +import numpy as np +from unittest.mock import patch, MagicMock import boto3 +import pandas as pd +from moto import mock_aws from botocore.exceptions import ClientError -import numpy as np -from datetime import datetime - -# /home/lianmei/northcoders/projects/de-project-bentley/src/transform_lambda/transform_lambda.py -# import caplog -import logging - +import pytest +from src.transform_lambda.transform_lambda import read_from_s3_subfolder_to_df, list_existing_s3_files, bucket_name, process_to_parquet_and_upload_to_s3, lambda_handler logger = logging.getLogger() logger.setLevel(logging.INFO) @@ -40,7 +38,6 @@ def mock_extract_bucket(s3_client): ) return mock_extract_bucket - @pytest.fixture(scope="class") def mock_transform_bucket(s3_client): mock_transform_bucket = s3_client.create_bucket( @@ -49,6 +46,30 @@ def mock_transform_bucket(s3_client): ) return mock_transform_bucket +@pytest.fixture(scope="function") +def mock_df_creation_functions(): + with patch('your_module.create_dim_counterparty') as mock_counterparty, \ + patch('your_module.create_dim_date') as mock_date, \ + patch('your_module.create_dim_location') as mock_location, \ + patch('your_module.create_dim_staff') as mock_staff, \ + patch('your_module.create_dim_design') as mock_design, \ + patch('your_module.create_fact_sales_order') as mock_sales, \ + patch('your_module.create_fact_purchase_orders') as mock_purchase, \ + patch('your_module.create_fact_payment') as mock_payment, \ + patch('your_module.create_dim_currency') as mock_currency: + + yield { + 'counterparty': mock_counterparty, + 'date': mock_date, + 'location': mock_location, + 'staff': mock_staff, + 'design': mock_design, + 'sales': mock_sales, + 'purchase': mock_purchase, + 'payment': mock_payment, + 'currency': mock_currency + } + class TestReadFromS3: # @pytest.mark.skip(reason="The test is broken!") @@ -113,7 +134,6 @@ class TestReadFromS3: ) assert list(result.keys()) == tables assert result["Foods"].eq(expected_foods_df, axis="columns").all(axis=None) - # assert result["Cars"].eq(expected_cars_df, axis="columns").all(axis=None) class TestListExistingFiles: @@ -209,7 +229,7 @@ class TestProcessToParquetUploadS3: ) object_list = s3_client.list_objects_v2(Bucket='dummy_transform_buc') s3_uploaded_files = [obj['Key'] for obj in object_list.get('Contents', [])] - print(s3_uploaded_files, '<<<<<< the FILES IN DUMMY TRASN BUC') + # print(s3_uploaded_files, '<<<<<< the FILES IN DUMMY TRASN BUC') assert "flower_data.parquet" in s3_uploaded_files assert response == {"uploaded": ['flower_data'], "not_uploaded": []} @@ -262,5 +282,19 @@ class TestProcessToParquetUploadS3: assert response == {"uploaded": [], "not_uploaded": []} class TestLambdaHandler: - def test_(self): - pass
\ No newline at end of file + def test_func_reads_from_extract_bucket(self, s3_client, mock_extract_bucket, mock_transform_bucket): + mock_db = MagicMock() + mock_connect.return_value = mock_db + mock_csv = "id,name\n1,Lauryn\n2,Hill" + s3_client.put_object(Bucket='dummy_extract_buc', + Key="mock_table.csv", + Body=mock_csv) + + with patch('src.transform_lambda.transform_lambda.read_from_s3_subfolder_to_df') as mock_read: + mock_read.return_value = {'sample_table': pd.read_csv(io.StringIO(mock_csv))} + + lambda_handler({}, {}) + + mock_read.assert_called_once() + + |