diff options
| -rw-r--r-- | src/transform_lambda.py | 9 | ||||
| -rw-r--r-- | tests/test_fact_sales_order.py | 16 |
2 files changed, 15 insertions, 10 deletions
diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 44454e2..defa15d 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -51,11 +51,13 @@ def lambda_handler(event, context): try: db = connect_to_database() - bucket = bucket_name('transform') - + bucket = bucket_name("transform") + existing_s3_files = list_existing_s3_files(bucket) - dict_of_df = read_from_s3_subfolder_to_df(TABLES, bucket_name('extract'), client=boto3.client("s3")) + dict_of_df = read_from_s3_subfolder_to_df( + TABLES, bucket_name("extract"), client=boto3.client("s3") + ) immutable_df_dict = { "dim_counterparty": create_dim_counterparty(dict_of_df), @@ -130,6 +132,7 @@ def process_to_parquet_and_upload_to_s3( return status + def retrieve_secrets(): secret_name = "bentley-secrets" region_name = "eu-west-2" diff --git a/tests/test_fact_sales_order.py b/tests/test_fact_sales_order.py index c4fc9f4..dad245e 100644 --- a/tests/test_fact_sales_order.py +++ b/tests/test_fact_sales_order.py @@ -1,13 +1,13 @@ -<<<<<<< Updated upstream +import pandas as pd +from fact_sales_order import create_dim_design, create_dim_staff, create_dim_currency from src.fact_sales_order import ( create_dim_design, create_dim_staff, create_dim_currency, ) -======= -from fact_sales_order import create_dim_design, create_dim_staff, create_dim_currency ->>>>>>> Stashed changes -import pandas as pd +<< << << < Updated upstream +== == == = +>>>>>> > Stashed changes class TestCreateDimDesign: @@ -60,7 +60,8 @@ class TestCreateDimStaff: "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"], } - test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} + test_df = {"staff": pd.DataFrame( + data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) assert isinstance(result, pd.DataFrame) @@ -77,7 +78,8 @@ class TestCreateDimStaff: "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"], } - test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} + test_df = {"staff": pd.DataFrame( + data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) expected_d = { "staff_id": ["Hello", "Bye"], |