diff options
Diffstat (limited to 'src/transform_lambda/transform_lambda.py')
| -rw-r--r-- | src/transform_lambda/transform_lambda.py | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/src/transform_lambda/transform_lambda.py b/src/transform_lambda/transform_lambda.py index 93b2284..f782922 100644 --- a/src/transform_lambda/transform_lambda.py +++ b/src/transform_lambda/transform_lambda.py @@ -5,7 +5,7 @@ import logging import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -from dataframes import * +from src.transform_lambda.dataframes import * from botocore.exceptions import ClientError from pg8000.native import Connection, InterfaceError from datetime import datetime @@ -65,6 +65,8 @@ def lambda_handler(event, context): "dim_location": create_dim_location(dict_of_df), "dim_staff": create_dim_staff(dict_of_df), "dim_design": create_dim_design(dict_of_df), + "dim_transaction": create_dim_transaction(dict_of_df), + "dim_payment_type": create_dim_payment_type(dict_of_df), } mutable_df_dict = { @@ -73,7 +75,8 @@ def lambda_handler(event, context): "fact_payment": create_fact_payment(dict_of_df), "dim_currency": create_dim_currency(dict_of_df), } - + print(immutable_df_dict.values()) + print(mutable_df_dict.values()) status = process_to_parquet_and_upload_to_s3( existing_s3_files, immutable_df_dict, mutable_df_dict, bucket ) @@ -191,6 +194,9 @@ def bucket_name(bucket_prefix, client=boto3.client("s3")): if bucket_prefix in bucket["Name"] ] + if not bucket_filter: + raise ValueError(f"No bucket found with prefix: {bucket_prefix}") + return bucket_filter[0] |