diff options
| -rw-r--r-- | src/transform_lambda.py | 13 | ||||
| -rw-r--r-- | tests/test_fact_sales_order.py | 4 |
2 files changed, 8 insertions, 9 deletions
diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 3e74ee0..44454e2 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -6,9 +6,6 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq from src.dataframes import * - -# from src.extract_lambda import extract_bucket, DBConnectionException -import boto3 from botocore.exceptions import ClientError from pg8000.native import Connection, InterfaceError from datetime import datetime @@ -34,7 +31,7 @@ logging.basicConfig( logging.getLogger("botocore").setLevel(logging.WARNING) -tables = [ +TABLES = [ "sales_order", "transaction", "payment", @@ -54,12 +51,11 @@ def lambda_handler(event, context): try: db = connect_to_database() - bucket = bucket_name("transform") + bucket = bucket_name('transform') + existing_s3_files = list_existing_s3_files(bucket) - dict_of_df = read_from_s3_subfolder_to_df( - tables, extract_bucket(), client=boto3.client("s3") - ) + dict_of_df = read_from_s3_subfolder_to_df(TABLES, bucket_name('extract'), client=boto3.client("s3")) immutable_df_dict = { "dim_counterparty": create_dim_counterparty(dict_of_df), @@ -134,7 +130,6 @@ def process_to_parquet_and_upload_to_s3( return status - def retrieve_secrets(): secret_name = "bentley-secrets" region_name = "eu-west-2" diff --git a/tests/test_fact_sales_order.py b/tests/test_fact_sales_order.py index 87e3ade..c4fc9f4 100644 --- a/tests/test_fact_sales_order.py +++ b/tests/test_fact_sales_order.py @@ -1,8 +1,12 @@ +<<<<<<< Updated upstream from src.fact_sales_order import ( create_dim_design, create_dim_staff, create_dim_currency, ) +======= +from fact_sales_order import create_dim_design, create_dim_staff, create_dim_currency +>>>>>>> Stashed changes import pandas as pd |