diff options
Diffstat (limited to 'src/transform_lambda.py')
| -rw-r--r-- | src/transform_lambda.py | 9 |
1 files changed, 6 insertions, 3 deletions
diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 44454e2..defa15d 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -51,11 +51,13 @@ def lambda_handler(event, context): try: db = connect_to_database() - bucket = bucket_name('transform') - + bucket = bucket_name("transform") + existing_s3_files = list_existing_s3_files(bucket) - dict_of_df = read_from_s3_subfolder_to_df(TABLES, bucket_name('extract'), client=boto3.client("s3")) + dict_of_df = read_from_s3_subfolder_to_df( + TABLES, bucket_name("extract"), client=boto3.client("s3") + ) immutable_df_dict = { "dim_counterparty": create_dim_counterparty(dict_of_df), @@ -130,6 +132,7 @@ def process_to_parquet_and_upload_to_s3( return status + def retrieve_secrets(): secret_name = "bentley-secrets" region_name = "eu-west-2" |