diff options
| author | lian-manonog <lian.manonog@gmail.com> | 2024-08-23 11:46:59 +0100 |
|---|---|---|
| committer | lian-manonog <lian.manonog@gmail.com> | 2024-08-23 11:46:59 +0100 |
| commit | 3ff2182b8256594dfbfe7d8c7480d2ee70067ce5 (patch) | |
| tree | dc3ef2dfe2bef7a19b2e24a03aa8f45a1f49279c /src/transform_lambda.py | |
| parent | 2231ea89329bd500f7371b7395f5208f7a86c20e (diff) | |
| download | de-project-bentley-3ff2182b8256594dfbfe7d8c7480d2ee70067ce5.tar.gz de-project-bentley-3ff2182b8256594dfbfe7d8c7480d2ee70067ce5.zip | |
trying to resolce git index issue conflicts - commiting was the only solution
Diffstat (limited to 'src/transform_lambda.py')
| -rw-r--r-- | src/transform_lambda.py | 13 |
1 files changed, 4 insertions, 9 deletions
diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 3e74ee0..44454e2 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -6,9 +6,6 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq from src.dataframes import * - -# from src.extract_lambda import extract_bucket, DBConnectionException -import boto3 from botocore.exceptions import ClientError from pg8000.native import Connection, InterfaceError from datetime import datetime @@ -34,7 +31,7 @@ logging.basicConfig( logging.getLogger("botocore").setLevel(logging.WARNING) -tables = [ +TABLES = [ "sales_order", "transaction", "payment", @@ -54,12 +51,11 @@ def lambda_handler(event, context): try: db = connect_to_database() - bucket = bucket_name("transform") + bucket = bucket_name('transform') + existing_s3_files = list_existing_s3_files(bucket) - dict_of_df = read_from_s3_subfolder_to_df( - tables, extract_bucket(), client=boto3.client("s3") - ) + dict_of_df = read_from_s3_subfolder_to_df(TABLES, bucket_name('extract'), client=boto3.client("s3")) immutable_df_dict = { "dim_counterparty": create_dim_counterparty(dict_of_df), @@ -134,7 +130,6 @@ def process_to_parquet_and_upload_to_s3( return status - def retrieve_secrets(): secret_name = "bentley-secrets" region_name = "eu-west-2" |