diff options
| author | HastarTara <joslinrashleigh@gmail.com> | 2024-08-27 12:33:03 +0100 |
|---|---|---|
| committer | HastarTara <joslinrashleigh@gmail.com> | 2024-08-27 12:33:03 +0100 |
| commit | 836f71dbea59a35b2eeeeeb982a73c4366089722 (patch) | |
| tree | 271675bd10846c7de8c6e9436f4e6be4410a1c0d /src | |
| parent | c610d3fc42a610ca5daff80606f8e67f9d1e20f2 (diff) | |
| download | de-project-bentley-836f71dbea59a35b2eeeeeb982a73c4366089722.tar.gz de-project-bentley-836f71dbea59a35b2eeeeeb982a73c4366089722.zip | |
tests for bucket_name helper
Diffstat (limited to 'src')
| -rw-r--r-- | src/transform_lambda.py | 17 |
1 files changed, 11 insertions, 6 deletions
diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 2cd9272..cd9541d 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -1,3 +1,4 @@ +from src.dataframes import * import json import boto3 import re @@ -5,7 +6,6 @@ import logging import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -from dataframes import * from botocore.exceptions import ClientError from pg8000.native import Connection, InterfaceError from datetime import datetime @@ -183,13 +183,18 @@ def read_from_s3_subfolder_to_df(tables, bucket, client=boto3.client("s3")): def bucket_name(bucket_prefix, client=boto3.client("s3")): + # response = client.list_buckets() + # for bucket in response["Buckets"]: + # if bucket_prefix in bucket["Name"]: + # return bucket["Name"] + + response = client.list_buckets() bucket_filter = [ - bucket["Name"] - for bucket in response["Buckets"] - if bucket_prefix in bucket["Name"] - ] - + bucket["Name"] + for bucket in response["Buckets"] + if bucket_prefix in bucket["Name"] + ] return bucket_filter[0] |
