From 836f71dbea59a35b2eeeeeb982a73c4366089722 Mon Sep 17 00:00:00 2001 From: HastarTara Date: Tue, 27 Aug 2024 12:33:03 +0100 Subject: tests for bucket_name helper --- src/transform_lambda.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'src/transform_lambda.py') diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 2cd9272..cd9541d 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -1,3 +1,4 @@ +from src.dataframes import * import json import boto3 import re @@ -5,7 +6,6 @@ import logging import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -from dataframes import * from botocore.exceptions import ClientError from pg8000.native import Connection, InterfaceError from datetime import datetime @@ -183,13 +183,18 @@ def read_from_s3_subfolder_to_df(tables, bucket, client=boto3.client("s3")): def bucket_name(bucket_prefix, client=boto3.client("s3")): + # response = client.list_buckets() + # for bucket in response["Buckets"]: + # if bucket_prefix in bucket["Name"]: + # return bucket["Name"] + + response = client.list_buckets() bucket_filter = [ - bucket["Name"] - for bucket in response["Buckets"] - if bucket_prefix in bucket["Name"] - ] - + bucket["Name"] + for bucket in response["Buckets"] + if bucket_prefix in bucket["Name"] + ] return bucket_filter[0] -- cgit v1.2.3