diff options
| author | T-Aji <tolujbd2@gmail.com> | 2024-08-14 12:25:57 +0100 |
|---|---|---|
| committer | T-Aji <tolujbd2@gmail.com> | 2024-08-14 12:26:29 +0100 |
| commit | 4f0d6f287ae83d7cdc0df6988ab7b9de10912f16 (patch) | |
| tree | e998ca0a0c4adc8e23b8e573a8a91ff0624a3693 /src/extract_lambda.py | |
| parent | cdb4577b5ad7ae1f708797de6bbf17e289bfac14 (diff) | |
| download | de-project-bentley-4f0d6f287ae83d7cdc0df6988ab7b9de10912f16.tar.gz de-project-bentley-4f0d6f287ae83d7cdc0df6988ab7b9de10912f16.zip | |
feat/passing tests to helper function list_existing_s3_files
Diffstat (limited to 'src/extract_lambda.py')
| -rw-r--r-- | src/extract_lambda.py | 12 |
1 files changed, 7 insertions, 5 deletions
diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 11ea5d1..dc70590 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -18,6 +18,7 @@ password = os.getenv('password') host = os.getenv('host') port = os.getenv('port') + def lambda_handler(event, context): """This lambda function connects to the Totesys database, lists the contents of the ingestion bucket, and converts all tables to CSV and if any of those tables do not exist in, or are different to the ones in s3, it uploads them @@ -69,27 +70,28 @@ def connect_to_database(): raise - -def list_existing_s3_files(): +def list_existing_s3_files(bucket_name='extract_bucket', client=boto3.client('s3')): """Creates a dictionary and populates it with the results of listing the contents of the s3 bucket, then returns the populated dictionary """ - client = boto3.client('s3') + existing_files = {} try: - response = client.list_objects_v2(Bucket=ingestion_bucket) + response = client.list_objects_v2(Bucket='extract_bucket') if 'Contents' in response: for obj in response['Contents']: s3_key = obj['Key'] try: - file_obj = client.get_object(Bucket=ingestion_bucket, Key=s3_key) + file_obj = client.get_object(Bucket=bucket_name, Key=s3_key) file_content = file_obj['Body'].read().decode('utf-8') existing_files[s3_key] = file_content except ClientError as e: logger.error(f'Error retrieving S3 object {s3_key}: {e}') + else: + logger.error('The bucket is empty') except ClientError as e: logger.error(f'Error listing S3 objects: {e}') |
