diff options
Diffstat (limited to 'src/extract_lambda.py')
| -rw-r--r-- | src/extract_lambda.py | 11 |
1 files changed, 5 insertions, 6 deletions
diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 217efdb..15fe785 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -148,12 +148,13 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): WHERE table_schema='public' AND table_type='BASE TABLE';""" ) for table in tables: + print(tables) table_name = table[0] rows = db.run( - f"SELECT * FROM {identifier(table_name)} " "WHERE last_updated >= :latest;", - latest={datetime.strftime(latest_timestamp, "%H-%m-%d %H:%M:%S")}, + f"SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest;", + latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, ) - + print("rows", rows) # Creating a temporary file path and writing the column name to it followed by each row of data if rows: csv_file_path = f"/tmp/{table_name}.csv" @@ -183,7 +184,5 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): logger.error(f"Error uploading to S3: {e}") else: load_status["no change"].append(table_name) - logger.info( - f"No new data in {table_name} name. Latest data retrieved is from {latest_timestamp}." - ) + logger.info(f"No new data") return load_status |