Goglides Dev 🌱

Balkrishna Pandey
Balkrishna Pandey

Posted on

Python function to upload to s3 bucket

Upload to s3 bucket from Red Hat OpenShift AI.

import os
import boto3
from botocore.client import Config
from botocore.exceptions import ClientError

def upload_files(directory, bucket_name, s3_prefix, endpoint_url, access_key, secret_key, region_name):
    Upload all files from a directory to an S3 bucket under a specified prefix with added debug information.
    session = boto3.session.Session()
    s3_client = session.client(

    print("Client has been configured.")
    print(f"Attempting to upload files from {directory} to {bucket_name}/{s3_prefix}")

    if os.path.exists(directory):
        print(f"Directory {directory} exists.")
        # Check if the directory is empty
        if not os.listdir(directory):
            print("Directory is empty.")
        # Walk through the directory
        for subdir, dirs, files in os.walk(directory):
            if not files:
                print(f"No files found in the directory: {subdir}")
            for file in files:
                full_path = os.path.join(subdir, file)
                relative_path = os.path.relpath(full_path, start=directory)
                s3_path = os.path.join(s3_prefix, relative_path)
                    with open(full_path, 'rb') as data:
                        s3_client.upload_fileobj(data, bucket_name, s3_path)
                        print(f"Successfully uploaded {full_path} to {bucket_name}/{s3_path}")
                except Exception as e:
                    print(f"Failed to upload {full_path}. Error: {e}")
        print(f"Directory {directory} does not exist.")

# Configuration from environment variables
endpoint_url = os.getenv('AWS_S3_ENDPOINT')
region_name = os.getenv('AWS_DEFAULT_REGION')
secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
bucket_name = os.getenv('AWS_S3_BUCKET')
access_key = os.getenv('AWS_ACCESS_KEY_ID')
# directory = '/opt/app-root/src/Mistral-7B-Instruct-v0.2'
# s3_prefix = 'models/Mistral-7B-Instruct-v0.2'

directory = '/opt/app-root/src/t5-small-pytorch'
s3_prefix = 'models/t5-small-pytorch'

upload_files(directory, bucket_name, s3_prefix, endpoint_url, access_key, secret_key, region_name)

Enter fullscreen mode Exit fullscreen mode

Top comments (0)