diff --git a/aws-modules/aws-s3-v2/README.md b/aws-modules/aws-s3-v2/README.md deleted file mode 100644 index 4ec342894e..0000000000 --- a/aws-modules/aws-s3-v2/README.md +++ /dev/null @@ -1,2 +0,0 @@ -## Relevant Articles -- [Listing All AWS S3 Objects in a Bucket Using Java](https://www.baeldung.com/java-aws-s3-list-bucket-objects) diff --git a/aws-modules/aws-s3-v2/pom.xml b/aws-modules/aws-s3-v2/pom.xml deleted file mode 100644 index 888d98e654..0000000000 --- a/aws-modules/aws-s3-v2/pom.xml +++ /dev/null @@ -1,50 +0,0 @@ - - 4.0.0 - com.baeldung.s3 - aws-s3-v2 - 1.0-SNAPSHOT - aws-s3-v2 - jar - - - com.baeldung - aws-modules - 1.0.0-SNAPSHOT - - - - UTF-8 - 2.20.52 - 3.6.1 - - - - - software.amazon.awssdk - s3 - ${aws.java.sdk.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - false - - - - package - - shade - - - - - - - diff --git a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/Main.java b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/Main.java deleted file mode 100644 index 7b7c95a0d0..0000000000 --- a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/Main.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.baeldung.s3.listobjects; - -import software.amazon.awssdk.regions.Region; - -public class Main { - - private static String AWS_BUCKET = "baeldung-tutorial-s3"; - public static Region AWS_REGION = Region.EU_CENTRAL_1; - - public static void main(String[] args) { - S3Service s3Service = new S3Service(AWS_REGION); - - s3Service.putObject(AWS_BUCKET, FileGenerator.generateFiles(1000, 1)); - s3Service.listBuckets(); - s3Service.listObjectsInBucket(AWS_BUCKET); - s3Service.listAllObjectsInBucket(AWS_BUCKET); - s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, 500); - s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, 500, "backup/"); - - s3Service.cleanup(); - } -} diff --git a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/S3Service.java b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/S3Service.java deleted file mode 100644 index 132123f214..0000000000 --- a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/S3Service.java +++ /dev/null @@ -1,129 +0,0 @@ -package com.baeldung.s3.listobjects; - -import java.util.List; - -import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.*; -import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; - -class S3Service { - - private S3Client s3Client; - - public S3Service(Region awsRegion) { - init(awsRegion); - } - - public S3Service(S3Client s3Client) { - this.s3Client = s3Client; - } - - public void init(Region awsRegion) { - this.s3Client = S3Client.builder() - .region(awsRegion) - .credentialsProvider(ProfileCredentialsProvider.create("default")) - .build(); - } - - public void listObjectsInBucket(String bucketName) { - ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() - .bucket(bucketName) - .build(); - ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request); - - List contents = listObjectsV2Response.contents(); - - System.out.println("Number of objects in the bucket: " + contents.stream().count()); - contents.stream().forEach(System.out::println); - } - - public void listAllObjectsInBucket(String bucketName) { - String nextContinuationToken = null; - long totalObjects = 0; - - do { - ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder() - .bucket(bucketName) - .continuationToken(nextContinuationToken); - - ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build()); - nextContinuationToken = response.nextContinuationToken(); - - totalObjects += response.contents().stream() - .peek(System.out::println) - .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); - } while (nextContinuationToken != null); - System.out.println("Number of objects in the bucket: " + totalObjects); - } - - public void listAllObjectsInBucketPaginated(String bucketName, int pageSize) { - ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() - .bucket(bucketName) - .maxKeys(pageSize) // Set the maxKeys parameter to control the page size - .build(); - - ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request); - long totalObjects = 0; - - for (ListObjectsV2Response page : listObjectsV2Iterable) { - long retrievedPageSize = page.contents().stream() - .peek(System.out::println) - .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); - totalObjects += retrievedPageSize; - System.out.println("Page size: " + retrievedPageSize); - } - System.out.println("Total objects in the bucket: " + totalObjects); - } - - public void listAllObjectsInBucketPaginatedWithPrefix(String bucketName, int pageSize, String prefix) { - ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() - .bucket(bucketName) - .maxKeys(pageSize) // Set the maxKeys parameter to control the page size - .prefix(prefix) - .build(); - - ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request); - long totalObjects = 0; - - for (ListObjectsV2Response page : listObjectsV2Iterable) { - long retrievedPageSize = page.contents().stream() - .peek(System.out::println) - .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); - totalObjects += retrievedPageSize; - System.out.println("Page size: " + retrievedPageSize); - } - System.out.println("Total objects in the bucket: " + totalObjects); - } - - public void listBuckets() { - // List all buckets - ListBucketsResponse listBucketsResponse = s3Client.listBuckets(); - - // Display the bucket names - List buckets = listBucketsResponse.buckets(); - System.out.println("Buckets:"); - for (Bucket bucket : buckets) { - System.out.println(bucket.name()); - } - } - - public void putObject(String bucketName, List files) { - try { - files.stream().forEach(file -> { - s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(file.getName()).build(), - RequestBody.fromByteBuffer(file.getContent())); - System.out.println("Uploaded file: " + file.getName()); - }); - } catch (S3Exception e) { - System.err.println("Upload failed"); - e.printStackTrace(); - } - } - - public void cleanup() { - this.s3Client.close(); - } -} \ No newline at end of file diff --git a/aws-modules/aws-s3-v2/src/main/resources/logback.xml b/aws-modules/aws-s3-v2/src/main/resources/logback.xml deleted file mode 100644 index 91d3d0cfa6..0000000000 --- a/aws-modules/aws-s3-v2/src/main/resources/logback.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - \ No newline at end of file diff --git a/aws-modules/aws-s3/README.md b/aws-modules/aws-s3/README.md index 109a898405..3389fdf454 100644 --- a/aws-modules/aws-s3/README.md +++ b/aws-modules/aws-s3/README.md @@ -8,3 +8,4 @@ This module contains articles about Simple Storage Service (S3) on AWS - [Multipart Uploads in Amazon S3 with Java](https://www.baeldung.com/aws-s3-multipart-upload) - [Using the JetS3t Java Client With Amazon S3](https://www.baeldung.com/jets3t-amazon-s3) - [Check if a Specified Key Exists in a Given S3 Bucket Using Java](https://www.baeldung.com/java-aws-s3-check-specified-key-exists) +- [Listing All AWS S3 Objects in a Bucket Using Java](https://www.baeldung.com/java-aws-s3-list-bucket-objects) \ No newline at end of file diff --git a/aws-modules/aws-s3/pom.xml b/aws-modules/aws-s3/pom.xml index 0ea99588d8..157aeb671d 100644 --- a/aws-modules/aws-s3/pom.xml +++ b/aws-modules/aws-s3/pom.xml @@ -1,7 +1,7 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 aws-s3 0.1.0-SNAPSHOT @@ -16,10 +16,11 @@ - com.amazonaws - aws-java-sdk - ${aws-java-sdk.version} + software.amazon.awssdk + s3 + ${aws.java.sdk.version} + commons-io commons-io @@ -60,6 +61,7 @@ + 2.20.52 1.10.L001 0.9.4.0006L diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3ObjectUtils.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3ObjectUtils.java deleted file mode 100644 index 1c31218ff9..0000000000 --- a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3ObjectUtils.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.baeldung.s3; - -import org.apache.http.HttpStatus; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3; - -public class AWSS3ObjectUtils { - - private AmazonS3 s3Client; - - public AWSS3ObjectUtils(AmazonS3 s3client) { - this.s3Client = s3client; - } - - public boolean doesObjectExistByDefaultMethod(String bucket, String key) { - return s3Client.doesObjectExist(bucket, key); - } - - public boolean doesObjectExistByListObjects(String bucket, String key) { - return s3Client.listObjects(bucket) - .getObjectSummaries() - .stream() - .filter(s3ObjectSummary -> s3ObjectSummary.getKey() - .equals(key)) - .findFirst() - .isPresent(); - } - - public boolean doesObjectExistByMetaData(String bucket, String key) { - try { - s3Client.getObjectMetadata(bucket, key); - return true; - } catch (AmazonServiceException e) { - if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) { - return false; - } else { - throw e; - } - } - } -} \ No newline at end of file diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3Service.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3Service.java deleted file mode 100644 index 64ed043f98..0000000000 --- a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3Service.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.baeldung.s3; - -import java.io.File; -import java.net.URL; -import java.util.List; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.CopyObjectResult; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.DeleteObjectsResult; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; - -public class AWSS3Service { - private final AmazonS3 s3client; - - public AWSS3Service() { - this(new AmazonS3Client() { - }); - } - - public AWSS3Service(AmazonS3 s3client) { - this.s3client = s3client; - } - - //is bucket exist? - public boolean doesBucketExist(String bucketName) { - return s3client.doesBucketExist(bucketName); - } - - //create a bucket - public Bucket createBucket(String bucketName) { - return s3client.createBucket(bucketName); - } - - //list all buckets - public List listBuckets() { - return s3client.listBuckets(); - } - - //delete a bucket - public void deleteBucket(String bucketName) { - s3client.deleteBucket(bucketName); - } - - //uploading object - public PutObjectResult putObject(String bucketName, String key, File file) { - return s3client.putObject(bucketName, key, file); - } - //uploading object and getting url - public URL getObjectURL(String bucketName, String key, File file) { - s3client.putObject(bucketName, key, file); - return s3client.getUrl(bucketName, key); - } - - //listing objects - public ObjectListing listObjects(String bucketName) { - return s3client.listObjects(bucketName); - } - - //get an object - public S3Object getObject(String bucketName, String objectKey) { - return s3client.getObject(bucketName, objectKey); - } - - //copying an object - public CopyObjectResult copyObject( - String sourceBucketName, - String sourceKey, - String destinationBucketName, - String destinationKey - ) { - return s3client.copyObject( - sourceBucketName, - sourceKey, - destinationBucketName, - destinationKey - ); - } - - //deleting an object - public void deleteObject(String bucketName, String objectKey) { - s3client.deleteObject(bucketName, objectKey); - } - - //deleting multiple Objects - public DeleteObjectsResult deleteObjects(DeleteObjectsRequest delObjReq) { - return s3client.deleteObjects(delObjReq); - } -} diff --git a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/File.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/File.java similarity index 85% rename from aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/File.java rename to aws-modules/aws-s3/src/main/java/com/baeldung/s3/File.java index f9fe7f6fb3..9ab817f7fb 100644 --- a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/File.java +++ b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/File.java @@ -1,21 +1,21 @@ -package com.baeldung.s3.listobjects; - -import java.nio.ByteBuffer; - -public class File { - private String name; - private ByteBuffer content; - - public File(String name, ByteBuffer content) { - this.name = name; - this.content = content; - } - - public String getName() { - return name; - } - - public ByteBuffer getContent() { - return content; - } -} +package com.baeldung.s3; + +import java.nio.ByteBuffer; + +public class File { + private String name; + private ByteBuffer content; + + public File(String name, ByteBuffer content) { + this.name = name; + this.content = content; + } + + public String getName() { + return name; + } + + public ByteBuffer getContent() { + return content; + } +} diff --git a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/FileGenerator.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/FileGenerator.java similarity index 91% rename from aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/FileGenerator.java rename to aws-modules/aws-s3/src/main/java/com/baeldung/s3/FileGenerator.java index 7c640591d7..7c35192d22 100644 --- a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/FileGenerator.java +++ b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/FileGenerator.java @@ -1,27 +1,27 @@ -package com.baeldung.s3.listobjects; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -public class FileGenerator { - - public static List generateFiles(int fileCount, int fileSize) { - List files = new ArrayList<>(); - for (int i = 0; i < fileCount; i++) { - String fileName = "file_" + i + ".txt"; - ByteBuffer fileContent = generateRandomBytes(fileSize); - files.add(new File(fileName, fileContent)); - } - return files; - } - - private static ByteBuffer generateRandomBytes(int size) { - byte[] array = new byte[size]; - new Random().nextBytes(array); - return ByteBuffer.wrap(array); - } - - -} +package com.baeldung.s3; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +public class FileGenerator { + + public static List generateFiles(int fileCount, int fileSize) { + List files = new ArrayList<>(); + for (int i = 0; i < fileCount; i++) { + String fileName = "file_" + i + ".txt"; + ByteBuffer fileContent = generateRandomBytes(fileSize); + files.add(new File(fileName, fileContent)); + } + return files; + } + + private static ByteBuffer generateRandomBytes(int size) { + byte[] array = new byte[size]; + new Random().nextBytes(array); + return ByteBuffer.wrap(array); + } + + +} diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/MultipartUpload.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/MultipartUpload.java index 711046c112..de45238994 100644 --- a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/MultipartUpload.java +++ b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/MultipartUpload.java @@ -1,56 +1,101 @@ package com.baeldung.s3; -import com.amazonaws.AmazonClientException; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.event.ProgressListener; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.*; -import java.io.File; -import java.util.concurrent.Executors; public class MultipartUpload { - public static void main(String[] args) throws Exception { + public static void main(String[] args) { String existingBucketName = "baeldung-bucket"; String keyName = "my-picture.jpg"; String filePath = "documents/my-picture.jpg"; - AmazonS3 amazonS3 = AmazonS3ClientBuilder - .standard() - .withCredentials(new DefaultAWSCredentialsProviderChain()) - .withRegion(Regions.DEFAULT_REGION) - .build(); + ProfileCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(); + Region region = Region.US_EAST_1; + S3Client s3 = S3Client.builder() + .region(region) + .credentialsProvider(credentialsProvider) + .build(); - int maxUploadThreads = 5; + // Initiate a multipart upload + CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() + .bucket(existingBucketName) + .key(keyName) + .build(); - TransferManager tm = TransferManagerBuilder - .standard() - .withS3Client(amazonS3) - .withMultipartUploadThreshold((long) (5 * 1024 * 1024)) - .withExecutorFactory(() -> Executors.newFixedThreadPool(maxUploadThreads)) - .build(); + CreateMultipartUploadResponse createResponse = s3.createMultipartUpload(createRequest); - ProgressListener progressListener = - progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred()); + String uploadId = createResponse.uploadId(); - PutObjectRequest request = new PutObjectRequest(existingBucketName, keyName, new File(filePath)); + // Prepare the parts to be uploaded + List completedParts = new ArrayList<>(); + int partNumber = 1; + ByteBuffer buffer = ByteBuffer.allocate(5 * 1024 * 1024); // Set your preferred part size (5 MB in this example) - request.setGeneralProgressListener(progressListener); + // Read the file and upload each part + try (RandomAccessFile file = new RandomAccessFile(filePath, "r")) { + long fileSize = file.length(); + long position = 0; - Upload upload = tm.upload(request); + while (position < fileSize) { + file.seek(position); + int bytesRead = file.getChannel().read(buffer); - try { - upload.waitForCompletion(); - System.out.println("Upload complete."); - } catch (AmazonClientException e) { - System.out.println("Error occurred while uploading file"); + buffer.flip(); + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(existingBucketName) + .key(keyName) + .uploadId(uploadId) + .partNumber(partNumber) + .contentLength((long) bytesRead) + .build(); + + UploadPartResponse response = s3.uploadPart(uploadPartRequest, RequestBody.fromByteBuffer(buffer)); + + completedParts.add(CompletedPart.builder() + .partNumber(partNumber) + .eTag(response.eTag()) + .build()); + + buffer.clear(); + position += bytesRead; + partNumber++; + } + } catch (IOException e) { e.printStackTrace(); } + + // Complete the multipart upload + CompletedMultipartUpload completedUpload = CompletedMultipartUpload.builder() + .parts(completedParts) + .build(); + + CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() + .bucket(existingBucketName) + .key(keyName) + .uploadId(uploadId) + .multipartUpload(completedUpload) + .build(); + + CompleteMultipartUploadResponse completeResponse = s3.completeMultipartUpload(completeRequest); + + // Print the object's URL + String objectUrl = s3.utilities().getUrl(GetUrlRequest.builder() + .bucket(existingBucketName) + .key(keyName) + .build()) + .toExternalForm(); + + System.out.println("Uploaded object URL: " + objectUrl); } } \ No newline at end of file diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java index 39a9b0050f..abf570f0d0 100644 --- a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java +++ b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java @@ -1,104 +1,69 @@ package com.baeldung.s3; import java.io.File; -import java.io.IOException; +import java.util.List; -import org.apache.commons.io.FileUtils; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import software.amazon.awssdk.regions.Region; public class S3Application { - private static final AWSCredentials credentials; - private static String bucketName; + private static String AWS_BUCKET = "baeldung-tutorial-s3"; + public static Region AWS_REGION = Region.EU_CENTRAL_1; - static { - //put your accesskey and secretkey here - credentials = new BasicAWSCredentials( - "", - "" - ); - } - - public static void main(String[] args) throws IOException { - //set-up the client - AmazonS3 s3client = AmazonS3ClientBuilder - .standard() - .withCredentials(new AWSStaticCredentialsProvider(credentials)) - .withRegion(Regions.US_EAST_2) - .build(); - - AWSS3Service awsService = new AWSS3Service(s3client); - - bucketName = "baeldung-bucket"; + public static void main(String[] args) { + S3Service s3Service = new S3Service(AWS_REGION); //creating a bucket - if(awsService.doesBucketExist(bucketName)) { + s3Service.createBucket(AWS_BUCKET); + + //check if a bucket exists + if(s3Service.doesBucketExist(AWS_BUCKET)) { System.out.println("Bucket name is not available." - + " Try again with a different Bucket name."); + + " Try again with a different Bucket name."); return; } - awsService.createBucket(bucketName); - - //list all the buckets - for(Bucket s : awsService.listBuckets() ) { - System.out.println(s.getName()); - } - - //deleting bucket - awsService.deleteBucket("baeldung-bucket-test2"); - - //uploading object - awsService.putObject( - bucketName, - "Document/hello.txt", - new File("/Users/user/Document/hello.txt") - ); - //uploading object and getting url - awsService.getObjectURL(bucketName, "Document/hello.txt", new File("/Users/user/Document/hello.txt")); + + s3Service.putObjects(AWS_BUCKET, FileGenerator.generateFiles(1000, 1)); + + //list all the buckets + s3Service.listBuckets(); + s3Service.listObjectsInBucket(AWS_BUCKET); + s3Service.listAllObjectsInBucket(AWS_BUCKET); + s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, 500); + s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, 500, "backup/"); + + + + //deleting bucket + s3Service.deleteBucket("baeldung-bucket-test2"); + + //uploading object + s3Service.putObject( + AWS_BUCKET, + "Document/hello.txt", + new File("/Users/user/Document/hello.txt") + ); //listing objects - ObjectListing objectListing = awsService.listObjects(bucketName); - for(S3ObjectSummary os : objectListing.getObjectSummaries()) { - System.out.println(os.getKey()); - } + s3Service.listObjects(AWS_BUCKET); //downloading an object - S3Object s3object = awsService.getObject(bucketName, "Document/hello.txt"); - S3ObjectInputStream inputStream = s3object.getObjectContent(); - FileUtils.copyInputStreamToFile(inputStream, new File("/Users/user/Desktop/hello.txt")); - + s3Service.getObject(AWS_BUCKET, "Document/hello.txt"); + //copying an object - awsService.copyObject( - "baeldung-bucket", - "picture/pic.png", - "baeldung-bucket2", - "Document/picture.png" + s3Service.copyObject( + "baeldung-bucket", + "picture/pic.png", + "baeldung-bucket2", + "Document/picture.png" ); - + //deleting an object - awsService.deleteObject(bucketName, "Document/hello.txt"); + s3Service.deleteObject(AWS_BUCKET, "Document/hello.txt"); //deleting multiple objects - String objkeyArr[] = { - "Document/hello2.txt", - "Document/picture.png" - }; - - DeleteObjectsRequest delObjReq = new DeleteObjectsRequest("baeldung-bucket") - .withKeys(objkeyArr); - awsService.deleteObjects(delObjReq); + List objKeyList = List.of("Document/hello2.txt", "Document/picture.png"); + s3Service.deleteObjects(AWS_BUCKET, objKeyList); } } diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java new file mode 100644 index 0000000000..f4f768d1b4 --- /dev/null +++ b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java @@ -0,0 +1,323 @@ +package com.baeldung.s3; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.Bucket; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListBucketsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +class S3Service { + + private S3Client s3Client; + + public S3Service(Region awsRegion) { + init(awsRegion); + } + + public S3Service(S3Client s3Client) { + this.s3Client = s3Client; + } + + public void init(Region awsRegion) { + this.s3Client = S3Client.builder() + .region(awsRegion) + .credentialsProvider(ProfileCredentialsProvider.create("default")) + .build(); + } + + //is bucket exist? + public boolean doesBucketExist(String bucketName) { + HeadBucketRequest headBucketRequest = HeadBucketRequest.builder() + .bucket(bucketName) + .build(); + + try { + s3Client.headBucket(headBucketRequest); + return true; + } catch (NoSuchBucketException e) { + return false; + } + } + + //create a bucket + public void createBucket(String bucketName) { + CreateBucketRequest bucketRequest = CreateBucketRequest.builder() + .bucket(bucketName) + .build(); + + s3Client.createBucket(bucketRequest); + } + + //delete a bucket + public void deleteBucket(String bucketName) { + try { + DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder() + .bucket(bucketName) + .build(); + + s3Client.deleteBucket(deleteBucketRequest); + System.out.println("Successfully deleted bucket : " + bucketName); + } catch (S3Exception e) { + System.err.println(e.getMessage()); + System.exit(1); + } + } + + //uploading object + public PutObjectResponse putObject(String bucketName, String key, java.io.File file) { + PutObjectRequest request = PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + return s3Client.putObject(request, Path.of(file.toURI()) ); + } + + //listing objects + public void listObjects(String bucketName) { + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .build(); + ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request); + + for(S3Object os : listObjectsV2Response.contents()) { + System.out.println(os.key()); + } + } + + //get an object + public void getObject(String bucketName, String objectKey) { + try { + GetObjectRequest objectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build(); + + ResponseBytes responseResponseBytes = s3Client.getObjectAsBytes(objectRequest); + + byte[] data = responseResponseBytes.asByteArray(); + + // Write the data to a local file. + java.io.File myFile = new java.io.File("/Users/user/Desktop/hello.txt" ); + OutputStream os = new FileOutputStream(myFile); + os.write(data); + System.out.println("Successfully obtained bytes from an S3 object"); + os.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } catch (S3Exception e) { + System.err.println(e.awsErrorDetails().errorMessage()); + System.exit(1); + } + } + + //copying an object + public CopyObjectResponse copyObject( + String sourceBucketName, + String sourceKey, + String destinationBucketName, + String destinationKey + ) { + CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder() + .sourceBucket(sourceBucketName) + .sourceKey(sourceKey) + .destinationBucket(destinationBucketName) + .destinationKey(destinationKey) + .build(); + + return s3Client.copyObject(copyObjectRequest); + } + + //deleting an object + public void deleteObject(String bucketName, String objectKey) { + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build(); + + s3Client.deleteObject(deleteObjectRequest); + } + + //deleting multiple Objects + public void deleteObjects(String bucketName, List keys ) { + + ArrayList toDelete = new ArrayList<>(); + for(String objKey : keys) { + toDelete.add(ObjectIdentifier.builder() + .key(objKey) + .build()); + } + + DeleteObjectsRequest deleteObjectRequest = DeleteObjectsRequest.builder() + .bucket(bucketName) + .delete(Delete.builder() + .objects(toDelete).build()) + .build(); + + s3Client.deleteObjects(deleteObjectRequest); + } + + public boolean doesObjectExistByListObjects(String bucketName, String key) { + + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .build(); + ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request); + + return listObjectsV2Response.contents() + .stream() + .filter(s3ObjectSummary -> s3ObjectSummary.getValueForField("key", String.class) + .equals(key)) + .findFirst() + .isPresent(); + } + + public boolean doesObjectExistByDefaultMethod(String bucket, String key) { + try { + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder() + .bucket(bucket) + .key(key) + .build(); + + s3Client.headObject(headObjectRequest); + + System.out.println("Object exists"); + return true; + } catch (S3Exception e) { + if (e.statusCode() == 404) { + System.out.println("Object does not exist"); + return false; + } else { + throw e; + } + } + } + + public void listBuckets() { + // List all buckets + ListBucketsResponse listBucketsResponse = s3Client.listBuckets(); + + // Display the bucket names + List buckets = listBucketsResponse.buckets(); + System.out.println("Buckets:"); + for (Bucket bucket : buckets) { + System.out.println(bucket.name()); + } + } + + public void listObjectsInBucket(String bucketName) { + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .build(); + ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request); + + List contents = listObjectsV2Response.contents(); + + System.out.println("Number of objects in the bucket: " + contents.stream().count()); + contents.stream().forEach(System.out::println); + } + + public void listAllObjectsInBucket(String bucketName) { + String nextContinuationToken = null; + long totalObjects = 0; + + do { + ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder() + .bucket(bucketName) + .continuationToken(nextContinuationToken); + + ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build()); + nextContinuationToken = response.nextContinuationToken(); + + totalObjects += response.contents().stream() + .peek(System.out::println) + .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); + } while (nextContinuationToken != null); + System.out.println("Number of objects in the bucket: " + totalObjects); + } + + public void listAllObjectsInBucketPaginated(String bucketName, int pageSize) { + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(pageSize) // Set the maxKeys parameter to control the page size + .build(); + + ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request); + long totalObjects = 0; + + for (ListObjectsV2Response page : listObjectsV2Iterable) { + long retrievedPageSize = page.contents().stream() + .peek(System.out::println) + .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); + totalObjects += retrievedPageSize; + System.out.println("Page size: " + retrievedPageSize); + } + System.out.println("Total objects in the bucket: " + totalObjects); + } + + public void listAllObjectsInBucketPaginatedWithPrefix(String bucketName, int pageSize, String prefix) { + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(pageSize) // Set the maxKeys parameter to control the page size + .prefix(prefix) + .build(); + + ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request); + long totalObjects = 0; + + for (ListObjectsV2Response page : listObjectsV2Iterable) { + long retrievedPageSize = page.contents().stream() + .peek(System.out::println) + .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); + totalObjects += retrievedPageSize; + System.out.println("Page size: " + retrievedPageSize); + } + System.out.println("Total objects in the bucket: " + totalObjects); + } + + public void putObjects(String bucketName, List files) { + try { + files.stream().forEach(file -> { + s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(file.getName()).build(), + RequestBody.fromByteBuffer(file.getContent())); + System.out.println("Uploaded file: " + file.getName()); + }); + } catch (S3Exception e) { + System.err.println("Upload failed"); + e.printStackTrace(); + } + } + public void cleanup() { + this.s3Client.close(); + } + +} \ No newline at end of file diff --git a/aws-modules/aws-s3/src/main/resources/logback.xml b/aws-modules/aws-s3/src/main/resources/logback.xml index 7d900d8ea8..88eee7b977 100644 --- a/aws-modules/aws-s3/src/main/resources/logback.xml +++ b/aws-modules/aws-s3/src/main/resources/logback.xml @@ -10,4 +10,7 @@ + + + \ No newline at end of file diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/jets3t/JetS3tLiveTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/jets3t/JetS3tLiveTest.java index cf0b203e10..cad7da6cc9 100644 --- a/aws-modules/aws-s3/src/test/java/com/baeldung/jets3t/JetS3tLiveTest.java +++ b/aws-modules/aws-s3/src/test/java/com/baeldung/jets3t/JetS3tLiveTest.java @@ -1,5 +1,13 @@ package com.baeldung.jets3t; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -10,8 +18,8 @@ import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; import org.jets3t.service.model.StorageObject; import org.jets3t.service.security.AWSCredentials; -import org.junit.BeforeClass; -import org.junit.Test; + + import java.io.*; import java.nio.file.Files; @@ -19,14 +27,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.stream.Collectors; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static junit.framework.TestCase.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; + + + public class JetS3tLiveTest { - private Log log = LogFactory.getLog(JetS3tLiveTest.class); + private final Log log = LogFactory.getLog(JetS3tLiveTest.class); private static final String BucketName = "baeldung-barfoo"; private static final String TestString = "test string"; @@ -35,7 +42,7 @@ public class JetS3tLiveTest { private static S3Service s3Service; - @BeforeClass + @BeforeAll public static void connectS3() throws Exception { // Replace with your keys @@ -50,7 +57,7 @@ public class JetS3tLiveTest { } @Test - public void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception { + void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception { // List buckets, get a count S3Bucket[] myBuckets = s3Service.listAllBuckets(); @@ -89,7 +96,7 @@ public class JetS3tLiveTest { } @Test - public void givenString_Uploaded_StringInfoIsAvailable() throws Exception { + void givenString_Uploaded_StringInfoIsAvailable() throws Exception { // Create a bucket S3Bucket bucket = createBucket(); @@ -120,7 +127,7 @@ public class JetS3tLiveTest { } @Test - public void givenStringUploaded_StringIsDownloaded() throws Exception { + void givenStringUploaded_StringIsDownloaded() throws Exception { // Get a bucket S3Bucket bucket = createBucket(); @@ -135,7 +142,7 @@ public class JetS3tLiveTest { String downloadedString = new BufferedReader(new InputStreamReader(stringObject.getDataInputStream())).lines().collect(Collectors.joining("\n")); // Verify - assertTrue(TestString.equals(downloadedString)); + assertEquals(TestString, downloadedString); // Clean up for next test @@ -144,7 +151,7 @@ public class JetS3tLiveTest { } @Test - public void givenBinaryFileUploaded_FileIsDownloaded() throws Exception { + void givenBinaryFileUploaded_FileIsDownloaded() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -169,7 +176,7 @@ public class JetS3tLiveTest { // Get hashes and compare String origMD5 = getFileMD5("src/test/resources/test.jpg"); String newMD5 = getFileMD5("src/test/resources/newtest.jpg"); - assertTrue(origMD5.equals(newMD5)); + assertEquals(origMD5, newMD5); // Clean up deleteObject("test.jpg"); @@ -186,7 +193,7 @@ public class JetS3tLiveTest { @Test - public void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception { + void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -233,7 +240,7 @@ public class JetS3tLiveTest { } @Test - public void whenFileCopied_CopyIsSame() throws Exception { + void whenFileCopied_CopyIsSame() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -260,7 +267,7 @@ public class JetS3tLiveTest { // Get hashes and compare String origMD5 = getFileMD5("src/test/resources/test.jpg"); String newMD5 = getFileMD5("src/test/resources/testcopy.jpg"); - assertTrue(origMD5.equals(newMD5)); + assertEquals(origMD5, newMD5); // Clean up deleteObject("test.jpg"); @@ -271,7 +278,7 @@ public class JetS3tLiveTest { @Test - public void whenFileRenamed_NewNameIsSame() throws Exception { + void whenFileRenamed_NewNameIsSame() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -297,7 +304,7 @@ public class JetS3tLiveTest { // Get hashes and compare String origMD5 = getFileMD5("src/test/resources/test.jpg"); String newMD5 = getFileMD5("src/test/resources/spidey.jpg"); - assertTrue(origMD5.equals(newMD5)); + assertEquals(origMD5, newMD5); // Clean up deleteObject("test.jpg"); @@ -307,7 +314,7 @@ public class JetS3tLiveTest { } @Test - public void whenFileMoved_NewInstanceIsSame() throws Exception { + void whenFileMoved_NewInstanceIsSame() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -338,7 +345,7 @@ public class JetS3tLiveTest { // Get hashes and compare String origMD5 = getFileMD5("src/test/resources/test.jpg"); String newMD5 = getFileMD5("src/test/resources/spidey.jpg"); - assertTrue(origMD5.equals(newMD5)); + assertEquals(origMD5, newMD5); // Clean up deleteBucket(); diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ObjectManualTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ObjectManualTest.java deleted file mode 100644 index 02cc22367a..0000000000 --- a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ObjectManualTest.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.baeldung.s3; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.junit.Before; -import org.junit.Test; - -import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; - -/** - * Required defined environment variables AWS_ACCESS_KEY_ID & AWS_ACCESS_KEY to access S3. - * Required S3 bucket and key that exist. - */ - -public class AWSS3ObjectManualTest { - - private static final String BUCKET = "your-bucket"; - private static final String KEY_THAT_EXIST = "your-key-that-exist"; - private AWSS3ObjectUtils s3ObjectUtils; - - @Before - public void setUp() { - AmazonS3 client = AmazonS3ClientBuilder.standard() - .withRegion(Regions.DEFAULT_REGION) - .withCredentials(new EnvironmentVariableCredentialsProvider()) - .build(); - - s3ObjectUtils = new AWSS3ObjectUtils(client); - } - - @Test - public void whenVerifyIfObjectExistByDefaultMethod_thenCorrect() { - assertTrue(s3ObjectUtils.doesObjectExistByDefaultMethod(BUCKET, KEY_THAT_EXIST), "Key: " + KEY_THAT_EXIST + " doesn't exist"); - - } - - @Test - public void whenVerifyIfObjectExistByListObjects_thenCorrect() { - assertTrue(s3ObjectUtils.doesObjectExistByListObjects(BUCKET, KEY_THAT_EXIST), "Key: " + KEY_THAT_EXIST + " doesn't exist"); - } - - @Test - public void whenVerifyIfObjectExistByMetaData_thenCorrect() { - assertTrue(s3ObjectUtils.doesObjectExistByMetaData(BUCKET, KEY_THAT_EXIST), "Key: " + KEY_THAT_EXIST + " doesn't exist"); - } -} diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ServiceIntegrationTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ServiceIntegrationTest.java deleted file mode 100644 index d386704513..0000000000 --- a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ServiceIntegrationTest.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.baeldung.s3; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.File; - -import org.junit.Before; -import org.junit.Test; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.CopyObjectResult; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.DeleteObjectsResult; -import com.amazonaws.services.s3.model.PutObjectResult; - -public class AWSS3ServiceIntegrationTest { - - private static final String BUCKET_NAME = "bucket_name"; - private static final String KEY_NAME = "key_name"; - private static final String BUCKET_NAME2 = "bucket_name2"; - private static final String KEY_NAME2 = "key_name2"; - - private AmazonS3 s3; - private AWSS3Service service; - - @Before - public void setUp() { - s3 = mock(AmazonS3.class); - service = new AWSS3Service(s3); - } - - @Test - public void whenInitializingAWSS3Service_thenNotNull() { - assertThat(new AWSS3Service()).isNotNull(); - } - - @Test - public void whenVerifyingIfS3BucketExist_thenCorrect() { - service.doesBucketExist(BUCKET_NAME); - verify(s3).doesBucketExist(BUCKET_NAME); - } - - @Test - public void whenVerifyingCreationOfS3Bucket_thenCorrect() { - service.createBucket(BUCKET_NAME); - verify(s3).createBucket(BUCKET_NAME); - } - - @Test - public void whenVerifyingListBuckets_thenCorrect() { - service.listBuckets(); - verify(s3).listBuckets(); - } - - @Test - public void whenDeletingBucket_thenCorrect() { - service.deleteBucket(BUCKET_NAME); - verify(s3).deleteBucket(BUCKET_NAME); - } - - @Test - public void whenVerifyingPutObject_thenCorrect() { - File file = mock(File.class); - PutObjectResult result = mock(PutObjectResult.class); - when(s3.putObject(anyString(), anyString(), (File) any())).thenReturn(result); - - assertThat(service.putObject(BUCKET_NAME, KEY_NAME, file)).isEqualTo(result); - verify(s3).putObject(BUCKET_NAME, KEY_NAME, file); - } - - @Test - public void whenVerifyingListObjects_thenCorrect() { - service.listObjects(BUCKET_NAME); - verify(s3).listObjects(BUCKET_NAME); - } - - @Test - public void whenVerifyingGetObject_thenCorrect() { - service.getObject(BUCKET_NAME, KEY_NAME); - verify(s3).getObject(BUCKET_NAME, KEY_NAME); - } - - @Test - public void whenVerifyingCopyObject_thenCorrect() { - CopyObjectResult result = mock(CopyObjectResult.class); - when(s3.copyObject(anyString(), anyString(), anyString(), anyString())).thenReturn(result); - - assertThat(service.copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2)).isEqualTo(result); - verify(s3).copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2); - } - - @Test - public void whenVerifyingDeleteObject_thenCorrect() { - service.deleteObject(BUCKET_NAME, KEY_NAME); - verify(s3).deleteObject(BUCKET_NAME, KEY_NAME); - } - - @Test - public void whenVerifyingDeleteObjects_thenCorrect() { - DeleteObjectsRequest request = mock(DeleteObjectsRequest.class); - DeleteObjectsResult result = mock(DeleteObjectsResult.class); - when(s3.deleteObjects((DeleteObjectsRequest)any())).thenReturn(result); - - assertThat(service.deleteObjects(request)).isEqualTo(result); - verify(s3).deleteObjects(request); - } - -} diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/MultipartUploadLiveTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/MultipartUploadLiveTest.java deleted file mode 100644 index e2a34b1c39..0000000000 --- a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/MultipartUploadLiveTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.baeldung.s3; - -import com.amazonaws.event.ProgressListener; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.util.concurrent.Executors; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class MultipartUploadLiveTest { - - private static final String BUCKET_NAME = "bucket_name"; - private static final String KEY_NAME = "picture.jpg"; - - private AmazonS3 amazonS3; - private TransferManager tm; - private ProgressListener progressListener; - - @Before - public void setup() { - amazonS3 = mock(AmazonS3.class); - tm = TransferManagerBuilder - .standard() - .withS3Client(amazonS3) - .withMultipartUploadThreshold((long) (5 * 1024 * 1025)) - .withExecutorFactory(() -> Executors.newFixedThreadPool(5)) - .build(); - progressListener = - progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred()); - } - - @Test - public void whenUploadingFileWithTransferManager_thenVerifyUploadRequested() { - File file = mock(File.class); - PutObjectResult s3Result = mock(PutObjectResult.class); - - when(amazonS3.putObject(anyString(), anyString(), (File) any())).thenReturn(s3Result); - when(file.getName()).thenReturn(KEY_NAME); - - PutObjectRequest request = new PutObjectRequest(BUCKET_NAME, KEY_NAME, file); - request.setGeneralProgressListener(progressListener); - - Upload upload = tm.upload(request); - - assertThat(upload).isNotNull(); - verify(amazonS3).putObject(request); - } -} diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceIntegrationTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceIntegrationTest.java new file mode 100644 index 0000000000..bf24bcaa43 --- /dev/null +++ b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceIntegrationTest.java @@ -0,0 +1,137 @@ +package com.baeldung.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.util.Collections; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.ListBucketsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; + +class S3ServiceIntegrationTest { + + private static final String BUCKET_NAME = "bucket_name"; + private static final String KEY_NAME = "key_name"; + private static final String BUCKET_NAME2 = "bucket_name2"; + private static final String KEY_NAME2 = "key_name2"; + + @Mock + private S3Client s3Client; + + private S3Service s3Service; + + private final String AWS_BUCKET = "baeldung-tutorial-s3"; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + s3Service = new S3Service(s3Client); + } + + @AfterEach + public void cleanup() { + s3Service.cleanup(); + Mockito.verify(s3Client).close(); + } + + @Test + void whenInitializingAWSS3Service_thenNotNull() { + assertThat(new S3Service(s3Client)).isNotNull(); + } + + @Test + void whenVerifyingIfS3BucketExist_thenCorrect() { + HeadBucketRequest headBucketRequest = HeadBucketRequest.builder() + .bucket(BUCKET_NAME) + .build(); + + s3Service.doesBucketExist(BUCKET_NAME); + verify(s3Client).headBucket(headBucketRequest); + } + + @Test + void whenVerifyingCreationOfS3Bucket_thenCorrect() { + CreateBucketRequest bucketRequest = CreateBucketRequest.builder() + .bucket(BUCKET_NAME) + .build(); + + s3Service.createBucket(BUCKET_NAME); + verify(s3Client).createBucket(bucketRequest); + } + + @Test + void whenVerifyingListBuckets_thenCorrect() { + when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build()); + + s3Service.listBuckets(); + Mockito.verify(s3Client).listBuckets(); + } + + @Test + void whenDeletingBucket_thenCorrect() { + DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder() + .bucket(BUCKET_NAME) + .build(); + + s3Service.deleteBucket(BUCKET_NAME); + verify(s3Client).deleteBucket(deleteBucketRequest); + } + + + + @Test + void whenVerifyingListObjects_thenCorrect() { + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build(); + + when(s3Client.listObjectsV2(request)).thenReturn(response); + + s3Service.listObjects(AWS_BUCKET); + verify(s3Client).listObjectsV2(request); + } + + + @Test + void whenVerifyingCopyObject_thenCorrect() { + CopyObjectRequest request = CopyObjectRequest.builder() + .sourceBucket(BUCKET_NAME) + .sourceKey(KEY_NAME) + .destinationBucket(BUCKET_NAME2) + .destinationKey(KEY_NAME2) + .build(); + + CopyObjectResponse result = CopyObjectResponse.builder().build(); + + when(s3Client.copyObject(request)).thenReturn(result); + + assertThat(s3Service.copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2)).isEqualTo(result); + verify(s3Client).copyObject(request); + } + + @Test + void whenVerifyingDeleteObject_thenCorrect() { + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() + .bucket(BUCKET_NAME) + .key(KEY_NAME) + .build(); + + s3Service.deleteObject(BUCKET_NAME, KEY_NAME); + verify(s3Client).deleteObject(deleteObjectRequest); + } +} diff --git a/aws-modules/aws-s3-v2/src/test/java/com/baeldung/s3/listobjects/S3ServiceLiveTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceLiveTest.java similarity index 85% rename from aws-modules/aws-s3-v2/src/test/java/com/baeldung/s3/listobjects/S3ServiceLiveTest.java rename to aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceLiveTest.java index 40b30f8669..c46af81564 100644 --- a/aws-modules/aws-s3-v2/src/test/java/com/baeldung/s3/listobjects/S3ServiceLiveTest.java +++ b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceLiveTest.java @@ -1,109 +1,109 @@ -package com.baeldung.s3.listobjects; - -import com.baeldung.s3.listobjects.S3Service; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.ListBucketsResponse; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; -import software.amazon.awssdk.services.s3.model.S3Object; -import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; - -import java.util.Arrays; -import java.util.Collections; - -import static org.mockito.Mockito.when; - -class S3ServiceLiveTest { - - @Mock - private S3Client s3Client; - - private S3Service s3Service; - - private String AWS_BUCKET = "baeldung-tutorial-s3"; - - @BeforeEach - public void setup() { - MockitoAnnotations.openMocks(this); - s3Service = new S3Service(s3Client); - } - - @AfterEach - public void cleanup() { - s3Service.cleanup(); - Mockito.verify(s3Client).close(); - } - - @Test - public void givenBucketName_whenListObjectsInBucket_thenReturnList() { - ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build(); - ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build(); - - when(s3Client.listObjectsV2(request)).thenReturn(response); - - s3Service.listObjectsInBucket(AWS_BUCKET); - Mockito.verify(s3Client).listObjectsV2(request); - } - - @Test - public void givenBucketName_whenListAllObjectsInBucket_thenReturnList() { - ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build(); - ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build(); - - when(s3Client.listObjectsV2(request)).thenReturn(response); - - s3Service.listAllObjectsInBucket(AWS_BUCKET); - Mockito.verify(s3Client).listObjectsV2(request); - } - - @Test - public void givenBucketNameAndPageSize_whenListAllObjectsInBucketPaginated_thenReturnPaginatedList() { - int pageSize = 10; - ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).build(); - - ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class); - - S3Object s3Object1 = S3Object.builder().key("object1").build(); - S3Object s3Object2 = S3Object.builder().key("object2").build(); - ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build(); - - when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable); - when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator()); - - s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, pageSize); - Mockito.verify(s3Client).listObjectsV2Paginator(request); - } - - @Test - public void givenBucketNamePageSizeAndPrefix_whenListAllObjectsInBucketPaginatedWithPrefix_thenReturnPaginatedList() { - int pageSize = 1; - String prefix = "folder/"; - ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).prefix(prefix).build(); - - ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class); - - S3Object s3Object1 = S3Object.builder().key("folder/object1").build(); - S3Object s3Object2 = S3Object.builder().key("folder/object2").build(); - ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build(); - - when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable); - when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator()); - - s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, pageSize, prefix); - Mockito.verify(s3Client).listObjectsV2Paginator(request); - } - - @Test - public void whenListBuckets_thenReturnBucketList() { - when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build()); - - s3Service.listBuckets(); - Mockito.verify(s3Client).listBuckets(); - } -} +package com.baeldung.s3; + +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListBucketsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +import java.util.Arrays; +import java.util.Collections; + +class S3ServiceLiveTest { + + @Mock + private S3Client s3Client; + + private S3Service s3Service; + + private String AWS_BUCKET = "baeldung-tutorial-s3"; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + s3Service = new S3Service(s3Client); + } + + @AfterEach + public void cleanup() { + s3Service.cleanup(); + Mockito.verify(s3Client).close(); + } + + @Test + void givenBucketName_whenListObjectsInBucket_thenReturnList() { + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build(); + + when(s3Client.listObjectsV2(request)).thenReturn(response); + + s3Service.listObjectsInBucket(AWS_BUCKET); + Mockito.verify(s3Client).listObjectsV2(request); + } + + @Test + void givenBucketName_whenListAllObjectsInBucket_thenReturnList() { + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build(); + + when(s3Client.listObjectsV2(request)).thenReturn(response); + + s3Service.listAllObjectsInBucket(AWS_BUCKET); + Mockito.verify(s3Client).listObjectsV2(request); + } + + @Test + void givenBucketNameAndPageSize_whenListAllObjectsInBucketPaginated_thenReturnPaginatedList() { + int pageSize = 10; + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).build(); + + ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class); + + S3Object s3Object1 = S3Object.builder().key("object1").build(); + S3Object s3Object2 = S3Object.builder().key("object2").build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build(); + + when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable); + when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator()); + + s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, pageSize); + Mockito.verify(s3Client).listObjectsV2Paginator(request); + } + + @Test + void givenBucketNamePageSizeAndPrefix_whenListAllObjectsInBucketPaginatedWithPrefix_thenReturnPaginatedList() { + int pageSize = 1; + String prefix = "folder/"; + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).prefix(prefix).build(); + + ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class); + + S3Object s3Object1 = S3Object.builder().key("folder/object1").build(); + S3Object s3Object2 = S3Object.builder().key("folder/object2").build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build(); + + when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable); + when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator()); + + s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, pageSize, prefix); + Mockito.verify(s3Client).listObjectsV2Paginator(request); + } + + @Test + void whenListBuckets_thenReturnBucketList() { + when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build()); + + s3Service.listBuckets(); + Mockito.verify(s3Client).listBuckets(); + } +} diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceManualTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceManualTest.java new file mode 100644 index 0000000000..defe75e346 --- /dev/null +++ b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceManualTest.java @@ -0,0 +1,51 @@ +package com.baeldung.s3; + + +import static org.junit.jupiter.api.Assertions.assertTrue; + + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import software.amazon.awssdk.services.s3.S3Client; + +/** + * Required defined environment variables AWS_ACCESS_KEY_ID & AWS_ACCESS_KEY to access S3. + * Required S3 bucket and key that exist. + */ + +class S3ServiceManualTest { + + private static final String BUCKET_NAME = "bucket_name"; + private static final String KEY_NAME = "key_name"; + @Mock + private S3Client s3Client; + + private S3Service s3Service; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + s3Service = new S3Service(s3Client); + } + + @AfterEach + public void cleanup() { + s3Service.cleanup(); + Mockito.verify(s3Client).close(); + } + + @Test + void whenVerifyIfObjectExistByDefaultMethod_thenCorrect() { + assertTrue(s3Service.doesObjectExistByDefaultMethod(BUCKET_NAME, KEY_NAME), "Key: " + KEY_NAME + " doesn't exist"); + } + + @Test + void whenVerifyIfObjectExistByListObjects_thenCorrect() { + assertTrue(s3Service.doesObjectExistByListObjects(BUCKET_NAME, KEY_NAME), "Key: " + KEY_NAME + " doesn't exist"); + } +} diff --git a/aws-modules/pom.xml b/aws-modules/pom.xml index 77872e3f8f..02473815b5 100644 --- a/aws-modules/pom.xml +++ b/aws-modules/pom.xml @@ -19,7 +19,6 @@ aws-miscellaneous aws-reactive aws-s3 - aws-s3-v2