/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { initClient(); Collection<InetSocketAddress> addrs = new LinkedList<>(); try { ObjectListing list = keyPrefix == null ? s3.listObjects(bucketName) : s3.listObjects(bucketName, keyPrefix); while (true) { for (S3ObjectSummary sum : list.getObjectSummaries()) { InetSocketAddress addr = addr(sum); if (addr != null) addrs.add(addr); } if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else break; } } catch (AmazonClientException e) { throw new IgniteSpiException("Failed to list objects in the bucket: " + bucketName, e); } return addrs; }
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest); } else { listObjectsResponse = s3.listNextBatchOfObjects(listObjectsResponse); for ( S3ObjectSummary s : listObjectsResponse.getObjectSummaries() ) { log.debug("Skipping " + s.getKey() + " as it does not match filter."); continue; if ( leavesRoot(destinationDirectory, s.getKey()) ) { throw new RuntimeException("Cannot download key " + s.getKey() + ", its relative path resolves outside the parent directory."); if ( !s.getKey().equals(prefix) && !listObjectsResponse.getCommonPrefixes().contains(s.getKey() + DEFAULT_DELIMITER) ) { objectSummaries.add(s); totalSize += s.getSize(); commonPrefixes.addAll(listObjectsResponse.getCommonPrefixes()); } while ( listObjectsResponse.isTruncated() ); } while ( !commonPrefixes.isEmpty() );
if (objectListing.isTruncated() && objectListing.getNextMarker() == null) { if (!objectListing.getObjectSummaries().isEmpty()) { nextMarker = objectListing.getObjectSummaries() .get(objectListing.getObjectSummaries().size() - 1) .getKey(); } else if (!objectListing.getCommonPrefixes().isEmpty()) { nextMarker = objectListing.getCommonPrefixes() .get(objectListing.getCommonPrefixes().size() - 1); } else { log.error("S3 response indicates truncated results, " objectListing.setNextMarker(nextMarker); objectListing.setBucketName(getText()); if (log.isDebugEnabled()) { log.debug("Examining listing for bucket: " + objectListing.getBucketName()); objectListing.setPrefix(decodeIfSpecified (checkForEmptyString(getText()), shouldSDKDecodeResponse)); objectListing.setMarker(decodeIfSpecified (checkForEmptyString(getText()), shouldSDKDecodeResponse)); objectListing.setNextMarker(decodeIfSpecified (getText(), shouldSDKDecodeResponse));
ObjectListing listing = s3.listObjects( bucketName, prefix ); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); while (listing.isTruncated()) { listing = s3.listNextBatchOfObjects (listing); summaries.addAll (listing.getObjectSummaries()); }
/** * Creates a new {@link ListObjectsRequest} object using the previous object listing. * @return A new {@link ListObjectsRequest} object using the previous object listing. */ public ListObjectsRequest toListObjectsRequest() { return new ListObjectsRequest(previousObjectListing.getBucketName(), previousObjectListing.getPrefix(), previousObjectListing.getNextMarker(), previousObjectListing.getDelimiter(), Integer.valueOf(previousObjectListing.getMaxKeys())) .withEncodingType(previousObjectListing.getEncodingType()); }
@Override public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest) throws SdkClientException, AmazonServiceException { listNextBatchOfObjectsRequest = beforeClientExecution(listNextBatchOfObjectsRequest); rejectNull(listNextBatchOfObjectsRequest, "The request object parameter must be specified when listing the next batch of objects in a bucket"); ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing(); if (!previousObjectListing.isTruncated()) { ObjectListing emptyListing = new ObjectListing(); emptyListing.setBucketName(previousObjectListing.getBucketName()); emptyListing.setDelimiter(previousObjectListing.getDelimiter()); emptyListing.setMarker(previousObjectListing.getNextMarker()); emptyListing.setMaxKeys(previousObjectListing.getMaxKeys()); emptyListing.setPrefix(previousObjectListing.getPrefix()); emptyListing.setEncodingType(previousObjectListing.getEncodingType()); emptyListing.setTruncated(false); return emptyListing; } return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest()); }
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) { final List<String> commonPrefixes = Lists.newArrayList(); try { logger.info("Listing objects"); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withDelimiter("/") .withBucketName(bucketName).withPrefix( prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); commonPrefixes.addAll(objectListing.getCommonPrefixes()); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException e) { logger.error("Error Message: " + e.getMessage()); } return commonPrefixes; }
AmazonS3 s3 = new AmazonS3Client(); // provide credentials, if need be ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() .withBucketName("cdn.generalsentiment.com"); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(objectSummary.getKey()); }
ObjectListing listing = s3Client.listObjects(bucket); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); List<String> keyList = new ArrayList(); int startkeyNumber = 0; int numberOfIteration = 0; while (listing.isTruncated()) { listing = s3Client.listNextBatchOfObjects(listing); summaries.addAll(listing.getObjectSummaries()); String summaryKey = summary.getKey(); keyList.add(summaryKey);
@Override public List<DataRecord> getAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be null"); List<DataRecord> metadataList = new ArrayList<DataRecord>(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing prevObjectListing = s3service.listObjects(listObjectsRequest); for (final S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { metadataList.add(new S3DataRecord(this, s3service, bucket, new DataIdentifier(stripMetaKeyPrefix(s3ObjSumm.getKey())), s3ObjSumm.getLastModified().getTime(), s3ObjSumm.getSize(), true)); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } return metadataList; }
@Override protected void doStart() { AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey); AmazonS3 s3Client = new AmazonS3Client(myCredentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ChannelProcessor channelProcessor = getChannelProcessor(); for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) { String file = s3ObjectSummary.getKey(); LOGGER.info("Read the content of {}", file); GetObjectRequest objectRequest = new GetObjectRequest(bucket, file); S3Object objectPortion = s3Client.getObject(objectRequest); try { long startTime = System.currentTimeMillis(); processLines(channelProcessor, objectPortion.getObjectContent()); LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime); } catch (IOException e) { LOGGER.warn("Cannot process the {}, skipping", file, e); } } }
bucketName = BUCKET_NAME_PREFIX + bucketNameSuffix; s3 = cfg != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(cred); s3.setEndpoint(bucketEndpoint); if (!s3.doesBucketExist(bucketName)) { try { s3.createBucket(bucketName); for (S3ObjectSummary sum : list.getObjectSummaries()) { S3CheckpointData data = read(sum.getKey()); if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
boolean isExist(String key) { ObjectListing objects = amazonS3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(key)); for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { if (objectSummary.getKey().equals(key)) { return true; } } return false; }
@Override public void list(String keyPrefix, FileListing callback) { checkArgument(keyPrefix != null, "keyPrefix is null"); String errorMessage = "listing files on bucket " + bucket + " prefix " + keyPrefix; ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(bucket); req.setPrefix(keyPrefix); ObjectListing listing; do { try { listing = getWithRetry(errorMessage, () -> client.listObjects(req)); } catch (StorageFileNotFoundException ex) { throw Throwables.propagate(ex.getCause()); } callback.accept(Lists.transform( listing.getObjectSummaries(), (summary) -> StorageObjectSummary.builder() .key(summary.getKey()) .contentLength(summary.getSize()) .lastModified(summary.getLastModified().toInstant()) .build() )); req.setMarker(listing.getNextMarker()); } while (listing.isTruncated()); }
ObjectListing object_listing = s3.listObjects(bucket_name); while (true) { for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) { S3ObjectSummary summary = (S3ObjectSummary)iterator.next(); s3.deleteObject(bucket_name, summary.getKey()); if (object_listing.isTruncated()) { object_listing = s3.listNextBatchOfObjects(object_listing); } else { break;
private void getFiles(String path) throws IOException{ AmazonS3 s3 = new AmazonS3Client( new BasicAWSCredentials(apiKey, apiKeySecret)); ObjectListing objList = s3.listObjects(bucket, path); for (S3ObjectSummary summary:objList.getObjectSummaries()) { //ignore folders if(! summary.getKey().endsWith(FOLDER_SUFFIX)){ S3Object obj = s3.getObject( new GetObjectRequest(bucket, summary.getKey())); logger.info("retrieving " + summary.getKey()); FileOutputStream fout = new FileOutputStream(TEMP_FOLDER + summary.getKey().substring(path.length())); InputStream in = obj.getObjectContent(); byte[] buf = new byte[1024]; int len; while ((len = in.read(buf)) > 0){ fout.write(buf, 0, len); } in.close(); fout.close(); } } } protected void afterPropertiesSet(GenericFileEndpoint<File> endpoint) throws Exception {
AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix("m"); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated());
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); req.setDelimiter(getDelimiter()); req.setMarker(getMarker()); req.setEncodingType(getEncodingType()); req.setRequesterPays(isRequesterPays()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }