StorageFileEntity storageFileEntity = new StorageFileEntity(); storageFileEntities.add(storageFileEntity); storageFileEntity.setStorageUnit(storageUnitEntity); storageFileEntity.setPath(storageFile.getFilePath()); storageFileEntity.setFileSizeBytes(storageFile.getFileSizeBytes()); storageFileEntity.setRowCount(storageFile.getRowCount()); Assert.isTrue(storageFileEntity.getPath().startsWith(expectedS3KeyPrefix), String .format("Specified storage file path \"%s\" does not match the expected S3 key prefix \"%s\".", storageFileEntity.getPath(), expectedS3KeyPrefix)); Assert.isTrue(storageFileEntity.getPath().startsWith(directoryPath), String .format("Storage file path \"%s\" does not match the storage directory path \"%s\".", storageFileEntity.getPath(), directoryPath));
/** * Creates a storage file from the storage file entity. * * @param storageFileEntity the storage file entity * * @return the storage file */ public StorageFile createStorageFileFromEntity(StorageFileEntity storageFileEntity) { StorageFile storageFile = new StorageFile(); storageFile.setFilePath(storageFileEntity.getPath()); storageFile.setFileSizeBytes(storageFileEntity.getFileSizeBytes()); storageFile.setRowCount(storageFileEntity.getRowCount()); return storageFile; }
/** * Validates storage file entity against the actual S3 objects reported by S3. * * @param storageFileEntity the storage file to be validated * @param s3BucketName the S3 bucket name * @param actualS3Keys the map of storage file paths to storage files as reported by S3 * @param validateFileSize specifies whether file size validation is required or not */ public void validateStorageFileEntity(StorageFileEntity storageFileEntity, String s3BucketName, Map<String, StorageFile> actualS3Keys, boolean validateFileSize) { if (!actualS3Keys.containsKey(storageFileEntity.getPath())) { throw new ObjectNotFoundException( String.format("Previously registered storage file not found at s3://%s/%s location.", s3BucketName, storageFileEntity.getPath())); } else if (validateFileSize) { // Validate the file size. StorageFile actualS3StorageFile = actualS3Keys.get(storageFileEntity.getPath()); Assert.isTrue(storageFileEntity.getFileSizeBytes() != null, String.format("Previously registered storage file \"%s\" has no file size specified.", storageFileEntity.getPath())); Assert.isTrue(storageFileEntity.getFileSizeBytes().equals(actualS3StorageFile.getFileSizeBytes()), String .format("Previously registered storage file \"%s\" has file size of %d bytes that does not match file size of %d bytes reported by S3.", storageFileEntity.getPath(), storageFileEntity.getFileSizeBytes(), actualS3StorageFile.getFileSizeBytes())); } }
/** * Creates storage file entities from the list of storage files. * * @param storageUnitEntity the storage unit entity * @param storageFiles the list of storage files * * @return the list storage file entities */ public List<StorageFileEntity> createStorageFileEntitiesFromStorageFiles(StorageUnitEntity storageUnitEntity, List<StorageFile> storageFiles) { List<StorageFileEntity> storageFileEntities = new ArrayList<>(); for (StorageFile storageFile : storageFiles) { StorageFileEntity storageFileEntity = new StorageFileEntity(); storageFileEntities.add(storageFileEntity); storageFileEntity.setStorageUnit(storageUnitEntity); storageFileEntity.setPath(storageFile.getFilePath()); storageFileEntity.setFileSizeBytes(storageFile.getFileSizeBytes()); storageFileEntity.setRowCount(storageFile.getRowCount()); storageFileDao.saveAndRefresh(storageFileEntity); } return storageFileEntities; }
/** * Returns a map of file paths to the storage file entities build from the list of storage file entities. * * @param storageFileEntities the collection of storage file entities * * @return the map of file paths to storage file entities */ public Map<String, StorageFileEntity> getStorageFileEntitiesMap(Collection<StorageFileEntity> storageFileEntities) { Map<String, StorageFileEntity> result = new HashMap<>(); for (StorageFileEntity storageFileEntity : storageFileEntities) { result.put(storageFileEntity.getPath(), storageFileEntity); } return result; }
storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey).getStorageUnit().getBusinessObjectData(); completeUploadSingleParamsDto.setSourceFilePath(sourceStorageFileEntity.getPath()); .withS3KeyPrefix(completeUploadSingleParamsDto.getSourceFilePath()).withHttpProxyHost(awsParamsDto.getHttpProxyHost()) .withHttpProxyPort(awsParamsDto.getHttpProxyPort()).build(); s3Dao.validateS3File(s3FileTransferRequestParamsDto, sourceStorageFileEntity.getFileSizeBytes()); .setTargetFilePath(IterableUtils.get(IterableUtils.get(targetBusinessObjectDataEntity.getStorageUnits(), 0).getStorageFiles(), 0).getPath()); assertS3ObjectKeyDoesNotExist(completeUploadSingleParamsDto.getTargetBucketName(), completeUploadSingleParamsDto.getTargetFilePath());
@Test public void testCreateStorageFileEntitiesFromStorageFiles() { // Create a storage unit entity. StorageUnitEntity storageUnitEntity = new StorageUnitEntity(); // Create a list of storage files List<StorageFile> storageFiles = Lists.newArrayList(new StorageFile(FILE_NAME, FILE_SIZE, ROW_COUNT), new StorageFile(FILE_NAME_2, FILE_SIZE_2, ROW_COUNT_2)); // Call the method under test. List<StorageFileEntity> result = storageFileDaoHelper.createStorageFileEntitiesFromStorageFiles(storageUnitEntity, storageFiles); // Validate the results. assertThat("Result size not equal to two.", result.size(), is(2)); assertThat("File size not equal.", result.get(0).getFileSizeBytes(), is(FILE_SIZE)); assertThat("Row count not equal.", result.get(0).getRowCount(), is(ROW_COUNT)); assertThat("File size not equal.", result.get(1).getFileSizeBytes(), is(FILE_SIZE_2)); assertThat("Row count not equal.", result.get(1).getRowCount(), is(ROW_COUNT_2)); // Verify the external calls. verify(storageFileDao, times(2)).saveAndRefresh(any(StorageFileEntity.class)); verifyNoMoreInteractionsHelper(); }
StorageEntity storageEntity = storageFileEntity.getStorageUnit().getStorage(); businessObjectFormat.getBusinessObjectFormatVersion(), businessObjectDataEntity.getPartitionValue(), businessObjectDataHelper.getSubPartitionValues(businessObjectDataEntity), businessObjectDataEntity.getVersion(), storageUnitEntity.getStorageName(), storageFileEntity.getPath());
List<StorageFileEntity> storageFileEntities = Arrays.asList(new StorageFileEntity(), new StorageFileEntity());
/** * Validates storage file entity against the actual S3 objects reported by S3. * * @param storageFileEntity the storage file to be validated * @param s3BucketName the S3 bucket name * @param actualS3Keys the map of storage file paths to storage files as reported by S3 * @param validateFileSize specifies whether file size validation is required or not */ public void validateStorageFileEntity(StorageFileEntity storageFileEntity, String s3BucketName, Map<String, StorageFile> actualS3Keys, boolean validateFileSize) { if (!actualS3Keys.containsKey(storageFileEntity.getPath())) { throw new ObjectNotFoundException( String.format("Previously registered storage file not found at s3://%s/%s location.", s3BucketName, storageFileEntity.getPath())); } else if (validateFileSize) { // Validate the file size. StorageFile actualS3StorageFile = actualS3Keys.get(storageFileEntity.getPath()); Assert.isTrue(storageFileEntity.getFileSizeBytes() != null, String.format("Previously registered storage file \"%s\" has no file size specified.", storageFileEntity.getPath())); Assert.isTrue(storageFileEntity.getFileSizeBytes().equals(actualS3StorageFile.getFileSizeBytes()), String .format("Previously registered storage file \"%s\" has file size of %d bytes that does not match file size of %d bytes reported by S3.", storageFileEntity.getPath(), storageFileEntity.getFileSizeBytes(), actualS3StorageFile.getFileSizeBytes())); } }
/** * Returns a map of file paths to the storage file entities build from the list of storage file entities. * * @param storageFileEntities the collection of storage file entities * * @return the map of file paths to storage file entities */ public Map<String, StorageFileEntity> getStorageFileEntitiesMap(Collection<StorageFileEntity> storageFileEntities) { Map<String, StorageFileEntity> result = new HashMap<>(); for (StorageFileEntity storageFileEntity : storageFileEntities) { result.put(storageFileEntity.getPath(), storageFileEntity); } return result; }
storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey).getStorageUnit().getBusinessObjectData(); completeUploadSingleParamsDto.setSourceFilePath(sourceStorageFileEntity.getPath()); .withS3KeyPrefix(completeUploadSingleParamsDto.getSourceFilePath()).withHttpProxyHost(awsParamsDto.getHttpProxyHost()) .withHttpProxyPort(awsParamsDto.getHttpProxyPort()).build(); s3Dao.validateS3File(s3FileTransferRequestParamsDto, sourceStorageFileEntity.getFileSizeBytes()); .setTargetFilePath(IterableUtils.get(IterableUtils.get(targetBusinessObjectDataEntity.getStorageUnits(), 0).getStorageFiles(), 0).getPath()); assertS3ObjectKeyDoesNotExist(completeUploadSingleParamsDto.getTargetBucketName(), completeUploadSingleParamsDto.getTargetFilePath());
StorageEntity storageEntity = storageFileEntity.getStorageUnit().getStorage(); businessObjectFormat.getBusinessObjectFormatVersion(), businessObjectDataEntity.getPartitionValue(), businessObjectDataHelper.getSubPartitionValues(businessObjectDataEntity), businessObjectDataEntity.getVersion(), storageUnitEntity.getStorageName(), storageFileEntity.getPath());
List<StorageFileEntity> storageFileEntities = Arrays.asList(new StorageFileEntity());
/** * Creates storage file entities from the list of storage files. * * @param storageUnitEntity the storage unit entity * @param storageFiles the list of storage files * * @return the list storage file entities */ public List<StorageFileEntity> createStorageFileEntitiesFromStorageFiles(StorageUnitEntity storageUnitEntity, List<StorageFile> storageFiles) { List<StorageFileEntity> storageFileEntities = new ArrayList<>(); for (StorageFile storageFile : storageFiles) { StorageFileEntity storageFileEntity = new StorageFileEntity(); storageFileEntities.add(storageFileEntity); storageFileEntity.setStorageUnit(storageUnitEntity); storageFileEntity.setPath(storageFile.getFilePath()); storageFileEntity.setFileSizeBytes(storageFile.getFileSizeBytes()); storageFileEntity.setRowCount(storageFile.getRowCount()); storageFileDao.saveAndRefresh(storageFileEntity); } return storageFileEntities; }
/** * Creates a storage file from the storage file entity. * * @param storageFileEntity the storage file entity * * @return the storage file */ public StorageFile createStorageFileFromEntity(StorageFileEntity storageFileEntity) { StorageFile storageFile = new StorageFile(); storageFile.setFilePath(storageFileEntity.getPath()); storageFile.setFileSizeBytes(storageFileEntity.getFileSizeBytes()); storageFile.setRowCount(storageFileEntity.getRowCount()); return storageFile; }
metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Standard); metadata.setOngoingRestore(false); s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFileEntity.getPath(), new ByteArrayInputStream(new byte[storageFileEntity.getFileSizeBytes().intValue()]), metadata), NO_S3_CLIENT); Iterables.get(storageUnitEntity.getStorageFiles(), 0).getPath(), StorageClass.Standard.toString(), StorageClass.Glacier.toString(), S3_BUCKET_NAME), e.getMessage());
for (StorageFileEntity storageFile : storageFileEntities) storageFilePaths.add(storageFile.getPath());
List<StorageFileEntity> storageFileEntities = Arrays.asList(new StorageFileEntity());
/** * Creates and persists a new storage file entity. * * @param storageUnitEntity the storage unit entity * @param filePath the file path * @param fileSizeInBytes the size of the file in bytes * @param rowCount the count of rows in the file * * @return the newly created storage file entity. */ public StorageFileEntity createStorageFileEntity(StorageUnitEntity storageUnitEntity, String filePath, Long fileSizeInBytes, Long rowCount) { StorageFileEntity storageFileEntity = new StorageFileEntity(); storageFileEntity.setStorageUnit(storageUnitEntity); storageFileEntity.setPath(filePath); storageFileEntity.setFileSizeBytes(fileSizeInBytes); storageFileEntity.setRowCount(rowCount); return storageFileDao.saveAndRefresh(storageFileEntity); } }