/** * Read a {@link org.apache.gobblin.data.management.copy.OwnerAndPermission} from a {@link java.io.DataInput}. * @throws IOException */ public static OwnerAndPermission read(DataInput input) throws IOException { OwnerAndPermission oap = new OwnerAndPermission(); oap.readFields(input); return oap; } }
public CopyEntity createCopyableFile(int fileNumber) throws IOException { Path originPath = new Path(ORIGIN_PATH, fileNumber + ".file"); FileStatus origin = new FileStatus(0, false, 0, 0, 0, originPath); Path targetPath = new Path(TARGET_PATH, fileNumber + ".file"); return new CopyableFile(origin, targetPath, new OwnerAndPermission(null, null, null), Lists.<OwnerAndPermission>newArrayList(), null, PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps .<String, String>newHashMap(), ""); }
@Test public void testSerializeDeserialzeNulls() throws Exception { CopyableFile copyableFile = new CopyableFile(null, null, new OwnerAndPermission("owner", "group", FsPermission.getDefault()), Lists.newArrayList(new OwnerAndPermission(null, "group2", FsPermission .getDefault())), "checksum".getBytes(), PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps.<String, String>newHashMap(), ""); String serialized = CopyEntity.serialize(copyableFile); CopyEntity deserialized = CopyEntity.deserialize(serialized); Assert.assertEquals(deserialized, copyableFile); }
/** * Computes the correct {@link OwnerAndPermission} obtained from replicating source owner and permissions and applying * the {@link PreserveAttributes} rules in copyConfiguration. * @throws IOException */ public static OwnerAndPermission resolveReplicatedOwnerAndPermission(FileSystem fs, Path path, CopyConfiguration copyConfiguration) throws IOException { PreserveAttributes preserve = copyConfiguration.getPreserve(); Optional<FileStatus> originFileStatus = copyConfiguration.getCopyContext().getFileStatus(fs, path); if (!originFileStatus.isPresent()) { throw new IOException(String.format("Origin path %s does not exist.", path)); } String group = null; if (copyConfiguration.getTargetGroup().isPresent()) { group = copyConfiguration.getTargetGroup().get(); } else if (preserve.preserve(Option.GROUP)) { group = originFileStatus.get().getGroup(); } return new OwnerAndPermission(preserve.preserve(Option.OWNER) ? originFileStatus.get().getOwner() : null, group, preserve.preserve(Option.PERMISSION) ? originFileStatus.get().getPermission() : null); }
private static CopyableFile createCopyableFile(String path, String fileSet) { return new CopyableFile(new FileStatus(0, false, 0, 0, 0, new Path(path)), new Path(path), new OwnerAndPermission("owner", "group", FsPermission.getDefault()), null, null, PreserveAttributes.fromMnemonicString(""), fileSet, 0, 0, Maps.<String, String>newHashMap(), ""); }
@Test public void testSerializeDeserialze() throws Exception { CopyableFile copyableFile = new CopyableFile(new FileStatus(10, false, 12, 100, 12345, new Path("/path")), new Path("/destination"), new OwnerAndPermission("owner", "group", FsPermission.getDefault()), Lists.newArrayList(new OwnerAndPermission("owner2", "group2", FsPermission.getDefault())), "checksum".getBytes(), PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps .<String, String>newHashMap(), ""); DatasetDescriptor dataset = new DatasetDescriptor("hive", "db.table"); PartitionDescriptor descriptor = new PartitionDescriptor("datepartition=2018/09/05", dataset); copyableFile.setDestinationData(descriptor); String s = CopyEntity.serialize(copyableFile); CopyEntity de = CopyEntity.deserialize(s); Assert.assertEquals(de, copyableFile); }
/** * The method makes sure it always grants execute permissions for an owner if the <code>file</code> passed is a * directory. The publisher needs it to publish it to the final directory and list files under this directory. */ private static OwnerAndPermission addExecutePermissionsIfRequired(FileStatus file, OwnerAndPermission ownerAndPermission) { if (ownerAndPermission.getFsPermission() == null) { return ownerAndPermission; } if (!file.isDir()) { return ownerAndPermission; } return new OwnerAndPermission(ownerAndPermission.getOwner(), ownerAndPermission.getGroup(), addExecutePermissionToOwner(ownerAndPermission.getFsPermission())); }
/** * Find the test compressed file <code><filePath/code> in classpath and read it as a {@link FileAwareInputStream} */ private FileAwareInputStream getCompressedInputStream(final String filePath, final String newFileName) throws Exception { UnGzipConverter converter = new UnGzipConverter(); FileSystem fs = FileSystem.getLocal(new Configuration()); String fullPath = getClass().getClassLoader().getResource(filePath).getFile(); FileStatus status = fs.getFileStatus(testTempPath); OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); CopyableFile cf = CopyableFileUtils.getTestCopyableFile(filePath, new Path(testTempPath, newFileName).toString(), newFileName, ownerAndPermission); FileAwareInputStream fileAwareInputStream = FileAwareInputStream.builder().file(cf) .inputStream(fs.open(new Path(fullPath))).build(); Iterable<FileAwareInputStream> iterable = converter.convertRecord("outputSchema", fileAwareInputStream, new WorkUnitState()); return Iterables.getFirst(iterable, null); }
this.destinationOwnerAndPermission = new OwnerAndPermission(owner, group, permission);
@Test public void testWrite() throws Exception { String streamString = "testContents"; FileStatus status = fs.getFileStatus(testTempPath); OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); CopyableFile cf = CopyableFileUtils.getTestCopyableFile(ownerAndPermission); CopyableDatasetMetadata metadata = new CopyableDatasetMetadata(new TestCopyableDataset(new Path("/source"))); WorkUnitState state = TestUtils.createTestWorkUnitState(); state.setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(testTempPath, "staging").toString()); state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(testTempPath, "output").toString()); state.setProp(ConfigurationKeys.WRITER_FILE_PATH, RandomStringUtils.randomAlphabetic(5)); CopySource.serializeCopyEntity(state, cf); CopySource.serializeCopyableDataset(state, metadata); FileAwareInputStreamDataWriter dataWriter = new FileAwareInputStreamDataWriter(state, 1, 0); FileAwareInputStream fileAwareInputStream = FileAwareInputStream.builder().file(cf) .inputStream(StreamUtils.convertStream(IOUtils.toInputStream(streamString))).build(); dataWriter.write(fileAwareInputStream); dataWriter.commit(); Path writtenFilePath = new Path(new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), cf.getDatasetAndPartition(metadata).identifier()), cf.getDestination()); Assert.assertEquals(IOUtils.toString(new FileInputStream(writtenFilePath.toString())), streamString); }
@Test public void testWriteWithEncryption() throws Exception { byte[] streamString = "testEncryptedContents".getBytes("UTF-8"); byte[] expectedContents = new byte[streamString.length]; for (int i = 0; i < streamString.length; i++) { expectedContents[i] = (byte)((streamString[i] + 1) % 256); } FileStatus status = fs.getFileStatus(testTempPath); OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); CopyableFile cf = CopyableFileUtils.getTestCopyableFile(ownerAndPermission); CopyableDatasetMetadata metadata = new CopyableDatasetMetadata(new TestCopyableDataset(new Path("/source"))); WorkUnitState state = TestUtils.createTestWorkUnitState(); state.setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(testTempPath, "staging").toString()); state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(testTempPath, "output").toString()); state.setProp(ConfigurationKeys.WRITER_FILE_PATH, RandomStringUtils.randomAlphabetic(5)); state.setProp("writer.encrypt." + EncryptionConfigParser.ENCRYPTION_ALGORITHM_KEY, "insecure_shift"); CopySource.serializeCopyEntity(state, cf); CopySource.serializeCopyableDataset(state, metadata); FileAwareInputStreamDataWriter dataWriter = new FileAwareInputStreamDataWriter(state, 1, 0); FileAwareInputStream fileAwareInputStream = FileAwareInputStream.builder().file(cf) .inputStream(StreamUtils.convertStream(new ByteArrayInputStream(streamString))).build(); dataWriter.write(fileAwareInputStream); dataWriter.commit(); Path writtenFilePath = new Path(new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), cf.getDatasetAndPartition(metadata).identifier()), cf.getDestination()); Assert.assertTrue(writtenFilePath.getName().endsWith("insecure_shift"), "Expected encryption name to be appended to destination"); Assert.assertEquals(IOUtils.toByteArray(new FileInputStream(writtenFilePath.toString())), expectedContents); }
FsPermission readWrite = new FsPermission(FsAction.READ_WRITE, FsAction.READ_WRITE, FsAction.READ_WRITE); FsPermission dirReadWrite = new FsPermission(FsAction.ALL, FsAction.READ_WRITE, FsAction.READ_WRITE); OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), readWrite); List<OwnerAndPermission> ancestorOwnerAndPermissions = Lists.newArrayList(); ancestorOwnerAndPermissions.add(ownerAndPermission);
new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); CopyableFile cf = CopyableFileUtils.getTestCopyableFile(ownerAndPermission);
new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); CopyableFile cf = CopyableFileUtils.getTestCopyableFile(ownerAndPermission);
new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); CopyableFile cf = CopyableFileUtils.getTestCopyableFile(ownerAndPermission);
long originTimestamp = 23091986l; long upstreamTimestamp = 23091986l; OwnerAndPermission ownerAndPermission = new OwnerAndPermission("gobblin", "gobblin-dev", origin.getPermission());
/** * Read a {@link org.apache.gobblin.data.management.copy.OwnerAndPermission} from a {@link java.io.DataInput}. * @throws IOException */ public static OwnerAndPermission read(DataInput input) throws IOException { OwnerAndPermission oap = new OwnerAndPermission(); oap.readFields(input); return oap; } }
/** * Computes the correct {@link OwnerAndPermission} obtained from replicating source owner and permissions and applying * the {@link PreserveAttributes} rules in copyConfiguration. * @throws IOException */ public static OwnerAndPermission resolveReplicatedOwnerAndPermission(FileSystem fs, Path path, CopyConfiguration copyConfiguration) throws IOException { PreserveAttributes preserve = copyConfiguration.getPreserve(); Optional<FileStatus> originFileStatus = copyConfiguration.getCopyContext().getFileStatus(fs, path); if (!originFileStatus.isPresent()) { throw new IOException(String.format("Origin path %s does not exist.", path)); } String group = null; if (copyConfiguration.getTargetGroup().isPresent()) { group = copyConfiguration.getTargetGroup().get(); } else if (preserve.preserve(Option.GROUP)) { group = originFileStatus.get().getGroup(); } return new OwnerAndPermission(preserve.preserve(Option.OWNER) ? originFileStatus.get().getOwner() : null, group, preserve.preserve(Option.PERMISSION) ? originFileStatus.get().getPermission() : null); }
/** * The method makes sure it always grants execute permissions for an owner if the <code>file</code> passed is a * directory. The publisher needs it to publish it to the final directory and list files under this directory. */ private static OwnerAndPermission addExecutePermissionsIfRequired(FileStatus file, OwnerAndPermission ownerAndPermission) { if (ownerAndPermission.getFsPermission() == null) { return ownerAndPermission; } if (!file.isDir()) { return ownerAndPermission; } return new OwnerAndPermission(ownerAndPermission.getOwner(), ownerAndPermission.getGroup(), addExecutePermissionToOwner(ownerAndPermission.getFsPermission())); }
this.destinationOwnerAndPermission = new OwnerAndPermission(owner, group, permission);