/** * The method makes sure it always grants execute permissions for an owner if the <code>file</code> passed is a * directory. The publisher needs it to publish it to the final directory and list files under this directory. */ private static OwnerAndPermission addExecutePermissionsIfRequired(FileStatus file, OwnerAndPermission ownerAndPermission) { if (ownerAndPermission.getFsPermission() == null) { return ownerAndPermission; } if (!file.isDir()) { return ownerAndPermission; } return new OwnerAndPermission(ownerAndPermission.getOwner(), ownerAndPermission.getGroup(), addExecutePermissionToOwner(ownerAndPermission.getFsPermission())); }
/** * Read a {@link org.apache.gobblin.data.management.copy.OwnerAndPermission} from a {@link java.io.DataInput}. * @throws IOException */ public static OwnerAndPermission read(DataInput input) throws IOException { OwnerAndPermission oap = new OwnerAndPermission(); oap.readFields(input); return oap; } }
@Override public String explain() { String owner = this.destinationOwnerAndPermission != null && this.destinationOwnerAndPermission.getOwner() != null ? this.destinationOwnerAndPermission.getOwner() : "preserve"; String group = this.destinationOwnerAndPermission != null && this.destinationOwnerAndPermission.getGroup() != null ? this.destinationOwnerAndPermission.getGroup() : "preserve"; String permissions = this.destinationOwnerAndPermission != null && this.destinationOwnerAndPermission.getFsPermission() != null ? this.destinationOwnerAndPermission.getFsPermission().toString() : "preserve"; return String.format("Copy file %s to %s with owner %s, group %s, permission %s.", this.origin.getPath(), this.destination, owner, group, permissions); } }
FsPermission readWrite = new FsPermission(FsAction.READ_WRITE, FsAction.READ_WRITE, FsAction.READ_WRITE); FsPermission dirReadWrite = new FsPermission(FsAction.ALL, FsAction.READ_WRITE, FsAction.READ_WRITE); OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), readWrite); List<OwnerAndPermission> ancestorOwnerAndPermissions = Lists.newArrayList(); ancestorOwnerAndPermissions.add(ownerAndPermission); Assert.assertTrue(this.fs.exists(expectedOutputPath)); fileStatus = this.fs.getFileStatus(expectedOutputPath); Assert.assertEquals(fileStatus.getOwner(), ownerAndPermission.getOwner()); Assert.assertEquals(fileStatus.getGroup(), ownerAndPermission.getGroup()); Assert.assertEquals(fileStatus.getPermission(), readWrite);
public CopyEntity createCopyableFile(int fileNumber) throws IOException { Path originPath = new Path(ORIGIN_PATH, fileNumber + ".file"); FileStatus origin = new FileStatus(0, false, 0, 0, 0, originPath); Path targetPath = new Path(TARGET_PATH, fileNumber + ".file"); return new CopyableFile(origin, targetPath, new OwnerAndPermission(null, null, null), Lists.<OwnerAndPermission>newArrayList(), null, PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps .<String, String>newHashMap(), ""); }
/** * Sets the {@link FsPermission}, owner, group for the path passed. It will not throw exceptions, if operations * cannot be executed, will warn and continue. */ private void safeSetPathPermission(Path path, OwnerAndPermission ownerAndPermission) { try { if (ownerAndPermission.getFsPermission() != null) { this.fs.setPermission(path, ownerAndPermission.getFsPermission()); } } catch (IOException ioe) { log.warn("Failed to set permission for directory " + path, ioe); } String owner = Strings.isNullOrEmpty(ownerAndPermission.getOwner()) ? null : ownerAndPermission.getOwner(); String group = Strings.isNullOrEmpty(ownerAndPermission.getGroup()) ? null : ownerAndPermission.getGroup(); try { if (owner != null || group != null) { this.fs.setOwner(path, owner, group); } } catch (IOException ioe) { log.warn("Failed to set owner and/or group for path " + path, ioe); } }
@Test public void testSerializeDeserialzeNulls() throws Exception { CopyableFile copyableFile = new CopyableFile(null, null, new OwnerAndPermission("owner", "group", FsPermission.getDefault()), Lists.newArrayList(new OwnerAndPermission(null, "group2", FsPermission .getDefault())), "checksum".getBytes(), PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps.<String, String>newHashMap(), ""); String serialized = CopyEntity.serialize(copyableFile); CopyEntity deserialized = CopyEntity.deserialize(serialized); Assert.assertEquals(deserialized, copyableFile); }
private void ensureDirectoryExists(FileSystem fs, Path path, Iterator<OwnerAndPermission> ownerAndPermissionIterator) throws IOException { if (fs.exists(path)) { return; } if (ownerAndPermissionIterator.hasNext()) { OwnerAndPermission ownerAndPermission = ownerAndPermissionIterator.next(); if (path.getParent() != null) { ensureDirectoryExists(fs, path.getParent(), ownerAndPermissionIterator); } if (!fs.mkdirs(path)) { // fs.mkdirs returns false if path already existed. Do not overwrite permissions return; } if (ownerAndPermission.getFsPermission() != null) { log.debug("Applying permissions %s to path %s.", ownerAndPermission.getFsPermission(), path); fs.setPermission(path, addExecutePermissionToOwner(ownerAndPermission.getFsPermission())); } String group = ownerAndPermission.getGroup(); String owner = ownerAndPermission.getOwner(); if (group != null || owner != null) { log.debug("Applying owner %s and group %s to path %s.", owner, group, path); fs.setOwner(path, owner, group); } } else { fs.mkdirs(path); } }
long originTimestamp = 23091986l; long upstreamTimestamp = 23091986l; OwnerAndPermission ownerAndPermission = new OwnerAndPermission("gobblin", "gobblin-dev", origin.getPermission()); Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getGroup(), ownerAndPermission.getGroup()); Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getOwner(), ownerAndPermission.getOwner()); Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getFsPermission(), ownerAndPermission.getFsPermission());
/** * Computes the correct {@link OwnerAndPermission} obtained from replicating source owner and permissions and applying * the {@link PreserveAttributes} rules in copyConfiguration. * @throws IOException */ public static OwnerAndPermission resolveReplicatedOwnerAndPermission(FileSystem fs, Path path, CopyConfiguration copyConfiguration) throws IOException { PreserveAttributes preserve = copyConfiguration.getPreserve(); Optional<FileStatus> originFileStatus = copyConfiguration.getCopyContext().getFileStatus(fs, path); if (!originFileStatus.isPresent()) { throw new IOException(String.format("Origin path %s does not exist.", path)); } String group = null; if (copyConfiguration.getTargetGroup().isPresent()) { group = copyConfiguration.getTargetGroup().get(); } else if (preserve.preserve(Option.GROUP)) { group = originFileStatus.get().getGroup(); } return new OwnerAndPermission(preserve.preserve(Option.OWNER) ? originFileStatus.get().getOwner() : null, group, preserve.preserve(Option.PERMISSION) ? originFileStatus.get().getPermission() : null); }
/** * Read a {@link org.apache.gobblin.data.management.copy.OwnerAndPermission} from a {@link java.io.DataInput}. * @throws IOException */ public static OwnerAndPermission read(DataInput input) throws IOException { OwnerAndPermission oap = new OwnerAndPermission(); oap.readFields(input); return oap; } }
Assert.assertEquals(ownerAndPermission.getOwner(), null); Assert.assertEquals(ownerAndPermission.getGroup(), null); Assert.assertEquals(ownerAndPermission.getFsPermission(), null); Assert.assertEquals(ownerAndPermission.getOwner(), null); Assert.assertEquals(ownerAndPermission.getGroup(), "target"); Assert.assertEquals(ownerAndPermission.getFsPermission(), null); Assert.assertEquals(ownerAndPermission.getOwner(), "owner"); Assert.assertEquals(ownerAndPermission.getGroup(), "target"); Assert.assertEquals(ownerAndPermission.getFsPermission(), null); Assert.assertEquals(ownerAndPermission.getOwner(), "owner"); Assert.assertEquals(ownerAndPermission.getGroup(), "group"); Assert.assertEquals(ownerAndPermission.getFsPermission(), null); Assert.assertEquals(ownerAndPermission.getOwner(), "owner"); Assert.assertEquals(ownerAndPermission.getGroup(), "group"); Assert.assertEquals(ownerAndPermission.getFsPermission(), FsPermission.getDefault());
/** * The method makes sure it always grants execute permissions for an owner if the <code>file</code> passed is a * directory. The publisher needs it to publish it to the final directory and list files under this directory. */ private static OwnerAndPermission addExecutePermissionsIfRequired(FileStatus file, OwnerAndPermission ownerAndPermission) { if (ownerAndPermission.getFsPermission() == null) { return ownerAndPermission; } if (!file.isDir()) { return ownerAndPermission; } return new OwnerAndPermission(ownerAndPermission.getOwner(), ownerAndPermission.getGroup(), addExecutePermissionToOwner(ownerAndPermission.getFsPermission())); }
private static CopyableFile createCopyableFile(String path, String fileSet) { return new CopyableFile(new FileStatus(0, false, 0, 0, 0, new Path(path)), new Path(path), new OwnerAndPermission("owner", "group", FsPermission.getDefault()), null, null, PreserveAttributes.fromMnemonicString(""), fileSet, 0, 0, Maps.<String, String>newHashMap(), ""); }
Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getGroup(), origin.getGroup()); Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getOwner(), origin.getOwner()); Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getFsPermission(), origin.getPermission());
@Test public void testSerializeDeserialze() throws Exception { CopyableFile copyableFile = new CopyableFile(new FileStatus(10, false, 12, 100, 12345, new Path("/path")), new Path("/destination"), new OwnerAndPermission("owner", "group", FsPermission.getDefault()), Lists.newArrayList(new OwnerAndPermission("owner2", "group2", FsPermission.getDefault())), "checksum".getBytes(), PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps .<String, String>newHashMap(), ""); DatasetDescriptor dataset = new DatasetDescriptor("hive", "db.table"); PartitionDescriptor descriptor = new PartitionDescriptor("datepartition=2018/09/05", dataset); copyableFile.setDestinationData(descriptor); String s = CopyEntity.serialize(copyableFile); CopyEntity de = CopyEntity.deserialize(s); Assert.assertEquals(de, copyableFile); }
@Override public String explain() { String owner = this.destinationOwnerAndPermission != null && this.destinationOwnerAndPermission.getOwner() != null ? this.destinationOwnerAndPermission.getOwner() : "preserve"; String group = this.destinationOwnerAndPermission != null && this.destinationOwnerAndPermission.getGroup() != null ? this.destinationOwnerAndPermission.getGroup() : "preserve"; String permissions = this.destinationOwnerAndPermission != null && this.destinationOwnerAndPermission.getFsPermission() != null ? this.destinationOwnerAndPermission.getFsPermission().toString() : "preserve"; return String.format("Copy file %s to %s with owner %s, group %s, permission %s.", this.origin.getPath(), this.destination, owner, group, permissions); } }
this.destinationOwnerAndPermission = new OwnerAndPermission(owner, group, permission);
/** * Sets the {@link FsPermission}, owner, group for the path passed. It will not throw exceptions, if operations * cannot be executed, will warn and continue. */ private void safeSetPathPermission(Path path, OwnerAndPermission ownerAndPermission) { try { if (ownerAndPermission.getFsPermission() != null) { this.fs.setPermission(path, ownerAndPermission.getFsPermission()); } } catch (IOException ioe) { log.warn("Failed to set permission for directory " + path, ioe); } String owner = Strings.isNullOrEmpty(ownerAndPermission.getOwner()) ? null : ownerAndPermission.getOwner(); String group = Strings.isNullOrEmpty(ownerAndPermission.getGroup()) ? null : ownerAndPermission.getGroup(); try { if (owner != null || group != null) { this.fs.setOwner(path, owner, group); } } catch (IOException ioe) { log.warn("Failed to set owner and/or group for path " + path, ioe); } }
/** * Find the test compressed file <code><filePath/code> in classpath and read it as a {@link FileAwareInputStream} */ private FileAwareInputStream getCompressedInputStream(final String filePath, final String newFileName) throws Exception { UnGzipConverter converter = new UnGzipConverter(); FileSystem fs = FileSystem.getLocal(new Configuration()); String fullPath = getClass().getClassLoader().getResource(filePath).getFile(); FileStatus status = fs.getFileStatus(testTempPath); OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); CopyableFile cf = CopyableFileUtils.getTestCopyableFile(filePath, new Path(testTempPath, newFileName).toString(), newFileName, ownerAndPermission); FileAwareInputStream fileAwareInputStream = FileAwareInputStream.builder().file(cf) .inputStream(fs.open(new Path(fullPath))).build(); Iterable<FileAwareInputStream> iterable = converter.convertRecord("outputSchema", fileAwareInputStream, new WorkUnitState()); return Iterables.getFirst(iterable, null); }