private void distributeFiles() throws IOException { this.env.registerCachedFile(pythonTmpCachePath.getPath(), PythonConstants.FLINK_PYTHON_DC_ID); }
@Override public boolean filterPath(Path filePath) { if (getIncludeMatchers().isEmpty() && getExcludeMatchers().isEmpty()) { return false; } // compensate for the fact that Flink paths are slashed final String path = filePath.hasWindowsDrive() ? filePath.getPath().substring(1) : filePath.getPath(); final java.nio.file.Path nioPath = Paths.get(path); for (PathMatcher matcher : getIncludeMatchers()) { if (matcher.matches(nioPath)) { return shouldExclude(nioPath); } } return true; }
private static void addToZip(Path fileOrDirectory, FileSystem fs, Path rootDir, ZipOutputStream out) throws IOException { String relativePath = fileOrDirectory.getPath().replace(rootDir.getPath() + '/', ""); if (fs.getFileStatus(fileOrDirectory).isDir()) { out.putNextEntry(new ZipEntry(relativePath + '/')); for (FileStatus containedFile : fs.listStatus(fileOrDirectory)) { addToZip(containedFile.getPath(), fs, rootDir, out); } } else { ZipEntry entry = new ZipEntry(relativePath); out.putNextEntry(entry); try (FSDataInputStream in = fs.open(fileOrDirectory)) { IOUtils.copyBytes(in, out, false); } out.closeEntry(); } }
/** * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from * a local state. */ private void restoreInstanceDirectoryFromPath(Path source) throws IOException { FileSystem fileSystem = source.getFileSystem(); final FileStatus[] fileStatuses = fileSystem.listStatus(source); if (fileStatuses == null) { throw new IOException("Cannot list file statues. Directory " + source + " does not exist."); } for (FileStatus fileStatus : fileStatuses) { final Path filePath = fileStatus.getPath(); final String fileName = filePath.getName(); File restoreFile = new File(source.getPath(), fileName); File targetFile = new File(stateBackend.instanceRocksDBPath.getPath(), fileName); if (fileName.endsWith(SST_FILE_SUFFIX)) { // hardlink'ing the immutable sst-files. Files.createLink(targetFile.toPath(), restoreFile.toPath()); } else { // true copy for all other files. Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } } }
private void takeDBNativeCheckpoint(@Nonnull SnapshotDirectory outputDirectory) throws Exception { // create hard links of living files in the output path try ( ResourceGuard.Lease ignored = rocksDBResourceGuard.acquireResource(); Checkpoint checkpoint = Checkpoint.create(db)) { checkpoint.createCheckpoint(outputDirectory.getDirectory().getPath()); } catch (Exception ex) { try { outputDirectory.cleanup(); } catch (IOException cleanupEx) { ex = ExceptionUtils.firstOrSuppressed(cleanupEx, ex); } throw ex; } }
private RestoredDBInstance restoreDBInstanceFromStateHandle( IncrementalKeyedStateHandle restoreStateHandle, Path temporaryRestoreInstancePath) throws Exception { try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(stateBackend.numberOfTransferingThreads)) { rocksDBStateDownloader.transferAllStateDataToDirectory( restoreStateHandle, temporaryRestoreInstancePath, stateBackend.cancelStreamRegistry); } // read meta data List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = readMetaData(restoreStateHandle.getMetaStateHandle()); List<ColumnFamilyDescriptor> columnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots); List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(stateMetaInfoSnapshots.size() + 1); RocksDB restoreDb = stateBackend.openDB( temporaryRestoreInstancePath.getPath(), columnFamilyDescriptors, columnFamilyHandles); return new RestoredDBInstance(restoreDb, columnFamilyHandles, columnFamilyDescriptors, stateMetaInfoSnapshots); }
@Test public void testGenericRecord() throws IOException { final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath()); final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class); Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}"); outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE); outputFormat.setSchema(schema); output(outputFormat, schema); GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader); while (dataFileReader.hasNext()) { GenericRecord record = dataFileReader.next(); assertEquals(record.get("user_name").toString(), "testUser"); assertEquals(record.get("favorite_number"), 1); assertEquals(record.get("favorite_color").toString(), "blue"); } //cleanup FileSystem fs = FileSystem.getLocalFileSystem(); fs.delete(outputPath, false); }
@Test public void testMakeQualified() throws IOException { // make relative path qualified String path = "test/test"; Path p = new Path(path).makeQualified(FileSystem.getLocalFileSystem()); URI u = p.toUri(); assertEquals("file", u.getScheme()); assertEquals(null, u.getAuthority()); String q = new Path(FileSystem.getLocalFileSystem().getWorkingDirectory().getPath(), path).getPath(); assertEquals(q, u.getPath()); // make absolute path qualified path = "/test/test"; p = new Path(path).makeQualified(FileSystem.getLocalFileSystem()); u = p.toUri(); assertEquals("file", u.getScheme()); assertEquals(null, u.getAuthority()); assertEquals(path, u.getPath()); } }
Set<String> paths = new HashSet<>(filestatuses.length); for (FileStatus filestatus : filestatuses) { paths.add(filestatus.getPath().getPath());
Set<String> paths = new HashSet<>(filestatuses.length); for (FileStatus filestatus : filestatuses) { paths.add(filestatus.getPath().getPath());
org.apache.hadoop.fs.Path hPath = new org.apache.hadoop.fs.Path(fileSplit.getPath().getPath()); Reader orcReader = OrcFile.createReader(hPath, OrcFile.readerOptions(conf));
@Test public void testSerializationOnlyInProgress() throws IOException { final File testFolder = tempFolder.newFolder(); final FileSystem fs = FileSystem.get(testFolder.toURI()); final Path testBucket = new Path(testFolder.getPath(), "test"); final RecoverableWriter writer = fs.createRecoverableWriter(); final RecoverableFsDataOutputStream stream = writer.open(testBucket); stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8"))); final RecoverableWriter.ResumeRecoverable current = stream.persist(); final BucketState<String> bucketState = new BucketState<>( "test", testBucket, Long.MAX_VALUE, current, new HashMap<>()); final SimpleVersionedSerializer<BucketState<String>> serializer = new BucketStateSerializer<>( writer.getResumeRecoverableSerializer(), writer.getCommitRecoverableSerializer(), SimpleVersionedStringSerializer.INSTANCE ); final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); // to simulate that everything is over for file. stream.close(); final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); Assert.assertEquals(testBucket, recoveredState.getBucketPath()); FileStatus[] statuses = fs.listStatus(testBucket.getParent()); Assert.assertEquals(1L, statuses.length); Assert.assertTrue( statuses[0].getPath().getPath().startsWith( (new Path(testBucket.getParent(), ".test.inprogress")).toString()) ); }
@Override public void configure(ContainerSpecification container) throws IOException { if(krb5Conf != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(krb5Conf) .setDest(TARGET_PATH) .setCachable(true) .build()); container.getSystemProperties().setString(JAVA_SECURITY_KRB5_CONF, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(keytab != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(keytab) .setDest(TARGET_PATH) .setCachable(false) .build()); container.getDynamicConfiguration().setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(krb5Conf != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(krb5Conf) .setDest(TARGET_PATH) .setCachable(true) .build()); container.getSystemProperties().setString(JAVA_SECURITY_KRB5_CONF, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(keytab != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(keytab) .setDest(TARGET_PATH) .setCachable(false) .build()); container.getDynamicConfiguration().setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(krb5Conf != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(krb5Conf) .setDest(TARGET_PATH) .setCachable(true) .build()); container.getSystemProperties().setString(JAVA_SECURITY_KRB5_CONF, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(keytab != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(keytab) .setDest(TARGET_PATH) .setCachable(false) .build()); container.getDynamicConfiguration().setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(keytab != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(keytab) .setDest(TARGET_PATH) .setCachable(false) .build()); container.getDynamicConfiguration().setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(krb5Conf != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(krb5Conf) .setDest(TARGET_PATH) .setCachable(true) .build()); container.getSystemProperties().setString(JAVA_SECURITY_KRB5_CONF, TARGET_PATH.getPath()); } }