How to use

registerCachedFile

method

in

org.apache.flink.streaming.api.environment.StreamExecutionEnvironment

/**
 * Registers a file at the distributed cache under the given name. The file will be accessible
 * from any user-defined function in the (distributed) runtime under a local path. Files
 * may be local files (which will be distributed via BlobServer), or files in a distributed file system.
 * The runtime will copy the files temporarily to a local cache, if needed.
 *
 * <p>The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs via
 * {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and provides access
 * {@link org.apache.flink.api.common.cache.DistributedCache} via
 * {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
 *
 * @param filePath The path of the file, as a URI (e.g. "file:///some/path" or "hdfs://host:port/and/path")
 * @param name The name under which the file is registered.
 */
public void registerCachedFile(String filePath, String name) {
  registerCachedFile(filePath, name, false);
}

private void distributeFiles() throws IOException {
  this.env.registerCachedFile(pythonTmpCachePath.getPath(), PythonConstants.FLINK_PYTHON_DC_ID);
}

public static void main(String[] args) throws Exception {
  final ParameterTool params = ParameterTool.fromArgs(args);
  final Path inputFile = Paths.get(params.getRequired("inputFile"));
  final Path inputDir = Paths.get(params.getRequired("inputDir"));
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(1);
  env.registerCachedFile(inputFile.toString(), "test_data", false);
  env.registerCachedFile(inputDir.toString(), "test_dir", false);
  final Path containedFile;
  try (Stream<Path> files = Files.list(inputDir)) {
    containedFile = files.findAny().orElseThrow(() -> new RuntimeException("Input directory must not be empty."));
  }
  env.fromElements(1)
    .map(new TestMapFunction(
      inputFile.toAbsolutePath().toString(),
      Files.size(inputFile),
      inputDir.toAbsolutePath().toString(),
      containedFile.getFileName().toString()))
    .writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE);
  env.execute("Distributed Cache Via Blob Test Program");
}

@Test
public void testStreamingDistributedCache() throws Exception {
  String textPath = createTempFile("count.txt", DATA);
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.registerCachedFile(textPath, "cache_test");
  env.readTextFile(textPath).flatMap(new WordChecker());
  env.execute();
}

/**
 * Registers a file at the distributed cache under the given name. The file will be accessible
 * from any user-defined function in the (distributed) runtime under a local path. Files
 * may be local files (as long as all relevant workers have access to it), or files in a distributed file system.
 * The runtime will copy the files temporarily to a local cache, if needed.
 *
 * <p>The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs via
 * {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and provides access
 * {@link org.apache.flink.api.common.cache.DistributedCache} via
 * {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
 *
 * @param filePath The path of the file, as a URI (e.g. "file:///some/path" or "hdfs://host:port/and/path")
 * @param name The name under which the file is registered.
 */
public void registerCachedFile(String filePath, String name) {
  registerCachedFile(filePath, name, false);
}

/**
 * Registers a file at the distributed cache under the given name. The file will be accessible
 * from any user-defined function in the (distributed) runtime under a local path. Files
 * may be local files (which will be distributed via BlobServer), or files in a distributed file system.
 * The runtime will copy the files temporarily to a local cache, if needed.
 *
 * <p>The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs via
 * {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and provides access
 * {@link org.apache.flink.api.common.cache.DistributedCache} via
 * {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
 *
 * @param filePath The path of the file, as a URI (e.g. "file:///some/path" or "hdfs://host:port/and/path")
 * @param name The name under which the file is registered.
 */
public void registerCachedFile(String filePath, String name) {
  registerCachedFile(filePath, name, false);
}

/**
 * Registers a file at the distributed cache under the given name. The file will be accessible
 * from any user-defined function in the (distributed) runtime under a local path. Files
 * may be local files (as long as all relevant workers have access to it), or files in a distributed file system.
 * The runtime will copy the files temporarily to a local cache, if needed.
 *
 * <p>The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs via
 * {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and provides access
 * {@link org.apache.flink.api.common.cache.DistributedCache} via
 * {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
 *
 * @param filePath The path of the file, as a URI (e.g. "file:///some/path" or "hdfs://host:port/and/path")
 * @param name The name under which the file is registered.
 */
public void registerCachedFile(String filePath, String name) {
  registerCachedFile(filePath, name, false);
}

/**
 * Registers a file at the distributed cache under the given name. The file will be accessible
 * from any user-defined function in the (distributed) runtime under a local path. Files
 * may be local files (which will be distributed via BlobServer), or files in a distributed file system.
 * The runtime will copy the files temporarily to a local cache, if needed.
 *
 * <p>The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs via
 * {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and provides access
 * {@link org.apache.flink.api.common.cache.DistributedCache} via
 * {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
 *
 * @param filePath The path of the file, as a URI (e.g. "file:///some/path" or "hdfs://host:port/and/path")
 * @param name The name under which the file is registered.
 */
public void registerCachedFile(String filePath, String name) {
  registerCachedFile(filePath, name, false);
}

for(URL url : classPathSet){
  String classFileName = String.format(CLASS_FILE_NAME_FMT, i);
  env.registerCachedFile(url.getPath(),  classFileName, true);
  i++;