/** * Given a {@link FsPermission} objects, set a key, value pair in the given {@link State} for the writer to * use when creating files. This method should be used in conjunction with {@link #deserializeWriterFilePermissions(State, int, int)}. */ public static void serializeWriterFilePermissions(State state, int numBranches, int branchId, FsPermission fsPermissions) { serializeFsPermissions(state, ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PERMISSIONS, numBranches, branchId), fsPermissions); }
/** * Create a new task ID for the job with the given job ID. * * @param jobId job ID * @param sequence task sequence number * @return new task ID */ public static String newTaskId(String jobId, int sequence) { return Id.Task.create(Id.parse(jobId).get(Id.Parts.INSTANCE_NAME), sequence).toString(); }
/** * Create an ID for a new multi-task (corresponding to a {@link org.apache.gobblin.source.workunit.MultiWorkUnit}) * for the job with the given job ID. * * @param jobId job ID * @param sequence multi-task sequence number * @return new multi-task ID */ public static String newMultiTaskId(String jobId, int sequence) { return Id.MultiTask.create(Id.parse(jobId).get(Id.Parts.INSTANCE_NAME), sequence).toString(); }
private Kafka08ConsumerClient(Config config) { super(config); bufferSize = ConfigUtils.getInt(config, CONFIG_KAFKA_BUFFER_SIZE_BYTES, CONFIG_KAFKA_BUFFER_SIZE_BYTES_DEFAULT); clientName = ConfigUtils.getString(config, CONFIG_KAFKA_CLIENT_NAME, CONFIG_KAFKA_CLIENT_NAME_DEFAULT); fetchCorrelationId = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_REQUEST_CORRELATION_ID, CONFIG_KAFKA_FETCH_REQUEST_CORRELATION_ID_DEFAULT); fetchTopicRetries = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_TOPIC_NUM_TRIES, CONFIG_KAFKA_FETCH_TOPIC_NUM_TRIES_DEFAULT); fetchOffsetRetries = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_OFFSET_NUM_TRIES, CONFIG_KAFKA_FETCH_OFFSET_NUM_TRIES_DEFAULT); }
/** * Return string value at <code>path</code> if <code>config</code> has path. If not return an empty string * * @param config in which the path may be present * @param path key to look for in the config object * @return string value at <code>path</code> if <code>config</code> has path. If not return an empty string */ public static String emptyIfNotPresent(Config config, String path) { return getString(config, path, StringUtils.EMPTY); }
public SequentialBasedBatchAccumulator(Config config) { this(ConfigUtils.getLong(config, Batch.BATCH_SIZE, Batch.BATCH_SIZE_DEFAULT), ConfigUtils.getLong(config, Batch.BATCH_TTL, Batch.BATCH_TTL_DEFAULT), ConfigUtils.getLong(config, Batch.BATCH_QUEUE_CAPACITY, Batch.BATCH_QUEUE_CAPACITY_DEFAULT)); }
/** * Get a {@link FileSystem} object for the uri specified at {@link ConfigurationKeys#WRITER_FILE_SYSTEM_URI}. * @throws IOException */ public static FileSystem getWriterFileSystem(State state, int numBranches, int branchId) throws IOException { return HadoopUtils.getOptionallyThrottledFileSystem(WriterUtils.getWriterFS(state, numBranches, branchId), state); } }
public SubsetFilesCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, String idenifier, List<FileStatus> subFiles) { this.rootPath = PathUtils.getPathWithoutSchemeAndAuthority(rootPath); this.fs = fs; this.files = subFiles; this.identifier = idenifier; this.props = properties; }
/** * Delete the temporary {@link #schemaDir} */ public void cleanupTempSchemas() throws IOException { HadoopUtils.deleteIfExists(this.fs, this.schemaDir, true); }
/** * @return In default implementation we just return 'Healthy'. */ @Override public Future<String> getHealth() { return new CompletedFuture("Healthy", null); }
/** * Create a new {@link HiveJdbcConnector} using the specified Hive server version. * @param hiveServerVersion is the Hive server version to use * @return a HiveJdbcConnector with the specified hiveServerVersion * @throws SQLException */ @SuppressWarnings("resource") public static HiveJdbcConnector newEmbeddedConnector(int hiveServerVersion) throws SQLException { return new HiveJdbcConnector().withHiveServerVersion(hiveServerVersion).withHiveEmbeddedConnection(); }
@Override public Void call() throws Exception { SerializationUtils.serializeState(ParallelRunner.this.fs, outputFilePath, state); return null; } }), "Serialize state to " + outputFilePath));
/** * Method to list out all files, or directory if no file exists, under a specified path. */ public static List<FileStatus> listMostNestedPathRecursively(FileSystem fs, Path path) throws IOException { return listMostNestedPathRecursively(fs, path, NO_OP_PATH_FILTER); }
/** * Given a path to copy, list all files rooted at the given path to copy * * @param fs the file system of the path * @param path root path to copy * @param fileFilter a filter only applied to root * @param includeEmptyDirectories a control to include empty directories for copy */ public static List<FileStatus> listFilesToCopyAtPath(FileSystem fs, Path path, PathFilter fileFilter, boolean includeEmptyDirectories) throws IOException { return listFilesToCopyAtPath(fs, path, fileFilter, false, includeEmptyDirectories); }
protected double getDatasetPriority(String datasetName) { double priority = LOW_PRIORITY; if (DatasetFilterUtils.stringInPatterns(datasetName, this.highPriority)) { priority = HIGH_PRIORITY; } else if (DatasetFilterUtils.stringInPatterns(datasetName, this.normalPriority)) { priority = NORMAL_PRIORITY; } return priority; }
/** * @return an auto returnable wrapper around a {@link IMetaStoreClient}. * @throws IOException * Note: if you must acquire multiple locks, please use {@link #safeGetClients} instead, as this call may deadlock. */ public AutoReturnableObject<IMetaStoreClient> getClient() throws IOException { return new AutoReturnableObject<>(this.pool); }
/** * Create an instance of {@link Either} with value of type {@link S}. * @param left value of this instance. * @return an instance of {@link Left}. */ public static <S, T> Either<S, T> left(S left) { return new Left<>(left); }
@Override public FileSystem call() throws Exception { FileSystem fs = ProxiedFileSystemUtils.createProxiedFileSystemUsingToken(this.userNameToProxyAs, this.userNameToken, this.uri, this.configuration); if (this.referenceFS != null) { return decorateFilesystemFromReferenceFS(fs, this.referenceFS); } return fs; } }
/** * Serialize an object into a String. The object is first serialized into a byte array, * which is converted into a String using {@link BaseEncoding#base64()}. * * @param obj A {@link Serializable} object * @return A String representing the input object * @throws IOException if it fails to serialize the object */ public static <T extends Serializable> String serialize(T obj) throws IOException { return serialize(obj, DEFAULT_ENCODING); }
/** * Given a {@link FsPermission} objects, set a key, value pair in the given {@link State} for the writer to * use when creating files. This method should be used in conjunction with {@link #deserializeWriterDirPermissions(State, int, int)}. */ public static void serializeWriterDirPermissions(State state, int numBranches, int branchId, FsPermission fsPermissions) { serializeFsPermissions(state, ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DIR_PERMISSIONS, numBranches, branchId), fsPermissions); }