final FileSystem fs = FSUtils.getFs(conf); FSUtils.deleteHDFSMetadataFiles(fileStatuses, fs, numFilesToRetain, fakeDelete); } else { log.warn("Cannot prune any files, the path {} is not a directory", metadataPath);
private static Configuration getConfiguration(@NonNull final JsonHoodieIngestionCommandLineOptions cmd) { final Configuration conf = new Configuration(); try { final FileSystem fs = FSUtils.getFs(conf); final Path dataFeedConfFile = new Path(cmd.getConfFile()); log.info("Loading configuration from {}", dataFeedConfFile.toString()); conf.loadYamlStream(fs.open(dataFeedConfFile), Optional.absent()); } catch (IOException e) { final String errorMessage = String.format("Unable to find configuration for %s", cmd.getConfFile()); log.error(errorMessage); throw new JobRuntimeException(errorMessage, e); } return conf; }
private void pruneMetadataFiles() { try { final Path metadataPath = new Path(this.baseMetadataPath); if (this.fileSystem.exists(metadataPath)) { final FileStatus[] fileStatuses = fileSystem.listStatus(metadataPath); if (fileStatuses.length > 0) { FSUtils.deleteHDFSMetadataFiles(fileStatuses, this.fileSystem, DEFAULT_NUM_METADATA_FILES_TO_RETAIN, false); } } } catch (final IOException e) { final String errMsg = String.format("IOException occurred while pruning metadata files. Message: %s", e.getMessage()); log.warn(errMsg); } }
public JobManagerMetadataTracker(@NonNull final Configuration config) throws IOException { final Optional<String> sourceType = config.getProperty(MetadataConstants.JOBMANAGER_METADATA_STORAGE); if (!sourceType.isPresent()) { throw new MissingPropertyException("Source information for the JobManager Metadata Tracker is missing."); } if (sourceType.get().toUpperCase().equals(MetadataConstants.JOBMANAGER_METADATA_SOURCE_HDFS)) { final Optional<String> basePath = config.getProperty(MetadataConstants.JOBMANAGER_METADATA_HDFS_BASEPATH); if (!basePath.isPresent()) { throw new MissingPropertyException("Base Path for HDFS JobManager Metadata Tracker is missing."); } this.metadataManager = new HDFSMetadataManager(FSUtils.getFs(config), basePath.get(), new AtomicBoolean(true)); this.shouldSaveChanges = new AtomicBoolean(true); } }
Assert.assertEquals(7, fileStatuses.length); FSUtils.deleteHDFSMetadataFiles(fileStatuses, this.fileSystem, 4, false); FSUtils.deleteHDFSMetadataFiles(fileStatuses, this.fileSystem, 2, true); Assert.assertEquals(numExpectedRemaining, remaining.length);
/** * Ensure that hoodie dataset is present. */ protected void initDataset() { try { HoodieUtil.initHoodieDataset(FSUtils.getFs(this.hoodieConf.getConf()), this.hoodieConf); } catch (IOException e) { log.error("Error initializing hoodie dataset.", e); throw new JobRuntimeException("Could not initialize hoodie dataset", e); } }
public static void main(final String[] args) throws ParseException, IOException { final CommandLineParser parser = new GnuParser(); final Options options = getCLIOptions(); final CommandLine cmd; try { cmd = parser.parse(options, args); } catch (final ParseException e) { final String cmdLineSyntax = "java -cp [jar_name] com.uber.marmaray.tools.HDFSMetadataPrinter " + "-m [METADATA_FILE]"; final String header = "This tools prints out all the metadata contents of a HDFS metadata file."; final String footer = "For help, please contact the Hadoop Data Platform team"; CommandLineUtil.printHelp(options, cmdLineSyntax, header, footer); throw e; } final String metadataFilePath = cmd.getOptionValue(METADATA_FILE_OPTION); Preconditions.checkState(!Strings.isNullOrEmpty(metadataFilePath)); log.info("Printing contents of metadata file: " + metadataFilePath); final Configuration conf = new Configuration(); final FileSystem fs = FSUtils.getFs(conf); try (final InputStream is = new BufferedInputStream(fs.open(new Path(metadataFilePath)))) { try (final ObjectInputStream input = new ObjectInputStream(is)) { final Map<String, StringValue> metadataMap = HDFSMetadataManager.deserialize(input); metadataMap.entrySet() .stream() .forEach(entry -> log.info("Key: " + entry.getKey() + " Value: " + entry.getValue().getValue())); } } }
/** * It initializes hoodie dataset * @param fs {@link FileSystem} * @param hoodieConf {@link HoodieConfiguration} * @throws IOException */ public static void initHoodieDataset(@NonNull final FileSystem fs, @NonNull final HoodieConfiguration hoodieConf) throws IOException { final Path hoodieMetaFolder = new Path(hoodieConf.getBasePath(), HoodieTableMetaClient.METAFOLDER_NAME); final Path hoodiePropertiesFile = new Path(hoodieMetaFolder.toString(), HoodieTableConfig.HOODIE_PROPERTIES_FILE); if (!fs.exists(hoodiePropertiesFile)) { HoodieTableMetaClient .initializePathAsHoodieDataset(FSUtils.getFs(hoodieConf.getConf()), hoodieConf.getBasePath(), hoodieConf.getHoodieInitProperties()); } }
public static void initErrorTableDataset(@NonNull final Configuration conf, @NotEmpty final String errorTableName) throws IOException { final ErrorTableConfiguration errorTableConf = new ErrorTableConfiguration(conf); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(errorTableName) .withBasePath(errorTableConf.getDestPath().toString()) .withTableName(errorTableName) .enableMetrics(false) .build(); HoodieUtil.initHoodieDataset(FSUtils.getFs(conf), hoodieConf); }
@Before public void setupTest() throws IOException { this.fileSystem = FSUtils.getFs(new Configuration()); final AtomicBoolean condition = new AtomicBoolean(true); final String metadataPath = new Path(HDFSTestConstants.BASE_METADATA_PATH, JOB_NAME).toString(); this.metadataManager = new HDFSMetadataManager(this.fileSystem, metadataPath, condition); }
@Before public void setupTest() throws IOException { this.parquetDataPath = FileHelperUtil.getResourcePath(getClass(), new Path("testData", "testPartition").toString()); this.fs = FSUtils.getFs(new Configuration()); }
@Before public void initClass() throws IOException { final Configuration conf = new Configuration(); this.fileSystem = FSUtils.getFs(conf); }
@Before public void setupTest() throws IOException { /** * We explicitly don't call close() in a tearDownTest() method as the Hadoop FileSystem object is cached * so if multiple threads are accessing can affect others if one thread closes it. */ this.fileSystem = FSUtils.getFs(new com.uber.marmaray.common.configuration.Configuration()); }
@Before public void setupTest() throws IOException { this.fileSystem = FSUtils.getFs(new Configuration()); }
@Before public void setupTest() throws IOException { this.fs = FSUtils.getFs(new Configuration()); }
@Before public void setupTest() { this.spark = Optional.of(SparkTestUtil.getSparkSession(getSparkConf(AbstractSparkTest.class.getName()))); this.jsc = Optional.of(new JavaSparkContext(this.spark.get().sparkContext())); this.sqlContext = Optional.of(SQLContext.getOrCreate(this.jsc.get().sc())); try { this.fileSystem = Optional.of(FSUtils.getFs(new Configuration())); } catch (IOException e) { log.error("Cannot initialize FileSystem object", e); } }
@Before public void setupTest() throws IOException { config = new Configuration(); config.setProperty(MetadataConstants.JOBMANAGER_METADATA_STORAGE, "hdfs"); config.setProperty(MetadataConstants.JOBMANAGER_METADATA_HDFS_BASEPATH, HDFSTestConstants.JOBMANAGER_BASE_METADATA_BASEPATH); this.fileSystem = FSUtils.getFs(config); final AtomicBoolean condition = new AtomicBoolean(true); basePath = new Path(HDFSTestConstants.JOBMANAGER_BASE_METADATA_BASEPATH); this.tracker = Optional.of(new JobManagerMetadataTracker(config)); }
private static Map<String, String> readMetadataInfo( @NonNull final HoodieConfiguration hoodieConf) { try { final FileSystem fs = FSUtils.getFs(hoodieConf.getConf()); HoodieUtil.initHoodieDataset(fs, hoodieConf); final HoodieTableMetaClient hoodieTableMetaClient = new HoodieTableMetaClient(new HadoopConfiguration(hoodieConf.getConf()).getHadoopConf(), hoodieConf.getBasePath(), true); final HoodieActiveTimeline hoodieActiveTimeline = hoodieTableMetaClient.getActiveTimeline(); final java.util.Optional<HoodieInstant> lastInstant = hoodieActiveTimeline.getCommitTimeline() .filterCompletedInstants().lastInstant(); if (lastInstant.isPresent()) { log.info("using hoodie instant for reading checkpoint info :{}", lastInstant.get().getTimestamp()); final HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(hoodieActiveTimeline.getInstantDetails(lastInstant.get()).get()); final String serCommitInfo = commitMetadata.getMetadata(HOODIE_METADATA_KEY); if (!Strings.isNullOrEmpty(serCommitInfo)) { return MapUtil.deserializeMap(serCommitInfo); } } return new HashMap<>(); } catch (IOException e) { log.error("failed to read metadata info", e); throw new JobRuntimeException("failed to read metadata information", e); } } }
Assert.assertEquals(1, getCommitFiles(basePath, FSUtils.getFs(new Configuration())).size());
final HoodieWriteClientWrapper hoodieWriteClientWrapper = hoodieSink.getMockWriteClient(); Assert.assertEquals(1, getCommitFiles(basePath, FSUtils.getFs(new Configuration())).size());