public HDFSMetadataManager(@NonNull final FileSystem fs, @NotEmpty final String baseMetadataPath, @NonNull final AtomicBoolean shouldSaveChanges) throws IOException { this.fileSystem = fs; this.baseMetadataPath = baseMetadataPath; this.shouldSaveChanges = shouldSaveChanges; this.metadataMap = loadMetadata(); }
/** * This method will load the latest metadata file within the base metadata path for the * stated metadataKey. * * @return Map of Metadata keys to values * @throws IOException */ public Map<String, StringValue> loadMetadata() throws IOException { log.info("Attempting to load metadata"); final Optional<FileStatus> fs = getLatestMetadataFile(); if (fs.isPresent()) { log.info("Loading metadata from: {}", fs.get().getPath()); return loadMetadata(fs.get().getPath()); } else { log.info("No metadata file found"); } return new HashMap<String, StringValue>(); }
private Optional<StringValue> getLatestCheckpoint(@NonNull HDFSMetadataManager metadataManager) throws IOException { final Map<String, StringValue> metadataMap = metadataManager.loadMetadata(); return metadataMap.containsKey(MetadataConstants.CHECKPOINT_KEY) ? Optional.of(metadataMap.get(MetadataConstants.CHECKPOINT_KEY)) : Optional.absent(); }
private Optional<StringValue> getLatestCheckpoint(@NonNull HDFSMetadataManager metadataManager) throws IOException { final Map<String, StringValue> metadataMap = metadataManager.loadMetadata(); return metadataMap.containsKey(MetadataConstants.CHECKPOINT_KEY) ? Optional.of(metadataMap.get(MetadataConstants.CHECKPOINT_KEY)) : Optional.absent(); } }
@Test public void testHDFSOverwriteCheckpointValue() throws IOException, InterruptedException { final StringValue val1 = new StringValue("testVal"); this.metadataManager.set(MetadataConstants.CHECKPOINT_KEY, val1); final StringValue val2 = new StringValue("testVal2"); this.metadataManager.set(MetadataConstants.CHECKPOINT_KEY, val2); final Optional<StringValue> readValue = this.metadataManager.get(MetadataConstants.CHECKPOINT_KEY); Assert.assertTrue(readValue.isPresent()); Assert.assertTrue(readValue.get().getValue().equals("testVal2")); this.metadataManager.saveChanges(); final Optional<FileStatus> fs = this.metadataManager.getLatestMetadataFile(); Assert.assertTrue(fs.isPresent()); final Map<String, StringValue> loadedMap = this.metadataManager.loadMetadata(fs.get().getPath()); validateDeserializedMapEqualsInMemoryMap(loadedMap); }
@Test public void testDeletionIsPropagated() throws Exception { final StringValue val1 = new StringValue("testVal"); this.metadataManager.set(MetadataConstants.CHECKPOINT_KEY, val1); this.metadataManager.saveChanges(); Optional<FileStatus> fs = this.metadataManager.getLatestMetadataFile(); Assert.assertTrue(fs.isPresent()); Map<String, StringValue> loadedMap = this.metadataManager.loadMetadata(fs.get().getPath()); validateDeserializedMapEqualsInMemoryMap(loadedMap); // reload the configuration setupTest(); Assert.assertTrue(this.metadataManager.get(MetadataConstants.CHECKPOINT_KEY).isPresent()); this.metadataManager.remove(MetadataConstants.CHECKPOINT_KEY); Assert.assertFalse(this.metadataManager.get(MetadataConstants.CHECKPOINT_KEY).isPresent()); this.metadataManager.saveChanges(); fs = this.metadataManager.getLatestMetadataFile(); Assert.assertFalse(this.metadataManager.get(MetadataConstants.CHECKPOINT_KEY).isPresent()); loadedMap = this.metadataManager.loadMetadata(fs.get().getPath()); validateDeserializedMapEqualsInMemoryMap(loadedMap); }
@Test public void testHDFSReadWriteSingleMetadataFile() throws IOException { // Test in memory final StringValue val = new StringValue("testVal"); this.metadataManager.set(MetadataConstants.CHECKPOINT_KEY, val); final Optional<StringValue> readValue = this.metadataManager.get(MetadataConstants.CHECKPOINT_KEY); Assert.assertTrue(readValue.isPresent()); Assert.assertTrue(readValue.get().getValue().equals("testVal")); this.metadataManager.set("foo", new StringValue("bar")); // Serialize the metadata map to a file this.metadataManager.saveChanges(); final Optional<FileStatus> fs = this.metadataManager.getLatestMetadataFile(); Assert.assertTrue(fs.isPresent()); // Deserialize the metadata map and check contents are the same final Map<String, StringValue> loadedMap = this.metadataManager.loadMetadata(fs.get().getPath()); validateDeserializedMapEqualsInMemoryMap(loadedMap); }