/** * Return true - if there are still valid volumes on the DataNode. */ @Override // FsDatasetSpi public boolean hasEnoughResource() { return getVolumes().size() >= validVolsRequired; }
/** * Return true - if there are still valid volumes on the DataNode. */ @Override // FsDatasetSpi public boolean hasEnoughResource() { return getVolumes().size() >= validVolsRequired; }
private boolean ramDiskConfigured() { for (FsVolumeImpl v: getVolumes()){ if (v.isTransientStorage()) { return true; } } return false; }
private void setupAsyncLazyPersistThreads() { for (FsVolumeImpl v: getVolumes()){ setupAsyncLazyPersistThread(v); } }
private void setupAsyncLazyPersistThreads() { for (FsVolumeImpl v: getVolumes()){ setupAsyncLazyPersistThread(v); } }
private boolean ramDiskConfigured() { for (FsVolumeImpl v: getVolumes()){ if (v.isTransientStorage()) { return true; } } return false; }
private Collection<VolumeInfo> getVolumeInfo() { Collection<VolumeInfo> info = new ArrayList<VolumeInfo>(); for (FsVolumeImpl volume : getVolumes()) { long used = 0; long free = 0; try (FsVolumeReference ref = volume.obtainReference()) { used = volume.getDfsUsed(); free = volume.getAvailable(); } catch (ClosedChannelException e) { continue; } catch (IOException e) { LOG.warn(e.getMessage()); used = 0; free = 0; } info.add(new VolumeInfo(volume, used, free)); } return info; }
List<FsVolumeImpl> curVolumes = getVolumes();
private Collection<VolumeInfo> getVolumeInfo() { Collection<VolumeInfo> info = new ArrayList<VolumeInfo>(); for (FsVolumeImpl volume : getVolumes()) { long used = 0; long free = 0; try (FsVolumeReference ref = volume.obtainReference()) { used = volume.getDfsUsed(); free = volume.getAvailable(); } catch (ClosedChannelException e) { continue; } catch (IOException e) { LOG.warn(e.getMessage()); used = 0; free = 0; } info.add(new VolumeInfo(volume, used, free)); } return info; }
@Override //FsDatasetSpi public synchronized void deleteBlockPool(String bpid, boolean force) throws IOException { List<FsVolumeImpl> curVolumes = getVolumes(); if (!force) { for (FsVolumeImpl volume : curVolumes) { try (FsVolumeReference ref = volume.obtainReference()) { if (!volume.isBPDirEmpty(bpid)) { LOG.warn(bpid + " has some block files, cannot delete unless forced"); throw new IOException("Cannot delete block pool, " + "it contains some block files"); } } catch (ClosedChannelException e) { // ignore. } } } for (FsVolumeImpl volume : curVolumes) { try (FsVolumeReference ref = volume.obtainReference()) { volume.deleteBPDirectories(bpid, force); } catch (ClosedChannelException e) { // ignore. } } }
@Override // FsDatasetSpi public StorageReport[] getStorageReports(String bpid) throws IOException { List<StorageReport> reports; synchronized (statsLock) { List<FsVolumeImpl> curVolumes = getVolumes(); reports = new ArrayList<>(curVolumes.size()); for (FsVolumeImpl volume : curVolumes) { try (FsVolumeReference ref = volume.obtainReference()) { StorageReport sr = new StorageReport(volume.toDatanodeStorage(), false, volume.getCapacity(), volume.getDfsUsed(), volume.getAvailable(), volume.getBlockPoolUsed(bpid), volume.getNonDfsUsed()); reports.add(sr); } catch (ClosedChannelException e) { continue; } } } return reports.toArray(new StorageReport[reports.size()]); }
@Override // FsDatasetSpi public StorageReport[] getStorageReports(String bpid) throws IOException { List<StorageReport> reports; synchronized (statsLock) { List<FsVolumeImpl> curVolumes = getVolumes(); reports = new ArrayList<>(curVolumes.size()); for (FsVolumeImpl volume : curVolumes) { try (FsVolumeReference ref = volume.obtainReference()) { StorageReport sr = new StorageReport(volume.toDatanodeStorage(), false, volume.getCapacity(), volume.getDfsUsed(), volume.getAvailable(), volume.getBlockPoolUsed(bpid), volume.getNonDfsUsed()); reports.add(sr); } catch (ClosedChannelException e) { continue; } } } return reports.toArray(new StorageReport[reports.size()]); }
@Override //FsDatasetSpi public synchronized void deleteBlockPool(String bpid, boolean force) throws IOException { List<FsVolumeImpl> curVolumes = getVolumes(); if (!force) { for (FsVolumeImpl volume : curVolumes) { try (FsVolumeReference ref = volume.obtainReference()) { if (!volume.isBPDirEmpty(bpid)) { LOG.warn(bpid + " has some block files, cannot delete unless forced"); throw new IOException("Cannot delete block pool, " + "it contains some block files"); } } catch (ClosedChannelException e) { // ignore. } } } for (FsVolumeImpl volume : curVolumes) { try (FsVolumeReference ref = volume.obtainReference()) { volume.deleteBPDirectories(bpid, force); } catch (ClosedChannelException e) { // ignore. } } }
private boolean transientFreeSpaceBelowThreshold() throws IOException { long free = 0; long capacity = 0; float percentFree = 0.0f; // Don't worry about fragmentation for now. We don't expect more than one // transient volume per DN. for (FsVolumeImpl v : getVolumes()) { try (FsVolumeReference ref = v.obtainReference()) { if (v.isTransientStorage()) { capacity += v.getCapacity(); free += v.getAvailable(); } } catch (ClosedChannelException e) { // ignore. } } if (capacity == 0) { return false; } percentFree = (float) ((double)free * 100 / capacity); return (percentFree < lowWatermarkFreeSpacePercentage) || (free < lowWatermarkFreeSpaceBytes); }
private boolean transientFreeSpaceBelowThreshold() throws IOException { long free = 0; long capacity = 0; float percentFree = 0.0f; // Don't worry about fragmentation for now. We don't expect more than one // transient volume per DN. for (FsVolumeImpl v : getVolumes()) { try (FsVolumeReference ref = v.obtainReference()) { if (v.isTransientStorage()) { capacity += v.getCapacity(); free += v.getAvailable(); } } catch (ClosedChannelException e) { // ignore. } } if (capacity == 0) { return false; } percentFree = (float) ((double)free * 100 / capacity); return (percentFree < lowWatermarkFreeSpacePercentage) || (free < lowWatermarkFreeSpaceBytes); }
new HashMap<String, BlockListAsLongs.Builder>(); List<FsVolumeImpl> curVolumes = getVolumes(); for (FsVolumeSpi v : curVolumes) { builders.put(v.getStorageID(), BlockListAsLongs.builder());
new HashMap<String, BlockListAsLongs.Builder>(); List<FsVolumeImpl> curVolumes = getVolumes(); for (FsVolumeSpi v : curVolumes) { builders.put(v.getStorageID(), BlockListAsLongs.builder());
@Test public void testAddVolumes() throws IOException { final int numNewVolumes = 3; final int numExistingVolumes = dataset.getVolumes().size(); final int totalVolumes = numNewVolumes + numExistingVolumes; Set<String> expectedVolumes = new HashSet<String>(); assertEquals(totalVolumes, dataset.getVolumes().size()); assertEquals(totalVolumes, dataset.storageMap.size()); for (int i = 0; i < numNewVolumes; i++) { actualVolumes.add( dataset.getVolumes().get(numExistingVolumes + i).getBasePath());
@Test(timeout = 5000) public void testRemoveNewlyAddedVolume() throws IOException { final int numExistingVolumes = dataset.getVolumes().size(); List<NamespaceInfo> nsInfos = new ArrayList<>(); for (String bpid : BLOCK_POOL_IDS) { nsInfos.add(new NamespaceInfo(0, CLUSTER_ID, bpid, 1)); } String newVolumePath = BASE_DIR + "/newVolumeToRemoveLater"; StorageLocation loc = StorageLocation.parse(newVolumePath); Storage.StorageDirectory sd = createStorageDirectory(new File(newVolumePath)); DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); when(storage.prepareVolume(eq(datanode), eq(loc.getFile()), anyListOf(NamespaceInfo.class))) .thenReturn(builder); dataset.addVolume(loc, nsInfos); assertEquals(numExistingVolumes + 1, dataset.getVolumes().size()); when(storage.getNumStorageDirs()).thenReturn(numExistingVolumes + 1); when(storage.getStorageDir(numExistingVolumes)).thenReturn(sd); Set<File> volumesToRemove = new HashSet<>(); volumesToRemove.add(loc.getFile()); dataset.removeVolumes(volumesToRemove, true); assertEquals(numExistingVolumes, dataset.getVolumes().size()); }
@Before public void setUp() throws IOException { datanode = mock(DataNode.class); storage = mock(DataStorage.class); this.conf = new Configuration(); this.conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 0); final DNConf dnConf = new DNConf(conf); when(datanode.getConf()).thenReturn(conf); when(datanode.getDnConf()).thenReturn(dnConf); final BlockScanner disabledBlockScanner = new BlockScanner(datanode, conf); when(datanode.getBlockScanner()).thenReturn(disabledBlockScanner); final ShortCircuitRegistry shortCircuitRegistry = new ShortCircuitRegistry(conf); when(datanode.getShortCircuitRegistry()).thenReturn(shortCircuitRegistry); createStorageDirs(storage, conf, NUM_INIT_VOLUMES); dataset = new FsDatasetImpl(datanode, storage, conf); for (String bpid : BLOCK_POOL_IDS) { dataset.addBlockPool(bpid, conf); } assertEquals(NUM_INIT_VOLUMES, dataset.getVolumes().size()); assertEquals(0, dataset.getNumFailedVolumes()); }