private StorageLocation findStorageLocationIfLoaded(final DataSegment segment) { for (StorageLocation location : getSortedList(locations)) { File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment, false)); if (localStorageDir.exists()) { return location; } } return null; }
@Override public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException { StorageLocation loc = findStorageLocationIfLoaded(segment); String storageDir = DataSegmentPusher.getDefaultStorageDir(segment, false); if (loc == null) { loc = loadSegmentWithRetry(segment, storageDir); } loc.addSegment(segment); return new File(loc.getPath(), storageDir); }
/** * location may fail because of IO failure, most likely in two cases:<p> * 1. druid don't have the write access to this location, most likely the administrator doesn't config it correctly<p> * 2. disk failure, druid can't read/write to this disk anymore */ private StorageLocation loadSegmentWithRetry(DataSegment segment, String storageDirStr) throws SegmentLoadingException { for (StorageLocation loc : getSortedList(locations)) { if (loc.canHandle(segment)) { File storageDir = new File(loc.getPath(), storageDirStr); try { loadInLocationWithStartMarker(segment, storageDir); return loc; } catch (SegmentLoadingException e) { log.makeAlert( e, "Failed to load segment in current location %s, try next location if any", loc.getPath().getAbsolutePath() ) .addData("location", loc.getPath().getAbsolutePath()) .emit(); cleanupCacheFiles(loc.getPath(), storageDir); } } } throw new SegmentLoadingException("Failed to load segment %s in all locations.", segment.getId()); }
@Override public void cleanup(DataSegment segment) { if (!config.isDeleteOnRemove()) { return; } StorageLocation loc = findStorageLocationIfLoaded(segment); if (loc == null) { log.info("Asked to cleanup something[%s] that didn't exist. Skipping.", segment); return; } // If storageDir.mkdirs() success, but downloadStartMarker.createNewFile() failed, // in this case, findStorageLocationIfLoaded() will think segment is located in the failed storageDir which is actually not. // So we should always clean all possible locations here for (StorageLocation location : getSortedList(locations)) { File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment, false)); if (localStorageDir.exists()) { // Druid creates folders of the form dataSource/interval/version/partitionNum. // We need to clean up all these directories if they are all empty. cleanupCacheFiles(location.getPath(), localStorageDir); location.removeSegment(segment); } } }
boolean canHandle(DataSegment segment) { if (available() < segment.getSize()) { log.warn( "Segment[%s:%,d] too large for storage[%s:%,d]. Check your druid.segmentCache.locations maxSize param", segment.getId(), segment.getSize(), getPath(), available() ); return false; } if (freeSpaceToKeep > 0) { long currFreeSpace = path.getFreeSpace(); if ((freeSpaceToKeep + segment.getSize()) > currFreeSpace) { log.warn( "Segment[%s:%,d] too large for storage[%s:%,d] to maintain suggested freeSpace[%d], current freeSpace is [%d].", segment.getId(), segment.getSize(), getPath(), available(), freeSpaceToKeep, currFreeSpace ); return false; } } return true; }
private StorageLocation findStorageLocationIfLoaded(final DataSegment segment) { for (StorageLocation location : getSortedList(locations)) { File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment, false)); if (localStorageDir.exists()) { return location; } } return null; }
@Override public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException { StorageLocation loc = findStorageLocationIfLoaded(segment); String storageDir = DataSegmentPusher.getDefaultStorageDir(segment, false); if (loc == null) { loc = loadSegmentWithRetry(segment, storageDir); } loc.addSegment(segment); return new File(loc.getPath(), storageDir); }
/** * location may fail because of IO failure, most likely in two cases:<p> * 1. druid don't have the write access to this location, most likely the administrator doesn't config it correctly<p> * 2. disk failure, druid can't read/write to this disk anymore */ private StorageLocation loadSegmentWithRetry(DataSegment segment, String storageDirStr) throws SegmentLoadingException { for (StorageLocation loc : getSortedList(locations)) { if (loc.canHandle(segment)) { File storageDir = new File(loc.getPath(), storageDirStr); try { loadInLocationWithStartMarker(segment, storageDir); return loc; } catch (SegmentLoadingException e) { log.makeAlert( e, "Failed to load segment in current location %s, try next location if any", loc.getPath().getAbsolutePath() ) .addData("location", loc.getPath().getAbsolutePath()) .emit(); cleanupCacheFiles(loc.getPath(), storageDir); } } } throw new SegmentLoadingException("Failed to load segment %s in all locations.", segment.getIdentifier()); }
@Override public void cleanup(DataSegment segment) { if (!config.isDeleteOnRemove()) { return; } StorageLocation loc = findStorageLocationIfLoaded(segment); if (loc == null) { log.info("Asked to cleanup something[%s] that didn't exist. Skipping.", segment); return; } // If storageDir.mkdirs() success, but downloadStartMarker.createNewFile() failed, // in this case, findStorageLocationIfLoaded() will think segment is located in the failed storageDir which is actually not. // So we should always clean all possible locations here for (StorageLocation location : getSortedList(locations)) { File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment, false)); if (localStorageDir.exists()) { // Druid creates folders of the form dataSource/interval/version/partitionNum. // We need to clean up all these directories if they are all empty. cleanupCacheFiles(location.getPath(), localStorageDir); location.removeSegment(segment); } } }
boolean canHandle(DataSegment segment) { if (available() < segment.getSize()) { log.warn( "Segment[%s:%,d] too large for storage[%s:%,d]. Check your druid.segmentCache.locations maxSize param", segment.getIdentifier(), segment.getSize(), getPath(), available() ); return false; } if (freeSpaceToKeep > 0) { long currFreeSpace = path.getFreeSpace(); if ((freeSpaceToKeep + segment.getSize()) > currFreeSpace) { log.warn( "Segment[%s:%,d] too large for storage[%s:%,d] to maintain suggested freeSpace[%d], current freeSpace is [%d].", segment.getIdentifier(), segment.getSize(), getPath(), available(), freeSpaceToKeep, currFreeSpace ); return false; } } return true; }