@Override public void cleanup(DataSegment segment) { if (!config.isDeleteOnRemove()) { return; } StorageLocation loc = findStorageLocationIfLoaded(segment); if (loc == null) { log.info("Asked to cleanup something[%s] that didn't exist. Skipping.", segment); return; } // If storageDir.mkdirs() success, but downloadStartMarker.createNewFile() failed, // in this case, findStorageLocationIfLoaded() will think segment is located in the failed storageDir which is actually not. // So we should always clean all possible locations here for (StorageLocation location : getSortedList(locations)) { File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment, false)); if (localStorageDir.exists()) { // Druid creates folders of the form dataSource/interval/version/partitionNum. // We need to clean up all these directories if they are all empty. cleanupCacheFiles(location.getPath(), localStorageDir); location.removeSegment(segment); } } }
@Test public void testStorageLocation() { long expectedAvail = 1000L; StorageLocation loc = new StorageLocation(new File("/tmp"), expectedAvail, null); verifyLoc(expectedAvail, loc); final DataSegment secondSegment = makeSegment("2012-01-02/2012-01-03", 23); loc.addSegment(makeSegment("2012-01-01/2012-01-02", 10)); expectedAvail -= 10; verifyLoc(expectedAvail, loc); loc.addSegment(makeSegment("2012-01-01/2012-01-02", 10)); verifyLoc(expectedAvail, loc); loc.addSegment(secondSegment); expectedAvail -= 23; verifyLoc(expectedAvail, loc); loc.removeSegment(makeSegment("2012-01-01/2012-01-02", 10)); expectedAvail += 10; verifyLoc(expectedAvail, loc); loc.removeSegment(makeSegment("2012-01-01/2012-01-02", 10)); verifyLoc(expectedAvail, loc); loc.removeSegment(secondSegment); expectedAvail += 23; verifyLoc(expectedAvail, loc); }
@Override public void cleanup(DataSegment segment) { if (!config.isDeleteOnRemove()) { return; } StorageLocation loc = findStorageLocationIfLoaded(segment); if (loc == null) { log.info("Asked to cleanup something[%s] that didn't exist. Skipping.", segment); return; } // If storageDir.mkdirs() success, but downloadStartMarker.createNewFile() failed, // in this case, findStorageLocationIfLoaded() will think segment is located in the failed storageDir which is actually not. // So we should always clean all possible locations here for (StorageLocation location : getSortedList(locations)) { File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment, false)); if (localStorageDir.exists()) { // Druid creates folders of the form dataSource/interval/version/partitionNum. // We need to clean up all these directories if they are all empty. cleanupCacheFiles(location.getPath(), localStorageDir); location.removeSegment(segment); } } }