throw new SegmentLoadingException("Unable to kill segment, failed to delete dir [%s]", partitionNumDir.toString()); throw new SegmentLoadingException("Unknown file type[%s]", path); throw new SegmentLoadingException(e, "Unable to kill segment");
private DataSegmentPuller getPuller(Map<String, Object> loadSpec) throws SegmentLoadingException { String type = MapUtils.getString(loadSpec, "type"); DataSegmentPuller loader = pullers.get(type); if (loader == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, pullers.keySet()); } return loader; } }
@Override public Set<DataSegment> findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException { final Map<String, Pair<DataSegment, Long>> timestampedSegments = new HashMap<>(); final File workingDir = new File(workingDirPath); if (!workingDir.isDirectory()) { throw new SegmentLoadingException("Working directory [%s] didn't exist !?", workingDir); } recursiveSearchSegments(timestampedSegments, workingDir, updateDescriptor); return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet()); }
private DataSegmentMover getMover(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentMover mover = movers.get(type); if (mover == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, movers.keySet()); } return mover; } }
private File getDirectory(DataSegment segment) throws SegmentLoadingException { final Map<String, Object> loadSpec = segment.getLoadSpec(); final File path = new File(MapUtils.getString(loadSpec, "path")); if (!path.exists()) { throw new SegmentLoadingException("Asked to load path[%s], but it doesn't exist.", path); } return path.getParentFile(); } }
private File getFile(DataSegment segment) throws SegmentLoadingException { final Map<String, Object> loadSpec = segment.getLoadSpec(); final File path = new File(MapUtils.getString(loadSpec, "path")); if (!path.exists()) { throw new SegmentLoadingException("Asked to load path[%s], but it doesn't exist.", path); } return path; } }
private DataSegmentKiller getKiller(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentKiller loader = killers.get(type); if (loader == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, killers.keySet()); } return loader; }
private File getFile(DataSegment segment) throws SegmentLoadingException { final Map<String, Object> loadSpec = segment.getLoadSpec(); final File path = new File(MapUtils.getString(loadSpec, "path")); if (!path.exists()) { throw new SegmentLoadingException("Asked to load path[%s], but it doesn't exist.", path); } return path; } }
private DataSegmentArchiver getArchiver(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentArchiver archiver = archivers.get(type); if (archiver == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, archivers.keySet()); } return archiver; } }
private DataSegmentArchiver getArchiver(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentArchiver archiver = archivers.get(type); if (archiver == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, archivers.keySet()); } return archiver; } }
private DataSegmentKiller getKiller(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentKiller loader = killers.get(type); if (loader == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, killers.keySet()); } return loader; }
private DataSegmentMover getMover(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentMover mover = movers.get(type); if (mover == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, movers.keySet()); } return mover; } }
private boolean isObjectInBucket(final S3Coords coords) throws SegmentLoadingException { try { return S3Utils.retryS3Operation( new Callable<Boolean>() { @Override public Boolean call() throws Exception { return S3Utils.isObjectInBucket(s3Client, coords.bucket, coords.path); } } ); } catch (S3ServiceException | IOException e) { throw new SegmentLoadingException(e, "S3 fail! Key[%s]", coords); } catch (Exception e) { throw Throwables.propagate(e); } }
@Override public Segment factorize(DataSegment dataSegment, File parentDir) throws SegmentLoadingException { try { return new QueryableIndexSegment(dataSegment.getIdentifier(), indexIO.loadIndex(parentDir)); } catch (IOException e) { throw new SegmentLoadingException(e, "%s", e.getMessage()); } } }
@Override public Segment getSegment(DataSegment segment) throws SegmentLoadingException { File segmentFiles = getSegmentFiles(segment); File factoryJson = new File(segmentFiles, "factory.json"); final SegmentizerFactory factory; if (factoryJson.exists()) { try { factory = jsonMapper.readValue(factoryJson, SegmentizerFactory.class); } catch (IOException e) { throw new SegmentLoadingException(e, "%s", e.getMessage()); } } else { factory = new MMappedQueryableSegmentizerFactory(indexIO); } return factory.factorize(segment, segmentFiles); }
@Override public QueryableIndex factorize(File parentDir) throws SegmentLoadingException { try { return IndexIO.loadIndex(parentDir); } catch (IOException e) { log.warn(e, "Got exception!!!! Going to delete parentDir[%s]", parentDir); try { FileUtils.deleteDirectory(parentDir); } catch (IOException e2) { log.error(e, "Problem deleting parentDir[%s]", parentDir); } throw new SegmentLoadingException(e, "%s", e.getMessage()); } } }
@Override public void kill(DataSegment segment) throws SegmentLoadingException { log.info("Killing segment [%s]", segment); Map<String, Object> loadSpec = segment.getLoadSpec(); final String containerName = MapUtils.getString(loadSpec, "containerName"); final String blobPath = MapUtils.getString(loadSpec, "blobPath"); final String dirPath = Paths.get(blobPath).getParent().toString(); try { azureStorage.emptyCloudBlobDirectory(containerName, dirPath); } catch(StorageException e){ throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getIdentifier(), e.getExtendedErrorInformation() == null ? null : e.getExtendedErrorInformation().getErrorMessage()); } catch (URISyntaxException e) { throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getIdentifier(), e.getReason()); } }
@Override public void cleanup(DataSegment segment) throws SegmentLoadingException { if (!config.isDeleteOnRemove()) { return; } StorageLocation loc = findStorageLocationIfLoaded(segment); if (loc == null) { log.info("Asked to cleanup something[%s] that didn't exist. Skipping.", segment); return; } try { File cacheFile = new File(loc.getPath(), DataSegmentPusherUtil.getStorageDir(segment)); log.info("Deleting directory[%s]", cacheFile); FileUtils.deleteDirectory(cacheFile); loc.removeSegment(segment); } catch (IOException e) { throw new SegmentLoadingException(e, e.getMessage()); } }
private Segment getAdapter(final DataSegment segment) throws SegmentLoadingException { final Segment adapter; try { adapter = segmentLoader.getSegment(segment); } catch (SegmentLoadingException e) { try { segmentLoader.cleanup(segment); } catch (SegmentLoadingException e1) { e.addSuppressed(e1); } throw e; } if (adapter == null) { throw new SegmentLoadingException("Null adapter from loadSpec[%s]", segment.getLoadSpec()); } return adapter; }
@Override public void kill(DataSegment segment) throws SegmentLoadingException { try { Map<String, Object> loadSpec = segment.getLoadSpec(); String s3Bucket = MapUtils.getString(loadSpec, "bucket"); String s3Path = MapUtils.getString(loadSpec, "key"); String s3DescriptorPath = S3Utils.descriptorPathForSegmentPath(s3Path); if (s3Client.isObjectInBucket(s3Bucket, s3Path)) { log.info("Removing index file[s3://%s/%s] from s3!", s3Bucket, s3Path); s3Client.deleteObject(s3Bucket, s3Path); } if (s3Client.isObjectInBucket(s3Bucket, s3DescriptorPath)) { log.info("Removing descriptor file[s3://%s/%s] from s3!", s3Bucket, s3DescriptorPath); s3Client.deleteObject(s3Bucket, s3DescriptorPath); } } catch (ServiceException e) { throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getIdentifier(), e); } }