static Path getPath(DataSegment dataSegment) { return new Path(String.valueOf(Objects.requireNonNull(dataSegment.getLoadSpec()).get("path"))); }
persistedSegment.getLoadSpec()); Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri())));
dataSegmentPusher.makeIndexPathName(persistedSegment, DruidStorageHandlerUtils.INDEX_ZIP)); Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", expectedFinalHadoopPath.toString()), persistedSegment.getLoadSpec()); Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri())));
persistedSegment.getLoadSpec()); Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri())));
persistedSegment.getLoadSpec()); Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri())));
@Override public void getSegmentFiles(DataSegment segment, File outDir) throws SegmentLoadingException { String key = (String) segment.getLoadSpec().get("key"); getSegmentFiles(key, outDir); } public com.metamx.common.FileUtils.FileCopyResult getSegmentFiles(final String key, final File outDir) throws SegmentLoadingException{
private File getPath(DataSegment segment) throws SegmentLoadingException { return new File(MapUtils.getString(segment.getLoadSpec(), PATH_KEY)); } }
private DataSegmentMover getMover(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentMover mover = movers.get(type); if (mover == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, movers.keySet()); } return mover; } }
boolean sameLoadSpec(DataSegment s1, DataSegment s2) { final S3LoadSpec s1LoadSpec = (S3LoadSpec) mapper.convertValue(s1.getLoadSpec(), LoadSpec.class); final S3LoadSpec s2LoadSpec = (S3LoadSpec) mapper.convertValue(s2.getLoadSpec(), LoadSpec.class); return Objects.equal(s1LoadSpec.getBucket(), s2LoadSpec.getBucket()) && Objects.equal( s1LoadSpec.getKey(), s2LoadSpec.getKey() ); } }
private File getDirectory(DataSegment segment) throws SegmentLoadingException { final Map<String, Object> loadSpec = segment.getLoadSpec(); final File path = new File(MapUtils.getString(loadSpec, "path")); if (!path.exists()) { throw new SegmentLoadingException("Asked to load path[%s], but it doesn't exist.", path); } return path.getParentFile(); } }
private DataSegmentArchiver getArchiver(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentArchiver archiver = archivers.get(type); if (archiver == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, archivers.keySet()); } return archiver; } }
private DataSegmentKiller getKiller(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentKiller loader = killers.get(type); if (loader == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, killers.keySet()); } return loader; }
private DataSegmentMover getMover(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentMover mover = movers.get(type); if (mover == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, movers.keySet()); } return mover; } }
public S3Coords(DataSegment segment) { Map<String, Object> loadSpec = segment.getLoadSpec(); bucket = MapUtils.getString(loadSpec, BUCKET); path = MapUtils.getString(loadSpec, KEY); if (path.startsWith("/")) { path = path.substring(1); } }
private File getFile(DataSegment segment) throws SegmentLoadingException { final Map<String, Object> loadSpec = segment.getLoadSpec(); final File path = new File(MapUtils.getString(loadSpec, "path")); if (!path.exists()) { throw new SegmentLoadingException("Asked to load path[%s], but it doesn't exist.", path); } return path; } }
private DataSegmentKiller getKiller(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentKiller loader = killers.get(type); if (loader == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, killers.keySet()); } return loader; }
private DataSegmentArchiver getArchiver(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentArchiver archiver = archivers.get(type); if (archiver == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, archivers.keySet()); } return archiver; } }
private File getFile(DataSegment segment) throws SegmentLoadingException { final Map<String, Object> loadSpec = segment.getLoadSpec(); final File path = new File(MapUtils.getString(loadSpec, "path")); if (!path.exists()) { throw new SegmentLoadingException("Asked to load path[%s], but it doesn't exist.", path); } return path; } }
@Override public void getSegmentFiles(DataSegment segment, File outDir) throws SegmentLoadingException { final Map<String, Object> loadSpec = segment.getLoadSpec(); final String containerName = MapUtils.getString(loadSpec, "containerName"); final String blobPath = MapUtils.getString(loadSpec, "blobPath"); getSegmentFiles(containerName, blobPath, outDir); }
private void loadInLocation(DataSegment segment, File storageDir) throws SegmentLoadingException { // LoadSpec isn't materialized until here so that any system can interpret Segment without having to have all the // LoadSpec dependencies. final LoadSpec loadSpec = jsonMapper.convertValue(segment.getLoadSpec(), LoadSpec.class); final LoadSpec.LoadSpecResult result = loadSpec.loadSegment(storageDir); if (result.getSize() != segment.getSize()) { log.warn( "Segment [%s] is different than expected size. Expected [%d] found [%d]", segment.getIdentifier(), segment.getSize(), result.getSize() ); } }