/** * @param connector SQL metadata connector to the metadata storage * @param metadataStorageTablesConfig Table config * * @return all the active data sources in the metadata storage */ static Collection<String> getAllDataSourceNames(SQLMetadataConnector connector, final MetadataStorageTablesConfig metadataStorageTablesConfig) { return connector.getDBI() .withHandle((HandleCallback<List<String>>) handle -> handle.createQuery(String.format( "SELECT DISTINCT(datasource) FROM %s WHERE used = true", metadataStorageTablesConfig.getSegmentsTable())) .fold(Lists.<String>newArrayList(), (druidDataSources, stringObjectMap, foldController, statementContext) -> { druidDataSources.add(MapUtils.getString(stringObjectMap, "datasource")); return druidDataSources; })); }
@Override public String apply(Map<String, Object> map) { return MapUtils.getString(map, key); } };
public static String getString(Map<String, Object> in, String key) { return getString(in, key, null); }
public static <RetVal> RetVal lookupStringValInMap(Map<String, Object> map, String key, Map<String, RetVal> lookupMap) { String lookupKey = getString(map, key); RetVal retVal = lookupMap.get(lookupKey); if (retVal == null) { throw new IAE("Unknown %s[%s], known values are%s", key, lookupKey, lookupMap.keySet()); } return retVal; }
private File getPath(DataSegment segment) throws SegmentLoadingException { return new File(MapUtils.getString(segment.getLoadSpec(), PATH_KEY)); } }
private DataSegmentMover getMover(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentMover mover = movers.get(type); if (mover == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, movers.keySet()); } return mover; } }
private DataSegmentArchiver getArchiver(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentArchiver archiver = archivers.get(type); if (archiver == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, archivers.keySet()); } return archiver; } }
private File getFile(DataSegment segment) throws SegmentLoadingException { final Map<String, Object> loadSpec = segment.getLoadSpec(); final File path = new File(MapUtils.getString(loadSpec, "path")); if (!path.exists()) { throw new SegmentLoadingException("Asked to load path[%s], but it doesn't exist.", path); } return path; } }
private DataSegmentKiller getKiller(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentKiller loader = killers.get(type); if (loader == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, killers.keySet()); } return loader; }
String s3Bucket = MapUtils.getString(loadSpec, "bucket"); String s3Path = MapUtils.getString(loadSpec, "key"); String s3DescriptorPath = S3Utils.descriptorPathForSegmentPath(s3Path); final String targetS3Bucket = MapUtils.getString(targetLoadSpec, "bucket"); final String targetS3BaseKey = MapUtils.getString(targetLoadSpec, "baseKey");
@Override public void kill(DataSegment segment) throws SegmentLoadingException { try { Map<String, Object> loadSpec = segment.getLoadSpec(); String s3Bucket = MapUtils.getString(loadSpec, "bucket"); String s3Path = MapUtils.getString(loadSpec, "key"); String s3DescriptorPath = S3Utils.descriptorPathForSegmentPath(s3Path); if (s3Client.isObjectInBucket(s3Bucket, s3Path)) { log.info("Removing index file[s3://%s/%s] from s3!", s3Bucket, s3Path); s3Client.deleteObject(s3Bucket, s3Path); } if (s3Client.isObjectInBucket(s3Bucket, s3DescriptorPath)) { log.info("Removing descriptor file[s3://%s/%s] from s3!", s3Bucket, s3DescriptorPath); s3Client.deleteObject(s3Bucket, s3DescriptorPath); } } catch (ServiceException e) { throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getIdentifier(), e); } }