private String removeHdfsFilesFromMapping(ResourceType t, String file){ String key = null; if (resourceMaps.getLocalHdfsLocationMap(t).containsValue(file)){ for (Map.Entry<String, String> entry : resourceMaps.getLocalHdfsLocationMap(t).entrySet()){ if (entry.getValue().equals(file)){ key = entry.getKey(); resourceMaps.getLocalHdfsLocationMap(t).remove(key); } } } return key; }
public Set<String> list_hdfs_resource(ResourceType type) { Set<String> set = resourceMaps.getResourceSet(type); Set<String> result = new HashSet<String>(); for (String file : set){ if (resourceMaps.getLocalHdfsLocationMap(ResourceType.FILE).containsKey(file)){ result.add(resourceMaps.getLocalHdfsLocationMap(ResourceType.FILE).get(file)); } if (resourceMaps.getLocalHdfsLocationMap(ResourceType.JAR).containsKey(file)){ result.add(resourceMaps.getLocalHdfsLocationMap(ResourceType.JAR).get(file)); } } return result; }
public Set<String> list_local_resource(ResourceType type) { Set<String> resources = new HashSet<String>(list_resource(type, null)); Set<String> set = resourceMaps.getResourceSet(type); for (String file : set){ if (resourceMaps.getLocalHdfsLocationMap(ResourceType.FILE).containsKey(file)){ resources.remove(file); } if (resourceMaps.getLocalHdfsLocationMap(ResourceType.JAR).containsKey(file)){ resources.remove(file); } } return resources; }
@VisibleForTesting protected List<URI> resolveAndDownload(ResourceType resourceType, String value, boolean convertToUnix) throws URISyntaxException, IOException { List<URI> uris = resourceDownloader.resolveAndDownload(value, convertToUnix); if (ResourceDownloader.isHdfsUri(value)) { assert uris.size() == 1 : "There should only be one URI localized-resource."; resourceMaps.getLocalHdfsLocationMap(resourceType).put(uris.get(0).toString(), value); } return uris; }