public String getFileName(final File atlasResource) { return atlasResource.getName(); }
private Set<File> getFilesOfType(final File file) { if (FilenameUtils.getExtension(file.getName()).equalsIgnoreCase(this.fileExtension)) { return Collections.singleton(file); } else if (file.isDirectory()) { return file .listFilesRecursively().stream().filter(subFile -> FilenameUtils .getExtension(subFile.getName()).equalsIgnoreCase(this.fileExtension)) .collect(Collectors.toSet()); } return new HashSet<>(); }
private void compareShardByShard(final Path before, final Path after, final Path outputDirectory) { final List<File> afterShardFiles = fetchAtlasFilesInDirectory(after); afterShardFiles.parallelStream().forEach(afterShardFile -> { final Path beforeShardPath = before.resolve(afterShardFile.getName()); final Atlas beforeAtlas = load(beforeShardPath); final Atlas afterAtlas = ATLAS_RESOURCE_LOADER.load(afterShardFile); compare(beforeAtlas, afterAtlas, outputDirectory); }); }
private void convertAtlases(final Path atlasDirectory, final Path geojsonDirectory) { final List<File> atlases = fetchAtlasFilesInDirectory(atlasDirectory); atlases.parallelStream().forEach(atlasFile -> { final Time time = Time.now(); final Atlas atlas = ATLAS_RESOURCE_LOADER.load(atlasFile); final String name = FilenameUtils.removeExtension(atlasFile.getName()) + FileSuffix.GEO_JSON.toString(); final File geojsonFile = new File(geojsonDirectory.resolve(name).toFile()); atlas.saveAsLineDelimitedGeoJsonFeatures(geojsonFile, ENTITY_PREDICATE, this.jsonMutator); logger.info("Saved {} in {}.", name, time.elapsedSince()); }); } }
/** * Get or create a {@link CountryBoundaryMap}. If the country-boundary-map parameter is set, * this will attempt to load the text or shape file from that parameter. Else, this will load * using the entire world as the country UNK (unknown). * * @param map * {@link CommandMap} containing the {@code COUNTRY_MAP_PARAMETER} * @return {@link CountryBoundaryMap} loaded from a file or default */ private CountryBoundaryMap getCountryBoundaryMap(final CommandMap map) { final Optional<File> countryMapOption = (Optional<File>) map .getOption(COUNTRY_MAP_PARAMETER); CountryBoundaryMap countryMap = CountryBoundaryMap .fromBoundaryMap(Collections.singletonMap("UNK", MultiPolygon.MAXIMUM)); if (countryMapOption.isPresent()) { final File countryMapFile = countryMapOption.get(); if (FilenameUtils.isExtension(countryMapFile.getName(), "txt")) { countryMap = CountryBoundaryMap.fromPlainText(countryMapFile); } else if (FilenameUtils.isExtension(countryMapFile.getName(), "shp")) { countryMap = CountryBoundaryMap.fromShapeFile(countryMapFile.getFile()); } } return countryMap; }
protected Iterable<PropertiesLocated> readGeoJsonItems(final Mode mode, final Iterable<File> files, final String filePrefix) { switch (mode) { case FILE: return Iterables.stream(files) .filter(file -> file.getName().endsWith(FileSuffix.GEO_JSON.toString())) .flatMap(this::readGeoJsonItems); case LINE: return Iterables.stream(files).filter(file -> file.getName().startsWith(filePrefix)) .flatMap(file -> file.lines()).map(line -> line.trim()) .filter(line -> !line.isEmpty()).flatMap(this::readGeoJsonItems); default: throw new CoreException("Invalid Mode"); } }
@Override protected int onRun(final CommandMap command) { final File inputFolder = (File) command.get(INPUT_FOLDER); final String atlasName = inputFolder.getName(); final File outputFolder = (File) command.get(OUTPUT_FOLDER); initialize(command); if (inputFolder != null) { this.logger.info("Loading Atlas from {}", inputFolder); final Atlas atlas = loadAtlas(command); processAtlas(atlasName, atlas, outputFolder.getPath()); } return 0; }
@Test public void fileCreationFromFileTest() { final File temp = File.temporaryFolder(); // Run AtlasJoinerSubCommand final String[] args = { "log-diff", String.format("-source=%s", GEO_11), String.format("-target=%s", GEO_21), String.format("-output=%s", temp.getPath()) }; new AtlasChecksCommand(args).runWithoutQuitting(args); final List<File> outputFiles = temp.listFilesRecursively(); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("additions-\\d+-1.log"))); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("changes-\\d+-1.log"))); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("subtractions-\\d+-1.log"))); temp.deleteRecursively(); }
@Test public void fileCreationFromFileTest() { final File temp = File.temporaryFolder(); // Run AtlasJoinerSubCommand final String[] args = { "geojson-diff", String.format("-source=%s", GEO_11), String.format("-target=%s", GEO_21), String.format("-output=%s", temp.getPath()) }; new AtlasChecksCommand(args).runWithoutQuitting(args); final List<File> outputFiles = temp.listFilesRecursively(); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("additions-\\d+-1.geojson"))); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("changes-\\d+-1.geojson"))); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("subtractions-\\d+-1.geojson"))); temp.deleteRecursively(); }
@Test public void fileCreationFromDirectoryTest() { final File temp = File.temporaryFolder(); // Run AtlasJoinerSubCommand final String[] args = { "geojson-diff", String.format("-source=%s", SOURCE_DIRECTORY), String.format("-target=%s", TARGET_DIRECTORY), String.format("-output=%s", temp.getPath()) }; new AtlasChecksCommand(args).runWithoutQuitting(args); final List<File> outputFiles = temp.listFilesRecursively(); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("additions-\\d+-1.geojson"))); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("changes-\\d+-2.geojson"))); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("subtractions-\\d+-3.geojson"))); temp.deleteRecursively(); } }
@Test public void fileCreationFromDirectoryTest() { final File temp = File.temporaryFolder(); // Run AtlasJoinerSubCommand final String[] args = { "log-diff", String.format("-source=%s", SOURCE_DIRECTORY), String.format("-target=%s", TARGET_DIRECTORY), String.format("-output=%s", temp.getPath()) }; new AtlasChecksCommand(args).runWithoutQuitting(args); final List<File> outputFiles = temp.listFilesRecursively(); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("additions-\\d+-1.log"))); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("changes-\\d+-2.log"))); Assert.assertTrue(outputFiles.stream() .anyMatch(file -> file.getName().matches("subtractions-\\d+-3.log"))); temp.deleteRecursively(); } }
@Override protected int onRun(final CommandMap command) { @SuppressWarnings("unchecked") final List<File> inputs = (List<File>) command.get(INPUT); final File output = (File) command.get(OUTPUT); output.mkdirs(); final MultiMap<String, AtlasStatistics> stats = new MultiMap<>(); for (final File input : inputs) { final String country = input.getName().split("_")[0]; final AtlasStatistics stat = AtlasStatistics.fromResource(input); stats.add(country, stat); } final Map<String, AtlasStatistics> countryStats = stats.reduceByKey(AtlasStatistics::merge); countryStats.forEach((country, stat) -> stat.save(output.child(country + ".csv"))); return 0; }
this.getFileSource().getName(), e);
@Override public Iterable<Resource> entries() { try (ZipFile file = new ZipFile(getFileSource().getFile()) { @Override public void close() { // Do nothing to close the file here, to avoid cutting the legs off the just created // ZipEntry-based resource. } }) { return Iterables.translate(Iterables.from(file.entries()), entry -> { return new InputStreamResource(inputStreamSupplier(file, entry)) .withName(entry.getName()); }); } catch (final IOException e) { throw new CoreException("Cannot get entries from the Zipfile {}.", this.getFileSource().getName(), e); } }
@Override protected int onRun(final CommandMap command) { final String delimiter = (String) command.get(DELIMITER); final File inputFile = (File) command.get(POLYLINES); final Iterable<List<PolyLine>> inputs = Iterables.stream(inputFile.lines()) .map(line -> StringList.split(line, delimiter).stream() .map(WKT_POLY_LINE_CONVERTER::backwardConvert) .collect(Collectors.toList())); try (SafeBufferedWriter writer = writer(command)) { for (final List<PolyLine> input : inputs) { writer.writeLine(new StringList( Iterables.stream(MULTIPLE_POLY_LINE_TO_POLYGONS_CONVERTER.convert(input)) .map(Polygon::toWkt)).join(delimiter)); } } catch (final Exception e) { throw new CoreException("Unable to convert polylines from {}", inputFile.getName(), e); } return 0; }