private static Supplier attr(final URL resource, final BiConsumer<Long, Long> attrs) throws Exception { if ("file".equals(resource.getProtocol())) { File file = new File(resource.toURI()); if (file.exists() && file.isFile()) { attrs.accept(file.length(), file.lastModified()); return () -> new FileInputStream(file); } return null; } else { URLConnection cnn = resource.openConnection(); cnn.setUseCaches(false); attrs.accept(cnn.getContentLengthLong(), cnn.getLastModified()); try { Closeables.closeQuietly(cnn.getInputStream()); } catch (NullPointerException ex) { // dir entries throw NPE :S return null; } return () -> resource.openStream(); } }
in = new BufferedReader(new InputStreamReader(new FileInputStream(metaFile), StandardCharsets.UTF_8)); Closeables.close(in, false);
private DirectBuffer readFrom(File file) throws IOException { FileInputStream stream = new FileInputStream(file); try { byteBuffer.clear(); stream.getChannel().read(byteBuffer); byteBuffer.flip(); return this; } finally { Closeables.closeQuietly(stream); } } }
public static void writePolicy(ClusteringPolicy policy, Path path) throws IOException { Path policyPath = new Path(path, POLICY_FILE_NAME); Configuration config = new Configuration(); FileSystem fs = FileSystem.get(policyPath.toUri(), config); SequenceFile.Writer writer = new SequenceFile.Writer(fs, config, policyPath, Text.class, ClusteringPolicyWritable.class); writer.append(new Text(), new ClusteringPolicyWritable(policy)); Closeables.close(writer, false); } }
/** * Read the model meta-data from the specified location. * * @throws IOException */ protected void readFromModel() throws IOException { Preconditions.checkArgument(this.modelPath != null, "Model path has not been set."); FSDataInputStream is = null; try { Path path = new Path(this.modelPath); FileSystem fs = path.getFileSystem(new Configuration()); is = new FSDataInputStream(fs.open(path)); this.readFields(is); } finally { Closeables.close(is, true); } }
/** * Write the model data to specified location. * * @throws IOException */ public void writeModelToFile() throws IOException { Preconditions.checkArgument(this.modelPath != null, "Model path has not been set."); FSDataOutputStream stream = null; try { Path path = new Path(this.modelPath); FileSystem fs = path.getFileSystem(new Configuration()); stream = fs.create(path, true); this.write(stream); } finally { Closeables.close(stream, false); } }
public static Vector read(Path path, Configuration conf) throws IOException { FileSystem fs = FileSystem.get(path.toUri(), conf); FSDataInputStream in = fs.open(path); try { return VectorWritable.readVector(in); } finally { Closeables.close(in, true); } } }
/** * @param state The final LanczosState to be serialized * @param outputPath The path (relative to the current Configuration's FileSystem) to save the output to. */ public void serializeOutput(LanczosState state, Path outputPath) throws IOException { int numEigenVectors = state.getIterationNumber(); log.info("Persisting {} eigenVectors and eigenValues to: {}", numEigenVectors, outputPath); Configuration conf = getConf() != null ? getConf() : new Configuration(); FileSystem fs = FileSystem.get(outputPath.toUri(), conf); SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, outputPath, IntWritable.class, VectorWritable.class); try { IntWritable iw = new IntWritable(); for (int i = 0; i < numEigenVectors; i++) { // Persist eigenvectors sorted by eigenvalues in descending order\ NamedVector v = new NamedVector(state.getRightSingularVector(numEigenVectors - 1 - i), "eigenVector" + i + ", eigenvalue = " + state.getSingularValue(numEigenVectors - 1 - i)); Writable vw = new VectorWritable(v); iw.set(i); seqWriter.append(iw, vw); } } finally { Closeables.close(seqWriter, false); } }
private void initializeM(Vector averageRatings) throws IOException { Random random = RandomUtils.getRandom(); FileSystem fs = FileSystem.get(pathToM(-1).toUri(), getConf()); SequenceFile.Writer writer = null; try { writer = new SequenceFile.Writer(fs, getConf(), new Path(pathToM(-1), "part-m-00000"), IntWritable.class, VectorWritable.class); IntWritable index = new IntWritable(); VectorWritable featureVector = new VectorWritable(); for (Vector.Element e : averageRatings.nonZeroes()) { Vector row = new DenseVector(numFeatures); row.setQuick(0, e.get()); for (int m = 1; m < numFeatures; m++) { row.setQuick(m, random.nextDouble()); } index.set(e.index()); featureVector.set(row); writer.append(index, featureVector); } } finally { Closeables.close(writer, false); } }
/** Write the list of labels into a map file */ public static int writeLabelIndex(Configuration conf, Iterable<String> labels, Path indexPath) throws IOException { FileSystem fs = FileSystem.get(indexPath.toUri(), conf); SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, indexPath, Text.class, IntWritable.class); int i = 0; try { for (String label : labels) { writer.append(new Text(label), new IntWritable(i++)); } } finally { Closeables.close(writer, false); } return i; }
public void serialize(Path output, Configuration conf) throws IOException { FileSystem fs = output.getFileSystem(conf); FSDataOutputStream out = fs.create(new Path(output, "naiveBayesModel.bin")); try { out.writeFloat(alphaI); VectorWritable.writeVector(out, weightsPerFeature); VectorWritable.writeVector(out, weightsPerLabel); VectorWritable.writeVector(out, perlabelThetaNormalizer); for (int row = 0; row < weightsPerLabelAndFeature.numRows(); row++) { VectorWritable.writeVector(out, weightsPerLabelAndFeature.viewRow(row)); } } finally { Closeables.close(out, false); } }
public static void write(Vector vector, Path path, Configuration conf, boolean laxPrecision) throws IOException { FileSystem fs = FileSystem.get(path.toUri(), conf); FSDataOutputStream out = fs.create(path); try { VectorWritable vectorWritable = new VectorWritable(vector); vectorWritable.setWritesLaxPrecision(laxPrecision); vectorWritable.write(out); } finally { Closeables.close(out, false); } }
public static Vector retrieveTimesSquaredOutputVector(Path outputVectorTmpPath, Configuration conf) throws IOException { Path outputFile = new Path(outputVectorTmpPath, OUTPUT_VECTOR_FILENAME + "/part-r-00000"); SequenceFileValueIterator<VectorWritable> iterator = new SequenceFileValueIterator<VectorWritable>(outputFile, true, conf); try { return iterator.next().get(); } finally { Closeables.close(iterator, true); } }
protected void persistVector(Path p, int key, Vector vector) throws IOException { SequenceFile.Writer writer = null; try { if (fs.exists(p)) { log.warn("{} exists, will overwrite", p); fs.delete(p, true); } writer = new SequenceFile.Writer(fs, conf, p, IntWritable.class, VectorWritable.class); writer.append(new IntWritable(key), new VectorWritable(vector)); } finally { Closeables.close(writer, false); } }
public static void storeWritable(Configuration conf, Path path, Writable writable) throws IOException { FileSystem fs = path.getFileSystem(conf); FSDataOutputStream out = fs.create(path); try { writable.write(out); } finally { Closeables.close(out, false); } }
private static <T> T read(FileInputStream stream, FileChannelFunction<T> function) throws IOException { try { return read(stream.getChannel(), function); } finally { Closeables.closeQuietly(stream); } }
public static void writeVectorsToSequenceFile(Iterable<? extends Vector> datapoints, Path path, Configuration conf) throws IOException { SequenceFile.Writer writer = null; try { writer = SequenceFile.createWriter(FileSystem.get(conf), conf, path, IntWritable.class, VectorWritable.class); int i = 0; for (Vector vector : datapoints) { writer.append(new IntWritable(i++), new VectorWritable(vector)); } } finally { Closeables.close(writer, true); } } }
public void testCloseQuietlyNull_inputStream() { Closeables.closeQuietly((InputStream) null); }