Refine search
@SuppressWarnings("deprecation") public static Map<String, String> readFromSequenceFile(Configuration conf, String inputPath) throws IOException { try (SequenceFile.Reader reader = new SequenceFile.Reader(getWorkingFileSystem(conf), new Path(inputPath), conf)) { Map<String, String> map = Maps.newHashMap(); Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf); Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) { map.put(key.toString(), value.toString()); } return map; } }
/** * Get token from the token sequence file. * @param authPath * @param proxyUserName * @return Token for proxyUserName if it exists. * @throws IOException */ private static Optional<Token<?>> getTokenFromSeqFile(String authPath, String proxyUserName) throws IOException { try (Closer closer = Closer.create()) { FileSystem localFs = FileSystem.getLocal(new Configuration()); SequenceFile.Reader tokenReader = closer.register(new SequenceFile.Reader(localFs, new Path(authPath), localFs.getConf())); Text key = new Text(); Token<?> value = new Token<>(); while (tokenReader.next(key, value)) { LOG.info("Found token for " + key); if (key.toString().equals(proxyUserName)) { return Optional.<Token<?>> of(value); } } } return Optional.absent(); } }
/** * Retrives a {@link Token} from a given sequence file for a specified user. The sequence file should contain a list * of key, value pairs where each key corresponds to a user and each value corresponds to a {@link Token} for that * user. * * @param userNameKey The name of the user to retrieve a {@link Token} for * @param tokenFilePath The path to the sequence file containing the {@link Token}s * * @return A {@link Token} for the given user name */ public static Optional<Token<?>> getTokenFromSeqFile(String userNameKey, Path tokenFilePath) throws IOException { log.info("Reading tokens from sequence file " + tokenFilePath); try (Closer closer = Closer.create()) { FileSystem localFs = FileSystem.getLocal(new Configuration()); @SuppressWarnings("deprecation") SequenceFile.Reader tokenReader = closer.register(new SequenceFile.Reader(localFs, tokenFilePath, localFs.getConf())); Text key = new Text(); Token<?> value = new Token<>(); while (tokenReader.next(key, value)) { log.debug("Found token for user: " + key); if (key.toString().equals(userNameKey)) { return Optional.<Token<?>> of(value); } } } log.warn("Did not find any tokens for user " + userNameKey); return Optional.absent(); }
protected Vector fetchVector(Path p, int keyIndex) throws IOException { if (!fs.exists(p)) { return null; } SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf); IntWritable key = new IntWritable(); VectorWritable vw = new VectorWritable(); while (reader.next(key, vw)) { if (key.get() == keyIndex) { return vw.get(); } } return null; }
public List<InputSplit> getSplits(JobContext job) throws IOException { Configuration conf = job.getConfiguration(); Path src = new Path(conf.get(INDIRECT_INPUT_FILE, null)); FileSystem fs = src.getFileSystem(conf); List<InputSplit> splits = new ArrayList<InputSplit>(); LongWritable key = new LongWritable(); Text value = new Text(); for (SequenceFile.Reader sl = new SequenceFile.Reader(fs, src, conf); sl.next(key, value);) { splits.add(new IndirectSplit(new Path(value.toString()), key.get())); } return splits; }
private void readPartitionFile(FileSystem fs, Configuration conf, Path path) throws IOException { @SuppressWarnings("deprecation") SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); ImmutableBytesWritable key = new ImmutableBytesWritable(); partitions = new ArrayList<ImmutableBytesWritable>(); while (reader.next(key)) { partitions.add(new ImmutableBytesWritable(key.copyBytes())); } reader.close(); if (!Ordering.natural().isOrdered(partitions)) { throw new IOException("Partitions are not ordered!"); } }
private static <K extends Writable, V extends Writable> List<Pair<K, V>> readSequenceFile(Path path, Class<K> keyClass, Class<V> valueClass) throws IOException { List<Pair<K, V>> records = Lists.newArrayList(); Configuration conf = new Configuration(); try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(path))) { K key = ReflectionUtils.newInstance(keyClass, conf); V value = ReflectionUtils.newInstance(valueClass, conf); while (reader.next(key, value)) { records.add(new ImmutablePair<K, V>(key, value)); key = ReflectionUtils.newInstance(keyClass, conf); value = ReflectionUtils.newInstance(valueClass, conf); } } return records; }
private Map<Text, CopyListingFileStatus> getListing(Path listingPath) throws Exception { SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(listingPath)); Text key = new Text(); CopyListingFileStatus value = new CopyListingFileStatus(); Map<Text, CopyListingFileStatus> values = new HashMap<>(); while (reader.next(key, value)) { values.put(key, value); key = new Text(); value = new CopyListingFileStatus(); } return values; }
protected Vector fetchVector(Path p, int keyIndex) throws IOException { if (!fs.exists(p)) { return null; } SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf); IntWritable key = new IntWritable(); VectorWritable vw = new VectorWritable(); while (reader.next(key, vw)) { if (key.get() == keyIndex) { return vw.get(); } } return null; }
private static Vector loadVector(Configuration conf, Path path) throws IOException { FileSystem fs = path.getFileSystem(conf); SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); Writable key = new IntWritable(); VectorWritable value = new VectorWritable(); try { if (!reader.next(key, value)) { throw new IOException("Input vector file is empty."); } return value.get(); } finally { reader.close(); } }
private void readPartitionFile(FileSystem fs, Configuration conf, Path path) throws IOException { @SuppressWarnings("deprecation") SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); ImmutableBytesWritable key = new ImmutableBytesWritable(); partitions = new ArrayList<>(); while (reader.next(key)) { partitions.add(new ImmutableBytesWritable(key.copyBytes())); } reader.close(); if (!Ordering.natural().isOrdered(partitions)) { throw new IOException("Partitions are not ordered!"); } }
private void verifyAll4Values(Path file) throws IOException { Reader reader = new Reader(conf, Reader.file(file)); assertEquals(1L, reader.next((Object) null)); assertEquals("one", reader.getCurrentValue((Object) null)); assertEquals(2L, reader.next((Object) null)); assertEquals("two", reader.getCurrentValue((Object) null)); assertEquals(3L, reader.next((Object) null)); assertEquals("three", reader.getCurrentValue((Object) null)); assertEquals(4L, reader.next((Object) null)); assertEquals("four", reader.getCurrentValue((Object) null)); assertNull(reader.next((Object) null)); reader.close(); } }
private void readPartitionFile(FileSystem fs, Configuration conf, Path path) throws IOException { @SuppressWarnings("deprecation") SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); ImmutableBytesWritable key = new ImmutableBytesWritable(); partitions = new ArrayList<>(); while (reader.next(key)) { partitions.add(new ImmutableBytesWritable(key.copyBytes())); } reader.close(); if (!Ordering.natural().isOrdered(partitions)) { throw new IOException("Partitions are not ordered!"); } }
private void verifyAll4Values(Path file) throws IOException { Reader reader = new Reader(conf, Reader.file(file)); assertEquals(1L, reader.next((Object) null)); assertEquals("one", reader.getCurrentValue((Object) null)); assertEquals(2L, reader.next((Object) null)); assertEquals("two", reader.getCurrentValue((Object) null)); assertEquals(3L, reader.next((Object) null)); assertEquals("three", reader.getCurrentValue((Object) null)); assertEquals(4L, reader.next((Object) null)); assertEquals("four", reader.getCurrentValue((Object) null)); assertNull(reader.next((Object) null)); reader.close(); } }
private static int getNumTerms(Configuration conf, Path dictionaryPath) throws IOException { FileSystem fs = dictionaryPath.getFileSystem(conf); Text key = new Text(); IntWritable value = new IntWritable(); int maxTermId = -1; for (FileStatus stat : fs.globStatus(dictionaryPath)) { SequenceFile.Reader reader = new SequenceFile.Reader(fs, stat.getPath(), conf); while (reader.next(key, value)) { maxTermId = Math.max(maxTermId, value.get()); } } return maxTermId + 1; }
public static AdmmReducerContext readPreviousAdmmReducerContext( Path previousIntermediateOutputLocationPath, FileSystem fs, Configuration conf) throws IOException { Path previousZPath = new Path(previousIntermediateOutputLocationPath, "<Z>"); SequenceFile.Reader reader = new SequenceFile.Reader(fs, previousZPath, conf); AdmmReducerContextWritable reduceContextWritable = new AdmmReducerContextWritable(); Writable key = NullWritable.get(); reader.next(key, reduceContextWritable); reader.close(); return reduceContextWritable.get(); }
private static int getNumTerms(Configuration conf, Path dictionaryPath) throws IOException { FileSystem fs = dictionaryPath.getFileSystem(conf); Text key = new Text(); IntWritable value = new IntWritable(); int maxTermId = -1; for (FileStatus stat : fs.globStatus(dictionaryPath)) { SequenceFile.Reader reader = new SequenceFile.Reader(fs, stat.getPath(), conf); while (reader.next(key, value)) { maxTermId = Math.max(maxTermId, value.get()); } } return maxTermId + 1; }
private static int getNumTerms(Configuration conf, Path dictionaryPath) throws IOException { FileSystem fs = dictionaryPath.getFileSystem(conf); Text key = new Text(); IntWritable value = new IntWritable(); int maxTermId = -1; for (FileStatus stat : fs.globStatus(dictionaryPath)) { SequenceFile.Reader reader = new SequenceFile.Reader(fs, stat.getPath(), conf); while (reader.next(key, value)) { maxTermId = Math.max(maxTermId, value.get()); } } return maxTermId + 1; }
public static void main(String[] args) throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); SequenceFile.Reader r = new SequenceFile.Reader(fs, new Path(args[0]), conf); Text routingKey = new Text(); MessageWritable message = new MessageWritable(); while (r.next(routingKey, message)) { System.out.println("###routing key: " + routingKey); System.out.println(SerDeFactory.create(args[1]).deserialize(message.getMessage().getPayload())); } r.close(); } }
public MDXSeqIterator(File seq) throws IOException, InstantiationException, IllegalAccessException { Configuration config = new Configuration(); Path path = new Path(seq.getAbsolutePath()); reader = new SequenceFile.Reader(FileSystem.get(config), path, config); key = (Text) reader.getKeyClass().newInstance(); value = (Text) reader.getValueClass().newInstance(); // Queue up: hasNext = reader.next(key, value); }