@Override public void process(OutputStream out) throws IOException { DataOutputStream dos = new DataOutputStream(out); dos.writeInt(key.getLength()); dos.write(key.getBytes(), 0, key.getLength()); final OutputStreamWritable fileData = new OutputStreamWritable(out, true); reader.getCurrentValue(fileData); }
@Override public Void call() throws Exception { Configuration conf = new Configuration(ParallelRunner.this.fs.getConf()); WritableShimSerialization.addToHadoopConfiguration(conf); try (@SuppressWarnings("deprecation") SequenceFile.Reader reader = new SequenceFile.Reader( ParallelRunner.this.fs, inputFilePath, conf)) { Writable key = keyClass.newInstance(); T state = stateClass.newInstance(); while (reader.next(key)) { state = (T) reader.getCurrentValue(state); states.add(state); state = stateClass.newInstance(); } if (deleteAfter) { HadoopUtils.deletePath(ParallelRunner.this.fs, inputFilePath, false); } } return null; } }), "Deserialize state from file " + inputFilePath));
/** Read the next key/value pair in the file into <code>key</code> and * <code>val</code>. Returns true if such a pair exists and false when at * end of file */ public synchronized boolean next(Writable key, Writable val) throws IOException { if (val.getClass() != getValueClass()) throw new IOException("wrong value class: "+val+" is not "+valClass); boolean more = next(key); if (more) { getCurrentValue(val); } return more; }
/** Return the value for the named key, or null if none exists. */ public synchronized Writable get(WritableComparable key, Writable val) throws IOException { if (seek(key)) { data.getCurrentValue(val); return val; } else return null; }
private static void printSequenceFile(FileSystem fs, Path p, Configuration conf) throws IOException { SequenceFile.Reader r = new SequenceFile.Reader(fs, p, conf); Object key = null; Object value = null; while ((key = r.next(key)) != null) { value = r.getCurrentValue(value); System.out.println(" Row: " + key + ", " + value); } r.close(); }
/** Return the value for the named key, or null if none exists. */ public synchronized Writable get(WritableComparable key, Writable val) throws IOException { if (seek(key)) { data.getCurrentValue(val); return val; } else return null; }
private void verify2Values(Path file) throws IOException { Reader reader = new Reader(conf, Reader.file(file)); assertEquals(1L, reader.next((Object) null)); assertEquals("one", reader.getCurrentValue((Object) null)); assertEquals(2L, reader.next((Object) null)); assertEquals("two", reader.getCurrentValue((Object) null)); assertNull(reader.next((Object) null)); reader.close(); }
private void verify2Values(Path file) throws IOException { Reader reader = new Reader(conf, Reader.file(file)); assertEquals(1L, reader.next((Object) null)); assertEquals("one", reader.getCurrentValue((Object) null)); assertEquals(2L, reader.next((Object) null)); assertEquals("two", reader.getCurrentValue((Object) null)); assertNull(reader.next((Object) null)); reader.close(); }
/** Return the value for the named key, or null if none exists. */ public Writable get(WritableComparable key, Writable val) throws IOException { long position = findPosition(key); if (position >= 0) { SequenceFile.Reader threadLocalData = data.get(); threadLocalData.seek(position); threadLocalData.getCurrentValue(val); return val; } else return null; }
private static void printSequenceFile(FileSystem fs, Path p, Configuration conf) throws IOException { SequenceFile.Reader r = new SequenceFile.Reader(fs, p, conf); Object key = null; Object value = null; while ((key = r.next(key)) != null) { value = r.getCurrentValue(value); System.out.println(" Row: " + key + ", " + value); } r.close(); }
private static void printSequenceFile(FileSystem fs, Path p, Configuration conf) throws IOException { SequenceFile.Reader r = new SequenceFile.Reader(fs, p, conf); Object key = null; Object value = null; while ((key = r.next(key)) != null) { value = r.getCurrentValue(value); System.out.println(" Row: " + key + ", " + value); } r.close(); }
/** Return the value for the named key, or null if none exists. */ public synchronized Writable get(WritableComparable key, Writable val) throws IOException { if (seek(key)) { data.getCurrentValue(val); return val; } else return null; }
assertNotNull(key); assertEquals("one", key.datum().toString()); value = (AvroValue<Integer>) reader.getCurrentValue(value); assertNotNull(value); assertEquals(1, value.datum().intValue()); assertNotNull(key); assertEquals("two", key.datum().toString()); value = (AvroValue<Integer>) reader.getCurrentValue(value); assertNotNull(value); assertEquals(2, value.datum().intValue());
assertNotNull(key); assertEquals("one", key.datum().toString()); value = (AvroValue<Integer>) reader.getCurrentValue(value); assertNotNull(value); assertEquals(1, value.datum().intValue()); assertNotNull(key); assertEquals("two", key.datum().toString()); value = (AvroValue<Integer>) reader.getCurrentValue(value); assertNotNull(value); assertEquals(2, value.datum().intValue());
sanitizeKeyForComparison ? sanitizeDatasetStatestoreNameFromDatasetURN(storeName, key.toString()) : key.toString(); writable = reader.getCurrentValue(writable); if (stringKey.equals(stateId)) { if (writable instanceof JobState.DatasetState) {
/** Tests that reading and writing ordinary Writables still works. */ @Test public void testReadWritables() throws IOException { Path sequenceFilePath = new Path(new File(mTempDir.getRoot(), "output.seq").getPath()); writeSequenceFile(sequenceFilePath, Text.class, IntWritable.class, null, null, new Text("one"), new IntWritable(1), new Text("two"), new IntWritable(2)); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); AvroSequenceFile.Reader.Options options = new AvroSequenceFile.Reader.Options() .withFileSystem(fs) .withInputPath(sequenceFilePath) .withConfiguration(conf); SequenceFile.Reader reader = new AvroSequenceFile.Reader(options); Text key = new Text(); IntWritable value = new IntWritable(); // Read the first record. assertTrue(reader.next(key)); assertEquals("one", key.toString()); reader.getCurrentValue(value); assertNotNull(value); assertEquals(1, value.get()); // Read the second record. assertTrue(reader.next(key)); assertEquals("two", key.toString()); reader.getCurrentValue(value); assertNotNull(value); assertEquals(2, value.get()); assertFalse("Should be no more records.", reader.next(key)); }
/** * Finds the record that is the closest match to the specified key. * * @param key - key that we're trying to find * @param val - data value if key is found * @param before - IF true, and <code>key</code> does not exist, return * the first entry that falls just before the <code>key</code>. Otherwise, * return the record that sorts just after. * @return - the key that was the closest match or null if eof. */ public synchronized WritableComparable getClosest(WritableComparable key, Writable val, final boolean before) throws IOException { int c = seekInternal(key, before); // If we didn't get an exact match, and we ended up in the wrong // direction relative to the query key, return null since we // must be at the beginning or end of the file. if ((!before && c > 0) || (before && c < 0)) { return null; } data.getCurrentValue(val); return nextKey; }
@Override public Pair<K,V> next(Pair<K,V> reuse) throws IOException { prepare(); if (!hasNext()) throw new NoSuchElementException(); Pair<K,V> result = reuse; if (result == null) result = new Pair<>(schema); result.key(keyConverter.convert(key)); reader.getCurrentValue(value); result.value(valConverter.convert(value)); // swap key and spareKey Writable k = key; key = spareKey; spareKey = k; ready = false; return result; }
@Override public void process(OutputStream out) throws IOException { final OutputStreamWritable fileData = new OutputStreamWritable(out, false); reader.getCurrentValue(fileData); } }
/** Return the value for the named key, or null if none exists. */ public synchronized Writable get(WritableComparable key, Writable val) throws IOException { if (seek(key)) { data.getCurrentValue(val); return val; } else return null; }