public SequenceFileReader(FileSystem fs, Path file, Map<String, Object> conf) throws IOException { super(fs, file); int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString()); this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize)); this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf()); this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf()); this.offset = new SequenceFileReader.Offset(0, 0, 0); }
public SequenceFileReader(FileSystem fs, Path file, Map<String, Object> conf, String offset) throws IOException { super(fs, file); int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString()); this.offset = new SequenceFileReader.Offset(offset); this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize)); this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf()); this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf()); skipToOffset(this.reader, this.offset, this.key); }
@SuppressWarnings("deprecation") public static Map<String, String> readFromSequenceFile(Configuration conf, String inputPath) throws IOException { try (SequenceFile.Reader reader = new SequenceFile.Reader(getWorkingFileSystem(conf), new Path(inputPath), conf)) { Map<String, String> map = Maps.newHashMap(); Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf); Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) { map.put(key.toString(), value.toString()); } return map; } }
Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf); Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);
Mockito.any(Configuration.class)).thenReturn(reader); Mockito.<Class<?>>when(reader.getKeyClass()).thenReturn( (Class<?>) LongWritable.class); Mockito.<Class<?>>when(reader.getValueClass()).thenReturn(
if (key != null && key.getClass() != getKeyClass()) { throw new IOException("wrong key class: "+key.getClass().getName() +" is not "+keyClass);
if (key.getClass() != getKeyClass()) throw new IOException("wrong key class: "+key.getClass().getName() +" is not "+keyClass);
/** Fills up the rawKey object with the key returned by the Reader * @return true if there is a key returned; false, otherwise * @throws IOException */ public boolean nextRawKey() throws IOException { if (in == null) { int bufferSize = getBufferSize(conf); Reader reader = new Reader(conf, Reader.file(segmentPathName), Reader.bufferSize(bufferSize), Reader.start(segmentOffset), Reader.length(segmentLength)); //sometimes we ignore syncs especially for temp merge files if (ignoreSync) reader.ignoreSync(); if (reader.getKeyClass() != keyClass) throw new IOException("wrong key class: " + reader.getKeyClass() + " is not " + keyClass); if (reader.getValueClass() != valClass) throw new IOException("wrong value class: "+reader.getValueClass()+ " is not " + valClass); this.in = reader; rawKey = new DataOutputBuffer(); } rawKey.reset(); int keyLength = in.nextRawKey(rawKey); return (keyLength >= 0); }
/** * Check if there's partition files for hfile, if yes replace the table splits, to make the job more reducers * @param conf the job configuration * @param path the hfile partition file * @throws IOException */ @SuppressWarnings("deprecation") private void reconfigurePartitions(Configuration conf, Path path) throws IOException { FileSystem fs = path.getFileSystem(conf); if (fs.exists(path)) { try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf)) { int partitionCount = 0; Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) { partitionCount++; } TotalOrderPartitioner.setPartitionFile(job.getConfiguration(), path); // The reduce tasks should be one more than partition keys job.setNumReduceTasks(partitionCount + 1); } } else { logger.info("File '" + path.toString() + " doesn't exist, will not reconfigure hfile Partitions"); } }
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
public CubeStatsResult(Path path, int precision) throws IOException { Configuration hadoopConf = HadoopUtil.getCurrentConfiguration(); Option seqInput = SequenceFile.Reader.file(path); try (Reader reader = new SequenceFile.Reader(hadoopConf, seqInput)) { LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), hadoopConf); BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), hadoopConf); while (reader.next(key, value)) { if (key.get() == 0L) { percentage = Bytes.toInt(value.getBytes()); } else if (key.get() == -1) { mapperOverlapRatio = Bytes.toDouble(value.getBytes()); } else if (key.get() == -2) { mapperNumber = Bytes.toInt(value.getBytes()); } else if (key.get() == -3) { sourceRecordCount = Bytes.toLong(value.getBytes()); } else if (key.get() > 0) { HLLCounter hll = new HLLCounter(precision); ByteArray byteArray = new ByteArray(value.getBytes()); hll.readRegisters(byteArray.asBuffer()); counterMap.put(key.get(), hll); } } } }
public void printFile(String path) throws Exception { FileSystem fileSystem = FileUtil.getFileSystem(path); Path fsPath = new Path(path); SequenceFile.Reader reader = new SequenceFile.Reader(fileSystem, fsPath, new Configuration()); LongWritable key = (LongWritable) reader.getKeyClass().newInstance(); BytesWritable value = (BytesWritable) reader.getValueClass().newInstance(); System.out.println("reading file " + path); while (reader.next(key, value)) { if (mPrintOffsetsOnly) { System.out.println(Long.toString(key.get())); } else { byte[] nonPaddedBytes = new byte[value.getLength()]; System.arraycopy(value.getBytes(), 0, nonPaddedBytes, 0, value.getLength()); System.out.println(Long.toString(key.get()) + ": " + new String(nonPaddedBytes)); } } } }
public SequenceFileReader(SequenceFile.Reader reader, Configuration conf) { this.reader = reader; this.schema = Pair.getPairSchema(WritableData.get().getSchema(reader.getKeyClass()), WritableData.get().getSchema(reader.getValueClass())); this.key = (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf); this.spareKey = (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf); this.value = (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf); if (WRITABLE_CONVERTERS.containsKey(reader.getKeyClass()) ) keyConverter = WRITABLE_CONVERTERS.get(reader.getKeyClass()); if (WRITABLE_CONVERTERS.containsKey(reader.getValueClass()) ) valConverter = WRITABLE_CONVERTERS.get(reader.getValueClass()); }
/** Returns the class of keys in this file. */ public Class<?> getKeyClass() { return data.getKeyClass(); }
@Ignore("convenient trial tool for dev") @Test public void test() throws IOException, InterruptedException { Configuration hconf = HadoopUtil.getCurrentConfiguration(); HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper(); Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null); mapper.doSetup(context); Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath)); Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf); Text value = new Text(); while (reader.next(key, value)) { mapper.map(key, value, context); } reader.close(); }
public SequenceFileReader(LogFilePath path) throws Exception { Configuration config = new Configuration(); Path fsPath = new Path(path.getLogFilePath()); FileSystem fs = FileUtil.getFileSystem(path.getLogFilePath()); this.mReader = new SequenceFile.Reader(fs, fsPath, config); this.mKey = (LongWritable) mReader.getKeyClass().newInstance(); this.mValue = (BytesWritable) mReader.getValueClass().newInstance(); }
public TextRecordInputStream(FileStatus f) throws IOException { final Path fpath = f.getPath(); final Configuration lconf = getConf(); r = new SequenceFile.Reader(lconf, SequenceFile.Reader.file(fpath)); key = ReflectionUtils.newInstance( r.getKeyClass().asSubclass(Writable.class), lconf); val = ReflectionUtils.newInstance( r.getValueClass().asSubclass(Writable.class), lconf); inbuf = new DataInputBuffer(); outbuf = new DataOutputBuffer(); }
public MessagePackSequenceFileReader(LogFilePath path) throws Exception { Configuration config = new Configuration(); Path fsPath = new Path(path.getLogFilePath()); FileSystem fs = FileUtil.getFileSystem(path.getLogFilePath()); this.mReader = new SequenceFile.Reader(fs, fsPath, config); this.mKey = (BytesWritable) mReader.getKeyClass().newInstance(); this.mValue = (BytesWritable) mReader.getValueClass().newInstance(); }
SeqRowReader(Configuration hconf, String path) throws IOException { reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path))); key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf); value = new Text(); }
private List<String> getSeqFileContents(FileSystem fs, Path... seqFiles) throws IOException { ArrayList<String> result = new ArrayList<>(); for (Path seqFile : seqFiles) { Path file = new Path(fs.getUri().toString() + seqFile.toString()); SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(file)); try { Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) { String keyValStr = Arrays.asList(key, value).toString(); result.add(keyValStr); } } finally { reader.close(); } }// for return result; }