public RecordReader<Text, Text> createRecordReader(InputSplit genericSplit, TaskAttemptContext context) throws IOException { context.setStatus(genericSplit.toString()); return new KeyValueLineRecordReader(context.getConfiguration()); }
public static int findSeparator(byte[] utf, int start, int length, byte sep) { return org.apache.hadoop.mapreduce.lib.input. KeyValueLineRecordReader.findSeparator(utf, start, length, sep); }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.nextKeyValue()) { innerValue = lineRecordReader.getCurrentValue(); line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } int pos = findSeparator(line, 0, lineLen, this.separator); setKeyValue(key, value, line, lineLen, pos); return true; }
@Override public Function<String, Stream<Pair<String, Long>>> extractOutputFunction( String outputDir, Configuration conf) { return part -> ExceptionUtils.unchecked( () -> { try (final KeyValueLineRecordReader reader = new KeyValueLineRecordReader(conf)) { final Path path = new Path(outputDir + "/" + part); final TaskAttemptContext taskContext = HadoopUtils.createTaskContext(new Configuration(), HadoopUtils.getJobID(), 0); reader.initialize( new FileSplit(path, 0L, Long.MAX_VALUE, new String[] {"localhost"}), taskContext); final List<Pair<String, Long>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add( Pair.of( reader.getCurrentKey().toString(), Long.valueOf(reader.getCurrentValue().toString()))); } return result.stream(); } }); }
/** Read key/value pair in a line. */ public synchronized boolean next(Text key, Text value) throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.next(dummyKey, innerValue)) { line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; int pos = findSeparator(line, 0, lineLen, this.separator); org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader. setKeyValue(key, value, line, lineLen, pos); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.nextKeyValue()) { innerValue = lineRecordReader.getCurrentValue(); line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } int pos = findSeparator(line, 0, lineLen, this.separator); setKeyValue(key, value, line, lineLen, pos); return true; }
/** Read key/value pair in a line. */ public synchronized boolean next(Text key, Text value) throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.next(dummyKey, innerValue)) { line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; int pos = findSeparator(line, 0, lineLen, this.separator); org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader. setKeyValue(key, value, line, lineLen, pos); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.nextKeyValue()) { innerValue = lineRecordReader.getCurrentValue(); line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } int pos = findSeparator(line, 0, lineLen, this.separator); setKeyValue(key, value, line, lineLen, pos); return true; }
public RecordReader<Text, Text> createRecordReader(InputSplit genericSplit, TaskAttemptContext context) throws IOException { context.setStatus(genericSplit.toString()); return new KeyValueLineRecordReader(context.getConfiguration()); }
public static int findSeparator(byte[] utf, int start, int length, byte sep) { return org.apache.hadoop.mapreduce.lib.input. KeyValueLineRecordReader.findSeparator(utf, start, length, sep); }
/** Read key/value pair in a line. */ public synchronized boolean next(Text key, Text value) throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.next(dummyKey, innerValue)) { line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; int pos = findSeparator(line, 0, lineLen, this.separator); org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader. setKeyValue(key, value, line, lineLen, pos); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.nextKeyValue()) { innerValue = lineRecordReader.getCurrentValue(); line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } int pos = findSeparator(line, 0, lineLen, this.separator); setKeyValue(key, value, line, lineLen, pos); return true; }
public RecordReader<Text, Text> createRecordReader(InputSplit genericSplit, TaskAttemptContext context) throws IOException { context.setStatus(genericSplit.toString()); return new KeyValueLineRecordReader(context.getConfiguration()); }
public static int findSeparator(byte[] utf, int start, int length, byte sep) { return org.apache.hadoop.mapreduce.lib.input. KeyValueLineRecordReader.findSeparator(utf, start, length, sep); }
/** Read key/value pair in a line. */ public synchronized boolean next(Text key, Text value) throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.next(dummyKey, innerValue)) { line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; int pos = findSeparator(line, 0, lineLen, this.separator); org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader. setKeyValue(key, value, line, lineLen, pos); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.nextKeyValue()) { innerValue = lineRecordReader.getCurrentValue(); line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } int pos = findSeparator(line, 0, lineLen, this.separator); setKeyValue(key, value, line, lineLen, pos); return true; }
public RecordReader<Text, Text> createRecordReader(InputSplit genericSplit, TaskAttemptContext context) throws IOException { context.setStatus(genericSplit.toString()); return new KeyValueLineRecordReader(context.getConfiguration()); }
public static int findSeparator(byte[] utf, int start, int length, byte sep) { return org.apache.hadoop.mapreduce.lib.input. KeyValueLineRecordReader.findSeparator(utf, start, length, sep); }
/** Read key/value pair in a line. */ public synchronized boolean next(Text key, Text value) throws IOException { byte[] line = null; int lineLen = -1; if (lineRecordReader.next(dummyKey, innerValue)) { line = innerValue.getBytes(); lineLen = innerValue.getLength(); } else { return false; } if (line == null) return false; int pos = findSeparator(line, 0, lineLen, this.separator); org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader. setKeyValue(key, value, line, lineLen, pos); return true; }
public RecordReader<Text, Text> createRecordReader(InputSplit genericSplit, TaskAttemptContext context) throws IOException { context.setStatus(genericSplit.toString()); return new KeyValueLineRecordReader(context.getConfiguration()); }