/** {@inheritDoc} */ @Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { final FileSplit file = (FileSplit)inputSplit; context.setStatus(file.toString()); final AvroColumnReader.Params params = new AvroColumnReader.Params(new HadoopInput(file.getPath(), context.getConfiguration())); params.setModel(ReflectData.get()); if (AvroJob.getInputKeySchema(context.getConfiguration()) != null) { params.setSchema(AvroJob.getInputKeySchema(context.getConfiguration())); } reader = new AvroColumnReader<>(params); rows = reader.getRowCount(); }
/** * Tests serialization of wrapper and the wrapped native split. * @throws Exception If fails. */ @Test public void testSerialization() throws Exception { FileSplit nativeSplit = new FileSplit(new Path("/path/to/file"), 100, 500, new String[]{"host1", "host2"}); assertEquals("/path/to/file:100+500", nativeSplit.toString()); HadoopSplitWrapper split = HadoopUtils.wrapSplit(10, nativeSplit, nativeSplit.getLocations()); assertEquals("[host1, host2]", Arrays.toString(split.hosts())); ByteArrayOutputStream buf = new ByteArrayOutputStream(); ObjectOutput out = new ObjectOutputStream(buf); out.writeObject(split); ObjectInput in = new ObjectInputStream(new ByteArrayInputStream(buf.toByteArray())); final HadoopSplitWrapper res = (HadoopSplitWrapper)in.readObject(); assertEquals("/path/to/file:100+500", HadoopUtils.unwrapSplit(res).toString()); GridTestUtils.assertThrows(log, new Callable<Object>() { @Override public Object call() throws Exception { res.hosts(); return null; } }, AssertionError.class, null); }
public String toString() { return fs.toString(); }
public String toString() { return fs.toString(); }
public String toString() { return fs.toString(); }
public String toString() { return fs.toString(); }
public String toString() { return fs.toString(); }
context.setStatus(split.toString()); context.progress();
while (reader.nextKeyValue() == true) { parser.parse(reader.getCurrentKey(), reader.getCurrentValue(), writer, inputSplits.get(i).toString());