@Override public void process(InputStream in, OutputStream out) throws IOException { try (CSVFileReader<Record> reader = new CSVFileReader<>( in, props, schema, Record.class)) { reader.initialize(); try (DataFileWriter<Record> w = writer.create(schema, out)) { while (reader.hasNext()) { try { Record record = reader.next(); w.append(record); written.incrementAndGet(); } catch (DatasetRecordException e) { failures.add(e); } } } } } });
@Override public void process(InputStream in, OutputStream out) throws IOException { try (CSVFileReader<Record> reader = new CSVFileReader<>( in, props, schema, Record.class)) { reader.initialize(); try (DataFileWriter<Record> w = writer.create(schema, out)) { while (reader.hasNext()) { try { Record record = reader.next(); w.append(record); written.incrementAndGet(); } catch (DatasetRecordException e) { failures.add(e); } } } } } });
Assert.assertTrue(reader.hasNext()); TestBean bean = reader.next(); Assert.assertEquals("str", bean.myStr); Assert.assertEquals(false, bean.myBool); Assert.assertTrue(reader.hasNext()); bean = reader.next(); Assert.assertEquals("str,2", bean.myStr); Assert.assertEquals(true, bean.myBool); Assert.assertTrue(reader.hasNext()); TestHelpers.assertThrows("Should complain about null as a number", DatasetRecordException.class, new Runnable() { Assert.assertTrue(reader.hasNext()); bean = reader.next(); Assert.assertEquals("str4", bean.myStr); Assert.assertFalse(reader.hasNext());
Assert.assertTrue(reader.hasNext()); TestBean bean = reader.next(); Assert.assertEquals("str", bean.myStr); Assert.assertEquals(false, bean.myBool); Assert.assertTrue(reader.hasNext()); bean = reader.next(); Assert.assertEquals("str,2", bean.myStr); Assert.assertEquals(true, bean.myBool); Assert.assertTrue(reader.hasNext()); TestHelpers.assertThrows("Should complain about null as a number", DatasetRecordException.class, new Runnable() { Assert.assertTrue(reader.hasNext()); bean = reader.next(); Assert.assertEquals("str4", bean.myStr); Assert.assertFalse(reader.hasNext());
Assert.assertTrue(reader.hasNext()); GenericData.Record rec = reader.next(); Assert.assertEquals("str", rec.get(0)); Assert.assertEquals(false, rec.get(3)); Assert.assertTrue(reader.hasNext()); rec = reader.next(); Assert.assertEquals("str,2", rec.get(0)); Assert.assertEquals(true, rec.get(3)); Assert.assertTrue(reader.hasNext()); TestHelpers.assertThrows("Should complain about null as a number", DatasetRecordException.class, new Runnable() { Assert.assertTrue(reader.hasNext()); TestHelpers.assertThrows("Should complain about missing default", DatasetRecordException.class, new Runnable() { Assert.assertFalse(reader.hasNext());
Assert.assertTrue(reader.hasNext()); GenericData.Record rec = reader.next(); Assert.assertEquals("str", rec.get(0)); Assert.assertEquals(false, rec.get(3)); Assert.assertTrue(reader.hasNext()); rec = reader.next(); Assert.assertEquals("str,2", rec.get(0)); Assert.assertEquals(true, rec.get(3)); Assert.assertTrue(reader.hasNext()); TestHelpers.assertThrows("Should complain about null as a number", DatasetRecordException.class, new Runnable() { Assert.assertTrue(reader.hasNext()); TestHelpers.assertThrows("Should complain about missing default", DatasetRecordException.class, new Runnable() { Assert.assertFalse(reader.hasNext());
Assert.assertTrue(reader.hasNext()); TestGenericRecord record = reader.next(); Assert.assertEquals("str", record.get(0)); Assert.assertEquals(false, record.get(3)); Assert.assertTrue(reader.hasNext()); record = reader.next(); Assert.assertEquals("str,2", record.get(0)); Assert.assertEquals(true, record.get(3)); Assert.assertTrue(reader.hasNext()); TestHelpers.assertThrows("Should complain about null as a number", DatasetRecordException.class, new Runnable() { Assert.assertTrue(reader.hasNext()); TestHelpers.assertThrows("Should complain about missing default", DatasetRecordException.class, new Runnable() { Assert.assertFalse(reader.hasNext());
@Test public void testBadNumericSchema() { final DatasetDescriptor desc = new DatasetDescriptor.Builder() .schema(TYPE_ERROR_SCHEMA) .build(); final CSVFileReader<GenericData.Record> reader = new CSVFileReader<GenericData.Record>(localfs, csvFile, desc, DataModelUtil.accessor(GenericData.Record.class, desc.getSchema())); reader.initialize(); Assert.assertTrue(reader.hasNext()); TestHelpers.assertThrows("Should reject float value for integer schema", DatasetRecordException.class, new Runnable() { @Override public void run() { reader.next(); } }); }
Assert.assertTrue(reader.hasNext()); GenericData.Record rec = reader.next(); Assert.assertEquals("str", rec.get(0)); Assert.assertEquals("false", rec.get(3)); Assert.assertTrue(reader.hasNext()); rec = reader.next(); Assert.assertEquals("str\t2", rec.get(0)); Assert.assertEquals("true", rec.get(3)); Assert.assertTrue(reader.hasNext()); rec = reader.next(); Assert.assertEquals("str3", rec.get(0)); Assert.assertEquals("missing value", rec.get(3)); Assert.assertFalse(reader.hasNext());
Assert.assertTrue(reader.hasNext()); GenericData.Record rec = reader.next(); Assert.assertEquals("str", rec.get(0)); Assert.assertEquals("false", rec.get(3)); Assert.assertTrue(reader.hasNext()); rec = reader.next(); Assert.assertEquals("str,2", rec.get(0)); Assert.assertEquals("true", rec.get(3)); Assert.assertTrue(reader.hasNext()); rec = reader.next(); Assert.assertEquals("str3", rec.get(0)); Assert.assertEquals("missing value", rec.get(3)); Assert.assertTrue(reader.hasNext()); rec = reader.next(); Assert.assertEquals("str4", rec.get(0)); Assert.assertEquals("", rec.get(3)); Assert.assertFalse(reader.hasNext());
Assert.assertTrue(reader.hasNext()); GenericData.Record rec = reader.next(); Assert.assertEquals("str", rec.get(0)); Assert.assertEquals("false", rec.get(3)); Assert.assertTrue(reader.hasNext()); rec = reader.next(); Assert.assertEquals("str\t2", rec.get(0)); Assert.assertEquals("true", rec.get(3)); Assert.assertTrue(reader.hasNext()); rec = reader.next(); Assert.assertEquals("str3", rec.get(0)); Assert.assertEquals("missing value", rec.get(3)); Assert.assertFalse(reader.hasNext());