@Override public void serialize(JobData jobData, JsonGenerator generator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException { generator.writeStartObject(); generator.writeFieldName("rowCount"); generator.writeNumber(jobData.getRowCount()); generator.writeFieldName("schema"); generator.writeStartArray(); for (Field field : jobData.getJobDataFragment().getSchema().getFields()) { writeField(field, generator, false); } generator.writeEndArray(); generator.writeFieldName("rows"); generator.writeStartArray(); final APIJobResultsSerializer jsonWriter = new APIJobResultsSerializer(generator); jsonWriter.setup(); for(RecordBatchHolder batchHolder : jobData.delegate.getRecordBatches()) { final EventBasedRecordWriter recordWriter = new EventBasedRecordWriter(batchHolder.getData().getContainer(), jsonWriter); for (int i = batchHolder.getStart(); i < batchHolder.getEnd(); i++) { recordWriter.writeOneRecord(i); } } generator.writeEndArray(); generator.writeEndObject(); } }
new EventBasedRecordWriter(batchHolder.getData().getContainer(), jsonWriter); for (int i = batchHolder.getStart(); i < batchHolder.getEnd(); i++) { context.setRowNum(currentRowInWriting++); recordWriter.writeOneRecord(i);
@Test public void writeAndReadEmptyListVectors() throws Exception { try (final VectorContainer batchData = createBatch(1, testEmptyListVector())) { final Path basePath = new Path(dateGenFolder.getRoot().getPath()); final ArrowFileMetadata metadata = writeArrowFile(batchData); try (final ArrowFileReader reader = new ArrowFileReader(FileSystem.get(FS_CONF), basePath, metadata, ALLOCATOR)) { final List<RecordBatchHolder> batchHolders = reader.read(0, 1); assertEquals(1, batchHolders.size()); assertNotNull(batchHolders.get(0).getData()); assertEquals(0, batchHolders.get(0).getStart()); assertEquals(1, batchHolders.get(0).getEnd()); final BatchSchema schema = batchHolders.get(0).getData().getContainer().getSchema(); assertEquals(1, schema.getFieldCount()); assertEquals("emptyListVector", schema.getColumn(0).getName()); assertEquals(MinorType.LIST, Types.getMinorTypeForArrowType(schema.getColumn(0).getType())); final VectorContainer batchContainer = batchHolders.get(0).getData().getContainer(); assertTrue(Iterators.size(batchContainer.iterator()) == 1); for (final VectorWrapper<?> wrapper : batchContainer) { assertTrue(wrapper.getValueVector() instanceof ListVector); assertTrue(((ListVector) (wrapper.getValueVector())).getDataVector() instanceof ZeroVector); } releaseBatches(batchHolders); } } }
/** * Helper method to verify that the batch holder contains valid data including the standard two columns * (colBit - BIT, colVarChar - VARCHAR) used in this test class. */ private static void verifyBatchHolder(RecordBatchHolder holder, int expStart, int expEnd) { assertNotNull(holder); assertNotNull(holder.getData()); assertEquals(expStart, holder.getStart()); assertEquals(expEnd, holder.getEnd()); // verify schema BatchSchema schema = holder.getData().getContainer().getSchema(); assertEquals(2, schema.getFieldCount()); assertEquals("colBit", schema.getColumn(0).getName()); assertEquals(MinorType.BIT, Types.getMinorTypeForArrowType(schema.getColumn(0).getType())); assertEquals("colVarChar", schema.getColumn(1).getName()); assertEquals(MinorType.VARCHAR, Types.getMinorTypeForArrowType(schema.getColumn(1).getType())); }