@Override public final int writeBatch(int offset, int length) throws IOException { if (this.eventBasedRecordWriter == null) { this.eventBasedRecordWriter = new EventBasedRecordWriter(incoming, this); } return eventBasedRecordWriter.write(offset, length); }
public EventBasedRecordWriter(VectorAccessible batch, RowBasedRecordWriter recordWriter) throws IOException { this.batch = batch; this.recordWriter = recordWriter; initFieldWriters(); }
@Override public void serialize(JobData jobData, JsonGenerator generator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException { generator.writeStartObject(); generator.writeFieldName("rowCount"); generator.writeNumber(jobData.getRowCount()); generator.writeFieldName("schema"); generator.writeStartArray(); for (Field field : jobData.getJobDataFragment().getSchema().getFields()) { writeField(field, generator, false); } generator.writeEndArray(); generator.writeFieldName("rows"); generator.writeStartArray(); final APIJobResultsSerializer jsonWriter = new APIJobResultsSerializer(generator); jsonWriter.setup(); for(RecordBatchHolder batchHolder : jobData.delegate.getRecordBatches()) { final EventBasedRecordWriter recordWriter = new EventBasedRecordWriter(batchHolder.getData().getContainer(), jsonWriter); for (int i = batchHolder.getStart(); i < batchHolder.getEnd(); i++) { recordWriter.writeOneRecord(i); } } generator.writeEndArray(); generator.writeEndObject(); } }
public MapJsonConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); int i = 0; for (String name : reader) { FieldReader fieldReader = reader.reader(name); FieldConverter converter = EventBasedRecordWriter.getConverter(JsonRecordWriter.this, i++, name, fieldReader.getMinorType(), fieldReader); converters.add(converter); } }
public UnionParquetConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); unionReader = (UnionReader) reader; NonNullableStructVector internalMap = new UnionVectorHelper(unionReader.data) .getInternalMap(); SingleStructReaderImpl mapReader = new SingleStructReaderImpl(internalMap); int i = 0; for (String name : mapReader) { FieldReader fieldReader = mapReader.reader(name); FieldConverter converter = EventBasedRecordWriter.getFieldConverter(ParquetRecordWriter .this, i, name, fieldReader.getMinorType(), unionReader); if (converter != null) { converterMap.put(name, converter); i++; } } }
for(RecordBatchHolder batchHolder : data.delegate.getRecordBatches()) { final EventBasedRecordWriter recordWriter = new EventBasedRecordWriter(batchHolder.getData().getContainer(), jsonWriter); recordWriter.writeOneRecord(i);
public MapParquetConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); int i = 0; for (String name : reader) { FieldReader fieldReader = reader.reader(name); FieldConverter converter = EventBasedRecordWriter.getConverter(ParquetRecordWriter.this, i, name, fieldReader.getMinorType(), fieldReader); if (converter != null) { converters.add(converter); i++; } } }
public static FieldConverter getConverter(final RowBasedRecordWriter recordWriter, final int fieldId, final String fieldName, final MinorType type, final FieldReader reader) { if (reader.getField().getFieldType().getType().getTypeID() == ArrowTypeID.Union) { return recordWriter.getNewUnionConverter(fieldId, fieldName, reader); } return getFieldConverter(recordWriter,fieldId, fieldName, type, reader); }
public ListJsonConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); FieldReader fieldReader = reader.reader(); innerConverter = EventBasedRecordWriter.getConverter(JsonRecordWriter.this, 0, "inner", fieldReader.getMinorType(), fieldReader); }
public ListParquetConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); int i = 0; FieldReader fieldReader = reader.reader(); innerConverter = (ParquetFieldConverter) EventBasedRecordWriter.getConverter(ParquetRecordWriter.this, i++, "element", fieldReader.getMinorType(), fieldReader); }
private void initFieldWriters() throws IOException { fieldConverters = Lists.newArrayList(); try { int fieldId = 0; for (VectorWrapper w : batch) { if (w.getField().getName().equalsIgnoreCase(WriterPrel.PARTITION_COMPARATOR_FIELD)) { continue; } FieldReader reader = w.getValueVector().getReader(); FieldConverter converter = getConverter(recordWriter, fieldId, w.getField().getName(), w.getValueVector().getMinorType(), reader); if (converter != null) { fieldId++; fieldConverters.add(converter); } } } catch(Exception e) { logger.error("Failed to create FieldWriter.", e); throw new IOException("Failed to initialize FieldWriters.", e); } }