public MinorType getMinorType() { return reader.getMinorType(); }
public MapParquetConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); int i = 0; for (String name : reader) { FieldReader fieldReader = reader.reader(name); FieldConverter converter = EventBasedRecordWriter.getConverter(ParquetRecordWriter.this, i, name, fieldReader.getMinorType(), fieldReader); if (converter != null) { converters.add(converter); i++; } } }
public MapJsonConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); int i = 0; for (String name : reader) { FieldReader fieldReader = reader.reader(name); FieldConverter converter = EventBasedRecordWriter.getConverter(JsonRecordWriter.this, i++, name, fieldReader.getMinorType(), fieldReader); converters.add(converter); } }
public void eval() { org.apache.arrow.vector.types.Types.MinorType type1; if (input1.isSet()) { type1 = input1.getMinorType(); } else { type1 = org.apache.arrow.vector.types.Types.MinorType.NULL; } org.apache.arrow.vector.types.Types.MinorType type2; if (input2.isSet()) { type2 = input2.getMinorType(); } else { type2 = org.apache.arrow.vector.types.Types.MinorType.NULL; } out.isSet = 1; out.value = com.dremio.exec.expr.fn.impl.UnionFunctions.compareTypes(type1, type2); } }
public void writeUnion(FieldReader reader, JsonOutputContext context) throws IOException { UNION_FIELD_WRITERS.get(getMinorTypeFromArrowMinorType(reader.getMinorType())).write(this, reader, context); }
public ListJsonConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); FieldReader fieldReader = reader.reader(); innerConverter = EventBasedRecordWriter.getConverter(JsonRecordWriter.this, 0, "inner", fieldReader.getMinorType(), fieldReader); }
public ListParquetConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); int i = 0; FieldReader fieldReader = reader.reader(); innerConverter = (ParquetFieldConverter) EventBasedRecordWriter.getConverter(ParquetRecordWriter.this, i++, "element", fieldReader.getMinorType(), fieldReader); }
@Override public void eval() { out.isSet = 1; if (!in.isSet() || in.readObject() == null) { out.value = 0; return; } if (in.getMinorType() != org.apache.arrow.vector.types.Types.MinorType.LIST) { throw new java.lang.UnsupportedOperationException( String.format("'array_length' is supported only on LIST type input. Given input type : %s", in.getMinorType().toString() ) ); } out.value = ((java.util.List<?>) in.readObject()).size(); } }
@Override public void eval() { if (in.isSet == 0) { out.isSet = 0; return; } final com.dremio.dac.proto.model.dataset.DataType booleanType = com.dremio.dac.proto.model.dataset.DataType.BOOLEAN; final com.dremio.dac.proto.model.dataset.DataType dataType = com.dremio.dac.explore.DataTypeUtil.getDataType(in.reader.getMinorType()); if (((dataType == booleanType) || castWhenPossibleHolder.value == 1) && com.dremio.dac.explore.udfs.CleanData.castToBoolean(in.reader, out)) { out.isSet = 1; } else { if (replaceWithNullHolder.value == 1) { out.isSet = 0; } else { out.value = defaultValueHolder.value; out.isSet = 1; } } } }
@Override public void eval() { if (in.isSet == 0) { out.isSet = 0; return; } final com.dremio.dac.proto.model.dataset.DataType floatType = com.dremio.dac.proto.model.dataset.DataType.FLOAT; final com.dremio.dac.proto.model.dataset.DataType dataType = com.dremio.dac.explore.DataTypeUtil.getDataType(in.reader.getMinorType()); if (((dataType == floatType) || castWhenPossibleHolder.value == 1) && com.dremio.dac.explore.udfs.CleanData.castToFloat(in.reader, out)) { out.isSet = 1; } else { if (replaceWithNullHolder.value == 1) { out.isSet = 0; } else { out.value = defaultValueHolder.value; out.isSet = 1; } } } }
@Override public void eval() { if (in.isSet == 0) { out.isSet = 0; return; } final com.dremio.dac.proto.model.dataset.DataType integer = com.dremio.dac.proto.model.dataset.DataType.INTEGER; // TODO: check data mode final com.dremio.dac.proto.model.dataset.DataType dataType = com.dremio.dac.explore.DataTypeUtil.getDataType(in.reader.getMinorType()); if ((dataType == integer || castWhenPossibleHolder.value == 1) && com.dremio.dac.explore.udfs.CleanData.castToInteger(in.reader, out)) { out.isSet = 1; } else { if (replaceWithNullHolder.value == 1) { out.isSet = 0; } else { out.value = defaultValueHolder.value; out.isSet = 1; } } } }
@Override public void eval() { out.isSet = in.isSet; if (out.isSet == 0) { return; } byte[] result = com.dremio.dac.explore.DataTypeUtil.getDataType(in.reader.getMinorType()).toString().getBytes(); out.start = 0; buffer = buffer.reallocIfNeeded(result.length); out.buffer = buffer; out.buffer.setBytes(0, result); out.end = result.length; } }
public void eval() { out.isSet = 1; byte[] type; if (input.isSet()) { type = input.getMinorType().name().getBytes(); } else { type = org.apache.arrow.vector.types.Types.MinorType.NULL.name().getBytes(); } buf = buf.reallocIfNeeded(type.length); buf.setBytes(0, type); out.buffer = buf; out.start = 0; out.end = type.length; } }
@Override public void eval() { if (!in.isSet()) { out.isSet = 0; return; } final com.dremio.dac.proto.model.dataset.DataType integer = com.dremio.dac.proto.model.dataset.DataType.INTEGER; // TODO: check data mode final com.dremio.dac.proto.model.dataset.DataType dataType = com.dremio.dac.explore.DataTypeUtil.getDataType(in.getMinorType()); if ((dataType == integer || castWhenPossibleHolder.value == 1) && com.dremio.dac.explore.udfs.CleanData.castToInteger(in, out)) { out.isSet = 1; } else { if (replaceWithNullHolder.value == 1) { out.isSet = 0; } else { out.value = defaultValueHolder.value; out.isSet = 1; } } } }
@Override public void eval() { if (!in.isSet()) { out.isSet = 0; return; } final com.dremio.dac.proto.model.dataset.DataType floatType = com.dremio.dac.proto.model.dataset.DataType.FLOAT; final com.dremio.dac.proto.model.dataset.DataType dataType = com.dremio.dac.explore.DataTypeUtil.getDataType(in.getMinorType()); if ((dataType == floatType || castWhenPossibleHolder.value == 1) && com.dremio.dac.explore.udfs.CleanData.castToFloat(in, out)) { out.isSet = 1; } else { if (replaceWithNullHolder.value == 1) { out.isSet = 0; } else { out.value = defaultValueHolder.value; out.isSet = 1; } } } }
@Override public void startField() throws IOException { context.reset(recordSizeLimit); gen.writeStartObject(); // For only union type column, type is written for every cell value gen.writeFieldName("t"); gen.writeVarChar(String.valueOf(com.dremio.dac.explore.DataTypeUtil.getDataType(reader.getMinorType()))); gen.writeFieldName("v"); }
public UnionParquetConverter(int fieldId, String fieldName, FieldReader reader) { super(fieldId, fieldName, reader); unionReader = (UnionReader) reader; NonNullableStructVector internalMap = new UnionVectorHelper(unionReader.data) .getInternalMap(); SingleStructReaderImpl mapReader = new SingleStructReaderImpl(internalMap); int i = 0; for (String name : mapReader) { FieldReader fieldReader = mapReader.reader(name); FieldConverter converter = EventBasedRecordWriter.getFieldConverter(ParquetRecordWriter .this, i, name, fieldReader.getMinorType(), unionReader); if (converter != null) { converterMap.put(name, converter); i++; } } }
@Override public void eval() { out.isSet = 1; if (in.isSet == 0) { out.value = 1; // null values are considered clean return; } final com.dremio.dac.proto.model.dataset.DataType dataType = com.dremio.dac.explore.DataTypeUtil.getDataType(in.reader.getMinorType()); if (dataType == expectedDataType || castWhenPossibleHolder.value == 1) { if (expectedDataType == com.dremio.dac.proto.model.dataset.DataType.TEXT) { // for strings we can always cast out.value = 1; } else if (expectedDataType == com.dremio.dac.proto.model.dataset.DataType.INTEGER) { out.value = com.dremio.dac.explore.udfs.CleanData.castToInteger(in.reader, bigIntHolder) ? 1 : 0; } else if (expectedDataType == com.dremio.dac.proto.model.dataset.DataType.FLOAT) { out.value = com.dremio.dac.explore.udfs.CleanData.castToFloat(in.reader, floatHolder) ? 1 : 0; } else { // TODO: more types out.value = 0; } } else { // TODO: more types out.value = 0; } }
public void eval() { if (in.isSet == 1) { if (in.reader.getMinorType() != org.apache.arrow.vector.types.Types.MinorType.LIST) { throw errorContext.error() .message("The input is not a LIST type") .build(); } org.apache.arrow.vector.complex.impl.ComplexCopier.copy(in.reader, (org.apache.arrow.vector.complex.writer.FieldWriter) out.rootAsList()); } } }
public void eval() { if (in.isSet == 1) { if (in.reader.getMinorType() != org.apache.arrow.vector.types.Types.MinorType.STRUCT) { throw errorContext.error() .message("The input is not a STRUCT type") .build(); } org.apache.arrow.vector.complex.impl.ComplexCopier.copy(in.reader, (org.apache.arrow.vector.complex.writer.FieldWriter) out.rootAsStruct()); } } }