type = new ArrowType.Int(32, true);
@Override public TypeMapping visit(Int type) { boolean signed = type.getIsSigned(); switch (type.getBitWidth()) { case 8: return primitive(INT32, signed ? INT_8 : UINT_8); case 16: return primitive(INT32, signed ? INT_16 : UINT_16); case 32: return primitive(INT32, signed ? INT_32 : UINT_32); case 64: return primitive(INT64, signed ? INT_64 : UINT_64); default: throw new IllegalArgumentException("Illegal int type: " + field); } }
DictionaryEncoding dictionary = getDictionary(); if (dictionary != null) { int dictionaryType = dictionary.getIndexType().getType(builder); org.apache.arrow.flatbuf.DictionaryEncoding.startDictionaryEncoding(builder); org.apache.arrow.flatbuf.DictionaryEncoding.addId(builder, dictionary.getId());
int bitWidth = intType.bitWidth(); boolean isSigned = intType.isSigned(); return new ArrowType.Int(bitWidth, isSigned);
final Field field1 = new Field(SchemaPath.getSimplePath("c0").getAsUnescapedPath(), true, new ArrowType.Int(64, true), null); final BigIntVector longVector = dict1.addOrGet(field1); longVector.allocateNew();
switch (typeID) { case Int: estimatedFieldSize = ((Int) field.getType()).getBitWidth() / 8; break; case FloatingPoint:
.addField(new Field("bool", FieldType.nullable(ArrowType.Bool.INSTANCE), null)) .addField(new Field("decimal", FieldType.nullable(new ArrowType.Decimal(0, 0)), null)) .addField(new Field("int", FieldType.nullable(new ArrowType.Int(8, false)), null)) .addField(new Field("date", FieldType.nullable(new ArrowType.Date(DateUnit.MILLISECOND)), null)) .addField(new Field("time", FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 8)), null))
org.apache.arrow.flatbuf.Int indexTypeFB = dictionaryFB.indexType(); if (indexTypeFB != null) { indexType = new Int(indexTypeFB.bitWidth(), indexTypeFB.isSigned());
if(arrowType instanceof ArrowType.Int) { val intType = (ArrowType.Int) arrowType; if(intType.getBitWidth() == 32) return new IntegerMetaData(field.getName()); else {
private static VectorContainer buildLongGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) { final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(64, true), null); final VectorContainer input = new VectorContainer(bufferAllocator); final BigIntVector longVector = input.addOrGet(field); longVector.allocateNew(); SortedSet<Long> values = Sets.newTreeSet(); for (Dictionary dictionary : dictionaries) { for (int i = 0; i <= dictionary.getMaxId(); ++i) { values.add(dictionary.decodeToLong(i)); } } if (existingDict != null) { final BigIntVector existingDictValues = existingDict.getValueAccessorById(BigIntVector.class, 0).getValueVector(); for (int i = 0; i < existingDict.getRecordCount(); ++i) { values.add(existingDictValues.get(i)); } } final Iterator<Long> iter = values.iterator(); int recordCount = 0; while (iter.hasNext()) { longVector.setSafe(recordCount++, iter.next()); } longVector.setValueCount(recordCount); input.setRecordCount(recordCount); input.buildSchema(BatchSchema.SelectionVectorMode.NONE); return input; }
/** * Create a field given the input {@link ColumnType} * and name * @param name the name of the field * @param columnType the column type to add * @return */ public static Field getFieldForColumn(String name,ColumnType columnType) { switch(columnType) { case Long: return field(name,new ArrowType.Int(64,false)); case Integer: return field(name,new ArrowType.Int(32,false)); case Double: return field(name,new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)); case Float: return field(name,new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)); case Boolean: return field(name, new ArrowType.Bool()); case Categorical: return field(name,new ArrowType.Utf8()); case Time: return field(name,new ArrowType.Date(DateUnit.MILLISECOND)); case Bytes: return field(name,new ArrowType.Binary()); case NDArray: return field(name,new ArrowType.Binary()); case String: return field(name,new ArrowType.Utf8()); default: throw new IllegalArgumentException("Column type invalid " + columnType); } }
private static VectorContainer buildIntegerGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) { final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(32, true), null); final VectorContainer input = new VectorContainer(bufferAllocator); final IntVector intVector = input.addOrGet(field); intVector.allocateNew(); final SortedSet<Integer> values = Sets.newTreeSet(); for (Dictionary dictionary : dictionaries) { for (int i = 0; i <= dictionary.getMaxId(); ++i) { values.add(dictionary.decodeToInt(i)); } } if (existingDict != null) { final IntVector existingDictValues = existingDict.getValueAccessorById(IntVector.class, 0).getValueVector(); for (int i = 0; i < existingDict.getRecordCount(); ++i) { values.add(existingDictValues.get(i)); } } final Iterator<Integer> iter = values.iterator(); int recordCount = 0; while (iter.hasNext()) { intVector.setSafe(recordCount++, iter.next()); } intVector.setValueCount(recordCount); input.setRecordCount(recordCount); input.buildSchema(BatchSchema.SelectionVectorMode.NONE); return input; }
@Override public JobUI answer(InvocationOnMock invocation) throws Throwable { String query = invocation.getArgumentAt(0, SqlQuery.class).getSql(); Job job = mock(Job.class); JobData jobData = mock(JobData.class); when(job.getData()).thenReturn(jobData); if ("SELECT * FROM dataset".equals(query)) { when(jobData.getJobResultsTable()).thenReturn("jobResults.previewJob"); } else if (query.contains("jobResults.previewJob")) { if (expFilter != null) { assertTrue(query, query.contains(expFilter)); } else { assertFalse(query, query.contains("WHERE")); } JobDataFragment fragment = mock(JobDataFragment.class); when(jobData.truncate(1)).thenReturn(fragment); when(fragment.getSchema()).thenReturn( BatchSchema.newBuilder() .addField(new Field("dremio_selection_count", true, new ArrowType.Int(64, true), null)) .build() ); when(fragment.extractValue("dremio_selection_count", 0)).thenReturn(expCount); } return new JobUI(job); } }
@Test public void testDataSetSchema() throws Exception { try( final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false); ) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); Field field1 = new Field("a", true, new Int(32, true), null); Field child1 = new Field("c", true, Utf8.INSTANCE, null); Field field2 = new Field("b", true, Struct.INSTANCE, ImmutableList.of(child1)); Schema schema = new Schema(ImmutableList.of(field1, field2)); FlatBufferBuilder builder = new FlatBufferBuilder(); schema.getSchema(builder); builder.finish(schema.getSchema(builder)); addSource(ns, "s"); addPhysicalDS(ns, "s.foo", builder.sizedByteArray()); ByteBuffer bb = ByteBuffer.wrap(DatasetHelper.getSchemaBytes(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("s.foo")))).toByteArray()); Schema returnedSchema = Schema.convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(bb)); assertEquals(schema, returnedSchema); } }
/** * * @param allocator * @param name * @param data * @return */ public static BigIntVector vectorFor(BufferAllocator allocator,String name,long[] data) { BigIntVector float8Vector = new BigIntVector(name,FieldType.nullable(new ArrowType.Int(64,true)),allocator); float8Vector.allocateNew(data.length); for(int i = 0; i < data.length; i++) { float8Vector.setSafe(i,data[i]); } float8Vector.setValueCount(data.length); return float8Vector; }
/** * * @param allocator * @param name * @param data * @return */ public static IntVector vectorFor(BufferAllocator allocator,String name,int[] data) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(data.length); for(int i = 0; i < data.length; i++) { float8Vector.setSafe(i,data[i]); } float8Vector.setValueCount(data.length); return float8Vector; }
@Override public TypeLayout visit(Int type) { return newFixedWidthTypeLayout(BufferLayout.dataBuffer(type.getBitWidth())); }
@Override public MinorType visit(Int type) { switch (type.getBitWidth()) { case 8: return type.getIsSigned() ? MinorType.TINYINT : MinorType.UINT1; case 16: return type.getIsSigned() ? MinorType.SMALLINT : MinorType.UINT2; case 32: return type.getIsSigned() ? MinorType.INT : MinorType.UINT4; case 64: return type.getIsSigned() ? MinorType.BIGINT : MinorType.UINT8; default: throw new IllegalArgumentException("only 8, 16, 32, 64 supported: " + type); } }
/** * * @param allocator * @param name * @return */ public static IntVector intVectorOf(BufferAllocator allocator,String name,int length) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(length); float8Vector.setValueCount(length); return float8Vector; }