@JsonCreator public DictionaryEncoding( @JsonProperty("id") long id, @JsonProperty("isOrdered") boolean ordered, @JsonProperty("indexType") Int indexType) { this.id = id; this.ordered = ordered; this.indexType = indexType == null ? new Int(32, true) : indexType; }
@Override public JobUI answer(InvocationOnMock invocation) throws Throwable { String query = invocation.getArgumentAt(0, SqlQuery.class).getSql(); Job job = mock(Job.class); JobData jobData = mock(JobData.class); when(job.getData()).thenReturn(jobData); if ("SELECT * FROM dataset".equals(query)) { when(jobData.getJobResultsTable()).thenReturn("jobResults.previewJob"); } else if (query.contains("jobResults.previewJob")) { if (expFilter != null) { assertTrue(query, query.contains(expFilter)); } else { assertFalse(query, query.contains("WHERE")); } JobDataFragment fragment = mock(JobDataFragment.class); when(jobData.truncate(1)).thenReturn(fragment); when(fragment.getSchema()).thenReturn( BatchSchema.newBuilder() .addField(new Field("dremio_selection_count", true, new ArrowType.Int(64, true), null)) .build() ); when(fragment.extractValue("dremio_selection_count", 0)).thenReturn(expCount); } return new JobUI(job); } }
private static VectorContainer buildIntegerGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) { final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(32, true), null); final VectorContainer input = new VectorContainer(bufferAllocator); final IntVector intVector = input.addOrGet(field); intVector.allocateNew(); final SortedSet<Integer> values = Sets.newTreeSet(); for (Dictionary dictionary : dictionaries) { for (int i = 0; i <= dictionary.getMaxId(); ++i) { values.add(dictionary.decodeToInt(i)); } } if (existingDict != null) { final IntVector existingDictValues = existingDict.getValueAccessorById(IntVector.class, 0).getValueVector(); for (int i = 0; i < existingDict.getRecordCount(); ++i) { values.add(existingDictValues.get(i)); } } final Iterator<Integer> iter = values.iterator(); int recordCount = 0; while (iter.hasNext()) { intVector.setSafe(recordCount++, iter.next()); } intVector.setValueCount(recordCount); input.setRecordCount(recordCount); input.buildSchema(BatchSchema.SelectionVectorMode.NONE); return input; }
int bitWidth = intType.bitWidth(); boolean isSigned = intType.isSigned(); return new ArrowType.Int(bitWidth, isSigned);
.addField(new Field("bool", FieldType.nullable(ArrowType.Bool.INSTANCE), null)) .addField(new Field("decimal", FieldType.nullable(new ArrowType.Decimal(0, 0)), null)) .addField(new Field("int", FieldType.nullable(new ArrowType.Int(8, false)), null)) .addField(new Field("date", FieldType.nullable(new ArrowType.Date(DateUnit.MILLISECOND)), null)) .addField(new Field("time", FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 8)), null))
type = new ArrowType.Int(32, true);
org.apache.arrow.flatbuf.Int indexTypeFB = dictionaryFB.indexType(); if (indexTypeFB != null) { indexType = new Int(indexTypeFB.bitWidth(), indexTypeFB.isSigned());
private static VectorContainer buildLongGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) { final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(64, true), null); final VectorContainer input = new VectorContainer(bufferAllocator); final BigIntVector longVector = input.addOrGet(field); longVector.allocateNew(); SortedSet<Long> values = Sets.newTreeSet(); for (Dictionary dictionary : dictionaries) { for (int i = 0; i <= dictionary.getMaxId(); ++i) { values.add(dictionary.decodeToLong(i)); } } if (existingDict != null) { final BigIntVector existingDictValues = existingDict.getValueAccessorById(BigIntVector.class, 0).getValueVector(); for (int i = 0; i < existingDict.getRecordCount(); ++i) { values.add(existingDictValues.get(i)); } } final Iterator<Long> iter = values.iterator(); int recordCount = 0; while (iter.hasNext()) { longVector.setSafe(recordCount++, iter.next()); } longVector.setValueCount(recordCount); input.setRecordCount(recordCount); input.buildSchema(BatchSchema.SelectionVectorMode.NONE); return input; }
/** * Create a field given the input {@link ColumnType} * and name * @param name the name of the field * @param columnType the column type to add * @return */ public static Field getFieldForColumn(String name,ColumnType columnType) { switch(columnType) { case Long: return field(name,new ArrowType.Int(64,false)); case Integer: return field(name,new ArrowType.Int(32,false)); case Double: return field(name,new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)); case Float: return field(name,new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)); case Boolean: return field(name, new ArrowType.Bool()); case Categorical: return field(name,new ArrowType.Utf8()); case Time: return field(name,new ArrowType.Date(DateUnit.MILLISECOND)); case Bytes: return field(name,new ArrowType.Binary()); case NDArray: return field(name,new ArrowType.Binary()); case String: return field(name,new ArrowType.Utf8()); default: throw new IllegalArgumentException("Column type invalid " + columnType); } }
@Test public void testDataSetSchema() throws Exception { try( final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false); ) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); Field field1 = new Field("a", true, new Int(32, true), null); Field child1 = new Field("c", true, Utf8.INSTANCE, null); Field field2 = new Field("b", true, Struct.INSTANCE, ImmutableList.of(child1)); Schema schema = new Schema(ImmutableList.of(field1, field2)); FlatBufferBuilder builder = new FlatBufferBuilder(); schema.getSchema(builder); builder.finish(schema.getSchema(builder)); addSource(ns, "s"); addPhysicalDS(ns, "s.foo", builder.sizedByteArray()); ByteBuffer bb = ByteBuffer.wrap(DatasetHelper.getSchemaBytes(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("s.foo")))).toByteArray()); Schema returnedSchema = Schema.convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(bb)); assertEquals(schema, returnedSchema); } }
/** * * @param allocator * @param name * @param data * @return */ public static BigIntVector vectorFor(BufferAllocator allocator,String name,long[] data) { BigIntVector float8Vector = new BigIntVector(name,FieldType.nullable(new ArrowType.Int(64,true)),allocator); float8Vector.allocateNew(data.length); for(int i = 0; i < data.length; i++) { float8Vector.setSafe(i,data[i]); } float8Vector.setValueCount(data.length); return float8Vector; }
/** * * @param allocator * @param name * @param data * @return */ public static IntVector vectorFor(BufferAllocator allocator,String name,int[] data) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(data.length); for(int i = 0; i < data.length; i++) { float8Vector.setSafe(i,data[i]); } float8Vector.setValueCount(data.length); return float8Vector; }
/** * * @param allocator * @param name * @param length the number of rows in the column vector * @return */ public static BigIntVector longVectorOf(BufferAllocator allocator,String name,int length) { BigIntVector float8Vector = new BigIntVector(name,FieldType.nullable(new ArrowType.Int(64,true)),allocator); float8Vector.allocateNew(length); float8Vector.setValueCount(length); return float8Vector; }
/** * * @param allocator * @param name * @return */ public static IntVector intVectorOf(BufferAllocator allocator,String name,int length) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(length); float8Vector.setValueCount(length); return float8Vector; }
protected void baseManyColumns() throws Exception { int columns = 1000; int leftColumns = columns; int rightColumns = columns; JoinInfo joinInfo = getJoinInfo(Arrays.asList(new JoinCondition("EQUALS", f("left_1"), f("right_1"))), JoinRelType.LEFT); Table expected = t(getHeader("right", rightColumns, "left", leftColumns), false, getData(columns, leftColumns, 1)); validateDual(joinInfo.operator, joinInfo.clazz, new ManyColumnsGenerator<IntVector>(getTestAllocator(), "left", leftColumns, 1, IntVector .class, new ArrowType.Int(32, true), BaseTestJoin::insertIntoIntVector), new ManyColumnsGenerator<IntVector>(getTestAllocator(), "right", rightColumns, 1, IntVector .class, new ArrowType.Int(32, true), BaseTestJoin::insertIntoIntVector), DEFAULT_BATCH, expected); }
@Test public void testGandivaPluggableRegistry() throws Exception { GandivaFunctionRegistry fnRegistry = new GandivaFunctionRegistry(DEFAULT_SABOT_CONFIG); FunctionCall fnCall = getAddFn(); GandivaFunctionHolder holder = (GandivaFunctionHolder)fnRegistry.getFunction(fnCall); Assert.assertNotNull(holder); ArrowType.Int int32 = new ArrowType.Int(32, true); CompleteType expectedReturnType = new CompleteType(int32); Assert.assertEquals(expectedReturnType, holder.getReturnType()); Assert.assertNotNull(holder.getExpr(fnCall.getName(), fnCall.args)); }
@Override ArrowType getType() { return new ArrowType.Int(32, true); }
@Override ArrowType getType() { return new ArrowType.Int(64, true); }
/** Helper method which creates a union vector with no data */ private static UnionVector testEmptyUnionVector() { final UnionVector unionVector = new UnionVector("unionVector", ALLOCATOR, null); unionVector.initializeChildrenFromFields( asList( Field.nullable("intType", new ArrowType.Int(32, true)), Field.nullable("decimalType", new ArrowType.Decimal(4, 10)) ) ); return unionVector; }