protected boolean isArray() { return type instanceof ArrayType || type instanceof BinaryType || type instanceof StringType || DecimalType.isByteArrayDecimalType(type); }
public AggregateHashMap(StructType schema, int capacity, double loadFactor, int maxSteps) { // We currently only support single key-value pair that are both longs assert (schema.size() == 2 && schema.fields()[0].dataType() == LongType && schema.fields()[1].dataType() == LongType); // capacity should be a power of 2 assert (capacity > 0 && ((capacity & (capacity - 1)) == 0)); this.maxSteps = maxSteps; numBuckets = (int) (capacity / loadFactor); batch = ColumnarBatch.allocate(schema, MemoryMode.ON_HEAP, capacity); buckets = new int[numBuckets]; Arrays.fill(buckets, -1); }
/** * @return true if UnsafeFixedWidthAggregationMap supports aggregation buffers with the given * schema, false otherwise. */ public static boolean supportsAggregationBufferSchema(StructType schema) { for (StructField field: schema.fields()) { if (!UnsafeRow.isMutable(field.dataType())) { return false; } } return true; }
private StructType createSchema() { StructType intervalType = new StructType(new StructField[] { new StructField("startTime", DataTypes.LongType, true, Metadata.empty()), new StructField("endTime", DataTypes.LongType, true, Metadata.empty()) }); DataType intervalsType = new ArrayType(intervalType, false); return new StructType(new StructField[] { new StructField("id", DataTypes.IntegerType, true, Metadata.empty()), new StructField("intervals", intervalsType, true, Metadata.empty()) }); }
@Test public void testCreateStructTypeFromList(){ List<StructField> fields1 = new ArrayList<>(); fields1.add(new StructField("id", DataTypes.StringType, true, Metadata.empty())); StructType schema1 = StructType$.MODULE$.apply(fields1); Assert.assertEquals(0, schema1.fieldIndex("id")); List<StructField> fields2 = Arrays.asList(new StructField("id", DataTypes.StringType, true, Metadata.empty())); StructType schema2 = StructType$.MODULE$.apply(fields2); Assert.assertEquals(0, schema2.fieldIndex("id")); }
@Override public InternalRow get() { Object[] values = new Object[requiredSchema.size()]; for (int i = 0; i < values.length; i++) { if ("i".equals(requiredSchema.apply(i).name())) { values[i] = start; } else if ("j".equals(requiredSchema.apply(i).name())) { values[i] = -start; } } return new GenericInternalRow(values); }
/** * Allocates columns to store elements of each field of the schema off heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OffHeapColumnVector[] allocateColumns(int capacity, StructType schema) { return allocateColumns(capacity, schema.fields()); }
/** * Allocates columns to store elements of each field off heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OffHeapColumnVector[] allocateColumns(int capacity, StructField[] fields) { OffHeapColumnVector[] vectors = new OffHeapColumnVector[fields.length]; for (int i = 0; i < fields.length; i++) { vectors[i] = new OffHeapColumnVector(capacity, fields[i].dataType()); } return vectors; }
public ColumnarRow(ColumnVector data, int rowId) { assert (data.dataType() instanceof StructType); this.data = data; this.rowId = rowId; this.numFields = ((StructType) data.dataType()).size(); }
@Test public void testCreateStructTypeFromList(){ List<StructField> fields1 = new ArrayList<>(); fields1.add(new StructField("id", DataTypes.StringType, true, Metadata.empty())); StructType schema1 = StructType$.MODULE$.apply(fields1); Assert.assertEquals(0, schema1.fieldIndex("id")); List<StructField> fields2 = Arrays.asList(new StructField("id", DataTypes.StringType, true, Metadata.empty())); StructType schema2 = StructType$.MODULE$.apply(fields2); Assert.assertEquals(0, schema2.fieldIndex("id")); }
/** * @return true if UnsafeFixedWidthAggregationMap supports aggregation buffers with the given * schema, false otherwise. */ public static boolean supportsAggregationBufferSchema(StructType schema) { for (StructField field: schema.fields()) { if (!UnsafeRow.isMutable(field.dataType())) { return false; } } return true; }
@Override public InternalRow get() { Object[] values = new Object[requiredSchema.size()]; for (int i = 0; i < values.length; i++) { if ("i".equals(requiredSchema.apply(i).name())) { values[i] = start; } else if ("j".equals(requiredSchema.apply(i).name())) { values[i] = -start; } } return new GenericInternalRow(values); }
/** * Allocates columns to store elements of each field of the schema off heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OffHeapColumnVector[] allocateColumns(int capacity, StructType schema) { return allocateColumns(capacity, schema.fields()); }
/** * Allocates columns to store elements of each field off heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OffHeapColumnVector[] allocateColumns(int capacity, StructField[] fields) { OffHeapColumnVector[] vectors = new OffHeapColumnVector[fields.length]; for (int i = 0; i < fields.length; i++) { vectors[i] = new OffHeapColumnVector(capacity, fields[i].dataType()); } return vectors; }
public ColumnarRow(ColumnVector data, int rowId) { assert (data.dataType() instanceof StructType); this.data = data; this.rowId = rowId; this.numFields = ((StructType) data.dataType()).size(); }
protected boolean isArray() { return type instanceof ArrayType || type instanceof BinaryType || type instanceof StringType || DecimalType.isByteArrayDecimalType(type); }
@Test public void testCreateStructTypeFromList(){ List<StructField> fields1 = new ArrayList<>(); fields1.add(new StructField("id", DataTypes.StringType, true, Metadata.empty())); StructType schema1 = StructType$.MODULE$.apply(fields1); Assert.assertEquals(0, schema1.fieldIndex("id")); List<StructField> fields2 = Arrays.asList(new StructField("id", DataTypes.StringType, true, Metadata.empty())); StructType schema2 = StructType$.MODULE$.apply(fields2); Assert.assertEquals(0, schema2.fieldIndex("id")); }
/** * @return true if UnsafeFixedWidthAggregationMap supports aggregation buffers with the given * schema, false otherwise. */ public static boolean supportsAggregationBufferSchema(StructType schema) { for (StructField field: schema.fields()) { if (!UnsafeRow.isMutable(field.dataType())) { return false; } } return true; }
/** * Allocates columns to store elements of each field of the schema on heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OnHeapColumnVector[] allocateColumns(int capacity, StructType schema) { return allocateColumns(capacity, schema.fields()); }
/** * Allocates columns to store elements of each field on heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OnHeapColumnVector[] allocateColumns(int capacity, StructField[] fields) { OnHeapColumnVector[] vectors = new OnHeapColumnVector[fields.length]; for (int i = 0; i < fields.length; i++) { vectors[i] = new OnHeapColumnVector(capacity, fields[i].dataType()); } return vectors; }