@Test public void testBeanWithoutGetter() { BeanWithoutGetter bean = new BeanWithoutGetter(); List<BeanWithoutGetter> data = Arrays.asList(bean); Dataset<Row> df = spark.createDataFrame(data, BeanWithoutGetter.class); Assert.assertEquals(df.schema().length(), 0); Assert.assertEquals(df.collectAsList().size(), 1); }
@Test public void testBeanWithoutGetter() { BeanWithoutGetter bean = new BeanWithoutGetter(); List<BeanWithoutGetter> data = Arrays.asList(bean); Dataset<Row> df = spark.createDataFrame(data, BeanWithoutGetter.class); Assert.assertEquals(df.schema().length(), 0); Assert.assertEquals(df.collectAsList().size(), 1); }
@Test public void testBeanWithoutGetter() { BeanWithoutGetter bean = new BeanWithoutGetter(); List<BeanWithoutGetter> data = Arrays.asList(bean); Dataset<Row> df = spark.createDataFrame(data, BeanWithoutGetter.class); Assert.assertEquals(df.schema().length(), 0); Assert.assertEquals(df.collectAsList().size(), 1); }
@Test public void testEmptyBean() { EmptyBean bean = new EmptyBean(); List<EmptyBean> data = Arrays.asList(bean); Dataset<EmptyBean> df = spark.createDataset(data, Encoders.bean(EmptyBean.class)); Assert.assertEquals(df.schema().length(), 0); Assert.assertEquals(df.collectAsList().size(), 1); }
@Test public void testEmptyBean() { EmptyBean bean = new EmptyBean(); List<EmptyBean> data = Arrays.asList(bean); Dataset<EmptyBean> df = spark.createDataset(data, Encoders.bean(EmptyBean.class)); Assert.assertEquals(df.schema().length(), 0); Assert.assertEquals(df.collectAsList().size(), 1); }
@Test public void testEmptyBean() { EmptyBean bean = new EmptyBean(); List<EmptyBean> data = Arrays.asList(bean); Dataset<EmptyBean> df = spark.createDataset(data, Encoders.bean(EmptyBean.class)); Assert.assertEquals(df.schema().length(), 0); Assert.assertEquals(df.collectAsList().size(), 1); }
long pageSizeBytes) { this.aggregationBufferSchema = aggregationBufferSchema; this.currentAggregationBuffer = new UnsafeRow(aggregationBufferSchema.length()); this.groupingKeyProjection = UnsafeProjection.create(groupingKeySchema); this.groupingKeySchema = groupingKeySchema;
long pageSizeBytes) { this.aggregationBufferSchema = aggregationBufferSchema; this.currentAggregationBuffer = new UnsafeRow(aggregationBufferSchema.length()); this.groupingKeyProjection = UnsafeProjection.create(groupingKeySchema); this.groupingKeySchema = groupingKeySchema;
boolean enablePerfMetrics) { this.aggregationBufferSchema = aggregationBufferSchema; this.currentAggregationBuffer = new UnsafeRow(aggregationBufferSchema.length()); this.groupingKeyProjection = UnsafeProjection.create(groupingKeySchema); this.groupingKeySchema = groupingKeySchema;
PrefixComparator prefixComparator = SortPrefixUtils.getPrefixComparator(keySchema); BaseOrdering ordering = GenerateOrdering.create(keySchema); KVComparator recordComparator = new KVComparator(ordering, keySchema.length()); boolean canUseRadixSort = keySchema.length() == 1 && SortPrefixUtils.canSortFullyWithPrefix(keySchema.apply(0)); serializerManager, taskContext, new KVComparator(ordering, keySchema.length()), prefixComparator, SparkEnv.get().conf().getInt("spark.shuffle.sort.initialBufferSize",
public static UnsafeExternalRowSorter create( StructType schema, Ordering<InternalRow> ordering, PrefixComparator prefixComparator, PrefixComputer prefixComputer, long pageSizeBytes, boolean canUseRadixSort) throws IOException { Supplier<RecordComparator> recordComparatorSupplier = () -> new RowComparator(ordering, schema.length()); return new UnsafeExternalRowSorter(schema, recordComparatorSupplier, prefixComparator, prefixComputer, pageSizeBytes, canUseRadixSort); }
protected FixedLengthRowBasedKeyValueBatch(StructType keySchema, StructType valueSchema, int maxRows, TaskMemoryManager manager) { super(keySchema, valueSchema, maxRows, manager); int keySize = keySchema.size() * 8; // each fixed-length field is stored in a 8-byte word int valueSize = valueSchema.size() * 8; klen = keySize + UnsafeRow.calculateBitSetWidthInBytes(keySchema.length()); vlen = valueSize + UnsafeRow.calculateBitSetWidthInBytes(valueSchema.length()); recordLength = klen + vlen + 8; } }
} else { orcVectorWrappers = new org.apache.spark.sql.vectorized.ColumnVector[resultSchema.length()];
} else { orcVectorWrappers = new org.apache.spark.sql.vectorized.ColumnVector[resultSchema.length()];
public static Row subsetRow(Row row, StructType subsetSchema) { Object[] values = new Object[subsetSchema.length()]; int i = 0; for (String fieldName : subsetSchema.fieldNames()) { values[i] = row.get(row.fieldIndex(fieldName)); i++; } Row subset = new RowWithSchema(subsetSchema, values); return subset; }