public static int calculateEstimatedMemorySize(int count) { // FUTURE: Partitioning not included yet. final int longMatchFlagsSize = (count + Long.SIZE - 1) / Long.SIZE; int size = 0; JavaDataModel jdm = JavaDataModel.get(); size += jdm.lengthForLongArrayOfSize(longMatchFlagsSize); size += jdm.primitive1(); size += (2 * jdm.object()); return size; } }
@Override public long getEstimatedMemorySize() { long size = 0; size += super.getEstimatedMemorySize(); size += unsafeReadPos == null ? 0 : unsafeReadPos.getEstimatedMemorySize(); size += JavaDataModel.get().lengthForLongArrayOfSize(slots.length); return size; } }
} else if (colTypeLowerCase.equals(serdeConstants.BIGINT_TYPE_NAME) || colTypeLowerCase.equals("long")) { return JavaDataModel.get().lengthForLongArrayOfSize(length); } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) { return JavaDataModel.get().lengthForByteArrayOfSize(length);
@Override public long getEstimatedMemorySize() { JavaDataModel jdm = JavaDataModel.get(); long size = 0; size += writeBuffers.getEstimatedMemorySize(); size += jdm.lengthForLongArrayOfSize(refs.length); // 11 primitive1 fields, 2 refs above with alignment size += JavaDataModel.alignUp(15 * jdm.primitive1(), jdm.memoryAlign()); return size; }
} else if (colTypeLowerCase.equals(serdeConstants.BIGINT_TYPE_NAME) || colTypeLowerCase.equals("long")) { return JavaDataModel.get().lengthForLongArrayOfSize(length); } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) { return JavaDataModel.get().lengthForByteArrayOfSize(length);
@Override public long getEstimatedMemorySize() { JavaDataModel jdm = JavaDataModel.get(); long size = super.getEstimatedMemorySize(); size += slotPairs == null ? 0 : jdm.lengthForLongArrayOfSize(slotPairs.length); size += (2 * jdm.primitive2()); size += (2 * jdm.primitive1()); size += jdm.object(); // adding 16KB constant memory for keyBinarySortableDeserializeRead as the rabit hole is deep to implement // MemoryEstimate interface, also it is constant overhead size += (16 * 1024L); return size; } }
@Override public long getAggregationBufferFixedSize() { if (bitSetSize < 0) { // Not pretty, but we need a way to get the size try { Aggregation agg = (Aggregation) getNewAggregationBuffer(); bitSetSize = agg.bf.getBitSet().length; } catch (Exception e) { throw new RuntimeException("Unexpected error while creating AggregationBuffer", e); } } // BloomFilter: object(BitSet: object(data: long[]), numBits: int, numHashFunctions: int) JavaDataModel model = JavaDataModel.get(); long bloomFilterSize = JavaDataModel.alignUp(model.object() + model.lengthForLongArrayOfSize(bitSetSize), model.memoryAlign()); return JavaDataModel.alignUp( model.object() + bloomFilterSize + model.primitive1() + model.primitive1(), model.memoryAlign()); }
@Override public int getAggregationBufferFixedSize() { if (bitSetSize < 0) { // Not pretty, but we need a way to get the size try { Aggregation agg = (Aggregation) getNewAggregationBuffer(); bitSetSize = agg.bf.getBitSet().length; } catch (Exception e) { throw new RuntimeException("Unexpected error while creating AggregationBuffer", e); } } // BloomFilter: object(BitSet: object(data: long[]), numBits: int, numHashFunctions: int) JavaDataModel model = JavaDataModel.get(); int bloomFilterSize = JavaDataModel.alignUp(model.object() + model.lengthForLongArrayOfSize(bitSetSize), model.memoryAlign()); return JavaDataModel.alignUp( model.object() + bloomFilterSize + model.primitive1() + model.primitive1(), model.memoryAlign()); }
compiledKeyWrapperBatch.keysFixedSize += model.lengthForLongArrayOfSize(compiledKeyWrapperBatch.longIndices.length); compiledKeyWrapperBatch.keysFixedSize += model.lengthForDoubleArrayOfSize(compiledKeyWrapperBatch.doubleIndices.length); compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.stringIndices.length);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForLongArrayOfSize(compiledKeyWrapperBatch.longIndices.length); compiledKeyWrapperBatch.keysFixedSize += model.lengthForDoubleArrayOfSize(compiledKeyWrapperBatch.doubleIndices.length); compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.stringIndices.length);
} else if (colType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME) || colType.equalsIgnoreCase("long")) { return JavaDataModel.get().lengthForLongArrayOfSize(length); } else if (colType.equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME)) { return JavaDataModel.get().lengthForByteArrayOfSize(length);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForLongArrayOfSize(compiledKeyWrapperBatch.longIndices.length); compiledKeyWrapperBatch.keysFixedSize += model.lengthForDoubleArrayOfSize(compiledKeyWrapperBatch.doubleIndices.length); compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.stringIndices.length);