private static ColumnPage newFixedByteArrayPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedByteArray, int eachValueSize, String compressorName) throws MemoryException { int pageSize = lvEncodedByteArray.length / eachValueSize; ColumnPage fixLengthByteArrayPage = createFixLengthByteArrayPage( new ColumnPageEncoderMeta(columnSpec, columnSpec.getSchemaDataType(), compressorName), pageSize, eachValueSize); byte[] data = null; int offset = 0; for (int i = 0; i < pageSize; i++) { data = new byte[eachValueSize]; System.arraycopy(lvEncodedByteArray, offset, data, 0, eachValueSize); fixLengthByteArrayPage.putBytes(i, data); offset += eachValueSize; } return fixLengthByteArrayPage; }
/** * Below method will be used to add column data to page * * @param rowId row number * @param bytes actual data */ @Override public void putBytes(int rowId, byte[] bytes) { if (null != pageLevelDictionary) { try { actualDataColumnPage.putBytes(rowId, bytes); dummyKey[0] = pageLevelDictionary.getDictionaryValue(bytes); encodedDataColumnPage.putBytes(rowId, keyGenerator.generateKey(dummyKey)); } catch (DictionaryThresholdReachedException e) { LOGGER.warn("Local Dictionary threshold reached for the column: " + actualDataColumnPage .getColumnSpec().getFieldName() + ", " + e.getMessage()); pageLevelDictionary = null; encodedDataColumnPage.freeMemory(); encodedDataColumnPage = null; } catch (KeyGenException e) { LOGGER.error("Unable to generate key for: " + actualDataColumnPage .getColumnSpec().getFieldName(), e); throw new RuntimeException(e); } } else { actualDataColumnPage.putBytes(rowId, bytes); } }
|| dataType == DataTypes.BYTE_ARRAY || dataType == DataTypes.VARCHAR) { putBytes(rowId, (byte[]) value); statsCollector.update((byte[]) value); } else if (dataType == DataTypes.FLOAT) {
int keyArray = (int) keyGenerator.getKeyArray(bytes, index)[0]; actualDataColumnPage .putBytes(rowId++, localDictionaryGenerator.getDictionaryKeyBasedOnValue(keyArray));