@Override public byte[] getDecimalPage() { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength]; for (int rowId = 0; rowId < pageSize; rowId++) { int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); copyBytes(rowId, data, offset, length); offset += length; } return data; }
@Override public byte[] getComplexParentFlattenedBytePage() throws IOException { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength]; for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { int length = (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId)); copyBytes(rowId, data, offset, length); offset += length; } return data; }
@Override public byte[][] getByteArrayPage() { byte[][] bytes = new byte[rowOffset.getActualRowCount() - 1][]; for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); byte[] rowData = new byte[length]; CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), rowData, CarbonUnsafe.BYTE_ARRAY_OFFSET, length); bytes[rowId] = rowData; } return bytes; }
@Override public byte[] getComplexChildrenLVFlattenedBytePage() throws IOException { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 2)]; for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { short length = (short) (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId)); ByteUtil.setShort(data, offset, length); copyBytes(rowId, data, offset + 2, length); offset += 2 + length; } return data; }
@Override void copyBytes(int rowId, byte[] dest, int destOffset, int length) { CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), dest, CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length); }
@Override void copyBytes(int rowId, byte[] dest, int destOffset, int length) { CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), dest, CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length); }
@Override public byte[] getLVFlattenedBytePage() throws IOException { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 4)]; for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); ByteUtil.setInt(data, offset, length); copyBytes(rowId, data, offset + 4, length); offset += 4 + length; } return data; }
@Override public void putBytes(int rowId, byte[] bytes, int offset, int length) { try { ensureMemory(length); } catch (MemoryException e) { throw new RuntimeException(e); } CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, baseAddress, baseOffset + rowOffset.getInt(rowId), length); }
@Override public void putBytes(int rowId, byte[] bytes, int offset, int length) { try { ensureMemory(length); } catch (MemoryException e) { throw new RuntimeException(e); } CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, baseAddress, baseOffset + rowOffset.getInt(rowId), length); }
@Override public void putBytes(int rowId, byte[] bytes) { // rowId * 4 represents the length of L in LV if (bytes.length > (Integer.MAX_VALUE - totalLength - rowId * 4)) { // since we later store a column page in a byte array, so its maximum size is 2GB throw new RuntimeException("Carbondata only support maximum 2GB size for one column page," + " exceed this limit at rowId " + rowId); } if (rowId == 0) { rowOffset.putInt(0, 0); } rowOffset.putInt(rowId + 1, rowOffset.getInt(rowId) + bytes.length); putBytesAtRow(rowId, bytes); totalLength += bytes.length; }
} else if (type == DataTypes.INT) { value = page.getInt(rowId); } else if (type == DataTypes.LONG) {
private static VarLengthColumnPageBase getVarLengthColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, int rowId, ColumnPage rowOffset, int offset, String compressorName) throws MemoryException { int lvEncodedOffset; int length; int numRows = rowId; VarLengthColumnPageBase page; int inputDataLength = offset; if (unsafe) { page = new UnsafeDecimalColumnPage( new ColumnPageEncoderMeta(columnSpec, dataType, compressorName), numRows, inputDataLength); } else { page = new SafeDecimalColumnPage( new ColumnPageEncoderMeta(columnSpec, dataType, compressorName), numRows); } // set total length and rowOffset in page page.totalLength = offset; page.rowOffset.freeMemory(); page.rowOffset = rowOffset; // set data in page lvEncodedOffset = 0; for (int i = 0; i < numRows; i++) { length = rowOffset.getInt(i + 1) - rowOffset.getInt(i); page.putBytes(i, lvEncodedBytes, lvEncodedOffset + lvLength, length); lvEncodedOffset += lvLength + length; } return page; }
@Override public long getLong(int rowId) { DataType dataType = columnPage.getDataType(); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { return converter.decodeLong(columnPage.getByte(rowId)); } else if (dataType == DataTypes.SHORT) { return converter.decodeLong(columnPage.getShort(rowId)); } else if (dataType == DataTypes.SHORT_INT) { return converter.decodeLong(columnPage.getShortInt(rowId)); } else if (dataType == DataTypes.INT) { return converter.decodeLong(columnPage.getInt(rowId)); } else if (dataType == DataTypes.LONG) { return columnPage.getLong(rowId); } else { throw new RuntimeException("internal error: " + this.toString()); } }
return getShort(rowId); } else if (dataType == DataTypes.INT) { return getInt(rowId); } else if (dataType == DataTypes.LONG) { return getLong(rowId);
/** * Get null at rowId */ private Object getNull(int rowId) { Object result; DataType dataType = columnPageEncoderMeta.getStoreDataType(); if (dataType == DataTypes.BOOLEAN) { result = getBoolean(rowId); } else if (dataType == DataTypes.BYTE) { result = getByte(rowId); if (columnPageEncoderMeta.getColumnSpec().getSchemaDataType() == DataTypes.BOOLEAN) { result = BooleanConvert.byte2Boolean((byte)result); } } else if (dataType == DataTypes.SHORT) { result = getShort(rowId); } else if (dataType == DataTypes.INT) { result = getInt(rowId); } else if (dataType == DataTypes.LONG) { result = getLong(rowId); } else if (dataType == DataTypes.DOUBLE) { result = getDouble(rowId); } else if (DataTypes.isDecimal(dataType)) { result = getDecimal(rowId); } else { throw new IllegalArgumentException("unsupported data type: " + dataType); } return result; }
@Override public double getDouble(int rowId) { DataType dataType = columnPage.getDataType(); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { return converter.decodeDouble(columnPage.getByte(rowId)); } else if (dataType == DataTypes.SHORT) { return converter.decodeDouble(columnPage.getShort(rowId)); } else if (dataType == DataTypes.SHORT_INT) { return converter.decodeDouble(columnPage.getShortInt(rowId)); } else if (dataType == DataTypes.INT) { return converter.decodeDouble(columnPage.getInt(rowId)); } else if (dataType == DataTypes.LONG) { return converter.decodeDouble(columnPage.getLong(rowId)); } else if (dataType == DataTypes.FLOAT) { return converter.decodeDouble(columnPage.getFloat(rowId)); } else if (dataType == DataTypes.DOUBLE) { return columnPage.getDouble(rowId); } else { throw new RuntimeException("internal error: " + this.toString()); } }
@Override public BigDecimal getDecimal(int rowId) { DecimalConverterFactory.DecimalConverter decimalConverter = ((DecimalColumnPage) columnPage).getDecimalConverter(); DataType dataType = columnPage.getDataType(); if (dataType == DataTypes.BYTE) { return decimalConverter.getDecimal(converter.decodeLong(columnPage.getByte(rowId))); } else if (dataType == DataTypes.SHORT) { return decimalConverter.getDecimal(converter.decodeLong(columnPage.getShort(rowId))); } else if (dataType == DataTypes.SHORT_INT) { return decimalConverter.getDecimal(converter.decodeLong(columnPage.getShortInt(rowId))); } else if (dataType == DataTypes.INT) { return decimalConverter.getDecimal(converter.decodeLong(columnPage.getInt(rowId))); } else if (dataType == DataTypes.LONG || DataTypes.isDecimal(dataType)) { return columnPage.getDecimal(rowId); } else { throw new RuntimeException("internal error: " + this.toString()); } }
data[i] = input.getInt(i);