public Object getFieldValue(_Fields field) { switch (field) { case VERSION: return getVersion(); case COLUMN_SCHEMA: return getColumn_schema(); case IS_FOOTER_PRESENT: return isIs_footer_present(); case TIME_STAMP: return getTime_stamp(); case IS_SPLITABLE: return isIs_splitable(); case SYNC_MARKER: return getSync_marker(); case COMPRESSOR_NAME: return getCompressor_name(); } throw new IllegalStateException(); }
/** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ public boolean isSet(_Fields field) { if (field == null) { throw new IllegalArgumentException(); } switch (field) { case VERSION: return isSetVersion(); case COLUMN_SCHEMA: return isSetColumn_schema(); case IS_FOOTER_PRESENT: return isSetIs_footer_present(); case TIME_STAMP: return isSetTime_stamp(); case IS_SPLITABLE: return isSetIs_splitable(); case SYNC_MARKER: return isSetSync_marker(); case COMPRESSOR_NAME: return isSetCompressor_name(); } throw new IllegalStateException(); }
case VERSION: if (value == null) { unsetVersion(); } else { setVersion((Integer)value); unsetColumn_schema(); } else { setColumn_schema((List<org.apache.carbondata.format.ColumnSchema>)value); unsetIs_footer_present(); } else { setIs_footer_present((Boolean)value); unsetTime_stamp(); } else { setTime_stamp((Long)value); unsetIs_splitable(); } else { setIs_splitable((Boolean)value); unsetSync_marker(); } else { setSync_marker((ByteBuffer)value); unsetCompressor_name(); } else { setCompressor_name((String)value);
/** * Below method will be used to prepare the file header object for carbondata file * * @param isFooterPresent is footer present in carbon data file * @param columnSchemaList list of column schema * @param schemaUpdatedTimeStamp schema updated time stamp to be used for restructure scenarios * @return file header thrift object */ public static FileHeader getFileHeader(boolean isFooterPresent, List<ColumnSchema> columnSchemaList, long schemaUpdatedTimeStamp) { FileHeader fileHeader = new FileHeader(); ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion(); fileHeader.setIs_footer_present(isFooterPresent); fileHeader.setColumn_schema(columnSchemaList); fileHeader.setVersion(version.number()); fileHeader.setTime_stamp(schemaUpdatedTimeStamp); return fileHeader; }
private void writeFileHeader() throws IOException { List<ColumnSchema> wrapperColumnSchemaList = CarbonUtil .getColumnSchemaList(carbonTable.getDimensionByTableName(carbonTable.getTableName()), carbonTable.getMeasureByTableName(carbonTable.getTableName())); int[] dimLensWithComplex = new int[wrapperColumnSchemaList.size()]; for (int i = 0; i < dimLensWithComplex.length; i++) { dimLensWithComplex[i] = Integer.MAX_VALUE; } int[] dictionaryColumnCardinality = CarbonUtil.getFormattedCardinality(dimLensWithComplex, wrapperColumnSchemaList); List<Integer> cardinality = new ArrayList<>(); List<org.apache.carbondata.format.ColumnSchema> columnSchemaList = AbstractFactDataWriter .getColumnSchemaListAndCardinality(cardinality, dictionaryColumnCardinality, wrapperColumnSchemaList); FileHeader fileHeader = CarbonMetadataUtil.getFileHeader(true, columnSchemaList, System.currentTimeMillis()); fileHeader.setIs_footer_present(false); fileHeader.setIs_splitable(true); fileHeader.setSync_marker(CarbonStreamOutputFormat.CARBON_SYNC_MARKER); fileHeader.setCompressor_name(compressorName); outputStream.write(CarbonUtil.getByteArray(fileHeader)); }
private byte[] getSyncMarker(String filePath) throws IOException { CarbonHeaderReader headerReader = new CarbonHeaderReader(filePath); FileHeader header = headerReader.readHeader(); // legacy store does not have this member if (header.isSetCompressor_name()) { compressorName = header.getCompressor_name(); } else { compressorName = CompressorFactory.NativeSupportedCompressor.SNAPPY.getName(); } return header.getSync_marker(); }
public DataFileFooter convertDataFileFooter(FileHeader fileHeader, FileFooter3 footer) { DataFileFooter dataFileFooter = new DataFileFooter(); dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) fileHeader.getVersion())); dataFileFooter.setNumberOfRows(footer.getNum_rows()); dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info())); dataFileFooter.setSchemaUpdatedTimeStamp(fileHeader.getTime_stamp()); List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>(); List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema(); for (int i = 0; i < table_columns.size(); i++) { columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(table_columns.get(i)));
if (header.isSetCompressor_name()) { compressorName = header.getCompressor_name(); } else { compressorName = CompressorFactory.NativeSupportedCompressor.SNAPPY.getName();
/** * Performs a deep copy on <i>other</i>. */ public FileHeader(FileHeader other) { __isset_bitfield = other.__isset_bitfield; this.version = other.version; if (other.isSetColumn_schema()) { List<org.apache.carbondata.format.ColumnSchema> __this__column_schema = new ArrayList<org.apache.carbondata.format.ColumnSchema>(other.column_schema.size()); for (org.apache.carbondata.format.ColumnSchema other_element : other.column_schema) { __this__column_schema.add(new org.apache.carbondata.format.ColumnSchema(other_element)); } this.column_schema = __this__column_schema; } this.is_footer_present = other.is_footer_present; this.time_stamp = other.time_stamp; this.is_splitable = other.is_splitable; if (other.isSetSync_marker()) { this.sync_marker = org.apache.thrift.TBaseHelper.copyBinary(other.sync_marker); } if (other.isSetCompressor_name()) { this.compressor_name = other.compressor_name; } }
@Override public List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException { CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(tableBlockInfo.getFilePath()); FileHeader fileHeader = carbonHeaderReader.readHeader(); List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>(); List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema(); for (int i = 0; i < table_columns.size(); i++) { columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(table_columns.get(i))); } return columnSchemaList; }
@Override public TBase create() { return new FileHeader(); } }, conf);
/** * Read and return the schema in the header */ public List<ColumnSchema> readSchema() throws IOException { FileHeader fileHeader = readHeader(); List<ColumnSchema> columnSchemaList = new ArrayList<>(); List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema(); for (org.apache.carbondata.format.ColumnSchema table_column : table_columns) { ColumnSchema col = thriftColumnSchemaToWrapperColumnSchema(table_column); col.setColumnReferenceId(col.getColumnUniqueId()); columnSchemaList.add(col); } return columnSchemaList; } }
public FileHeader deepCopy() { return new FileHeader(this); }
boolean this_present_column_schema = true && this.isSetColumn_schema(); boolean that_present_column_schema = true && that.isSetColumn_schema(); if (this_present_column_schema || that_present_column_schema) { if (!(this_present_column_schema && that_present_column_schema)) boolean this_present_is_footer_present = true && this.isSetIs_footer_present(); boolean that_present_is_footer_present = true && that.isSetIs_footer_present(); if (this_present_is_footer_present || that_present_is_footer_present) { if (!(this_present_is_footer_present && that_present_is_footer_present)) boolean this_present_time_stamp = true && this.isSetTime_stamp(); boolean that_present_time_stamp = true && that.isSetTime_stamp(); if (this_present_time_stamp || that_present_time_stamp) { if (!(this_present_time_stamp && that_present_time_stamp)) boolean this_present_is_splitable = true && this.isSetIs_splitable(); boolean that_present_is_splitable = true && that.isSetIs_splitable(); if (this_present_is_splitable || that_present_is_splitable) { if (!(this_present_is_splitable && that_present_is_splitable)) boolean this_present_sync_marker = true && this.isSetSync_marker(); boolean that_present_sync_marker = true && that.isSetSync_marker(); if (this_present_sync_marker || that_present_sync_marker) { if (!(this_present_sync_marker && that_present_sync_marker)) boolean this_present_compressor_name = true && this.isSetCompressor_name(); boolean that_present_compressor_name = true && that.isSetCompressor_name(); if (this_present_compressor_name || that_present_compressor_name) { if (!(this_present_compressor_name && that_present_compressor_name))
boolean present_column_schema = true && (isSetColumn_schema()); list.add(present_column_schema); if (present_column_schema) list.add(column_schema); boolean present_is_footer_present = true && (isSetIs_footer_present()); list.add(present_is_footer_present); if (present_is_footer_present) list.add(is_footer_present); boolean present_time_stamp = true && (isSetTime_stamp()); list.add(present_time_stamp); if (present_time_stamp) list.add(time_stamp); boolean present_is_splitable = true && (isSetIs_splitable()); list.add(present_is_splitable); if (present_is_splitable) list.add(is_splitable); boolean present_sync_marker = true && (isSetSync_marker()); list.add(present_sync_marker); if (present_sync_marker) list.add(sync_marker); boolean present_compressor_name = true && (isSetCompressor_name()); list.add(present_compressor_name); if (present_compressor_name)
lastComparison = Boolean.valueOf(isSetVersion()).compareTo(other.isSetVersion()); if (lastComparison != 0) { return lastComparison; if (isSetVersion()) { lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.version, other.version); if (lastComparison != 0) { lastComparison = Boolean.valueOf(isSetColumn_schema()).compareTo(other.isSetColumn_schema()); if (lastComparison != 0) { return lastComparison; if (isSetColumn_schema()) { lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column_schema, other.column_schema); if (lastComparison != 0) { lastComparison = Boolean.valueOf(isSetIs_footer_present()).compareTo(other.isSetIs_footer_present()); if (lastComparison != 0) { return lastComparison; if (isSetIs_footer_present()) { lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.is_footer_present, other.is_footer_present); if (lastComparison != 0) { lastComparison = Boolean.valueOf(isSetTime_stamp()).compareTo(other.isSetTime_stamp()); if (lastComparison != 0) { return lastComparison; if (isSetTime_stamp()) {
if (isSetIs_footer_present()) { if (!first) sb.append(", "); sb.append("is_footer_present:"); first = false; if (isSetTime_stamp()) { if (!first) sb.append(", "); sb.append("time_stamp:"); first = false; if (isSetIs_splitable()) { if (!first) sb.append(", "); sb.append("is_splitable:"); first = false; if (isSetSync_marker()) { if (!first) sb.append(", "); sb.append("sync_marker:"); if (isSetCompressor_name()) { if (!first) sb.append(", "); sb.append("compressor_name:");
if (struct.isSetIs_footer_present()) { optionals.set(0); if (struct.isSetTime_stamp()) { optionals.set(1); if (struct.isSetIs_splitable()) { optionals.set(2); if (struct.isSetSync_marker()) { optionals.set(3); if (struct.isSetCompressor_name()) { optionals.set(4); if (struct.isSetIs_footer_present()) { oprot.writeBool(struct.is_footer_present); if (struct.isSetTime_stamp()) { oprot.writeI64(struct.time_stamp); if (struct.isSetIs_splitable()) { oprot.writeBool(struct.is_splitable); if (struct.isSetSync_marker()) { oprot.writeBinary(struct.sync_marker); if (struct.isSetCompressor_name()) { oprot.writeString(struct.compressor_name);
public void write(org.apache.thrift.protocol.TProtocol oprot, FileHeader struct) throws org.apache.thrift.TException { struct.validate(); if (struct.isSetIs_footer_present()) { oprot.writeFieldBegin(IS_FOOTER_PRESENT_FIELD_DESC); oprot.writeBool(struct.is_footer_present); oprot.writeFieldEnd(); if (struct.isSetTime_stamp()) { oprot.writeFieldBegin(TIME_STAMP_FIELD_DESC); oprot.writeI64(struct.time_stamp); oprot.writeFieldEnd(); if (struct.isSetIs_splitable()) { oprot.writeFieldBegin(IS_SPLITABLE_FIELD_DESC); oprot.writeBool(struct.is_splitable); if (struct.isSetSync_marker()) { oprot.writeFieldBegin(SYNC_MARKER_FIELD_DESC); oprot.writeBinary(struct.sync_marker); if (struct.isSetCompressor_name()) { oprot.writeFieldBegin(COMPRESSOR_NAME_FIELD_DESC); oprot.writeString(struct.compressor_name);