@Deprecated ColumnWriteStoreBase( final PageWriteStore pageWriteStore, final ParquetProperties props) { this.props = props; this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO); this.columns = new TreeMap<>(); this.rowCountForNextSizeCheck = min(props.getMinRowCountForPageSizeCheck(), props.getPageRowCountLimit()); columnWriterProvider = new ColumnWriterProvider() { @Override public ColumnWriter getColumnWriter(ColumnDescriptor path) { ColumnWriterBase column = columns.get(path); if (column == null) { column = createColumnWriter(path, pageWriteStore.getPageWriter(path), props); columns.put(path, column); } return column; } }; }
ColumnWriteStoreBase( MessageType schema, PageWriteStore pageWriteStore, ParquetProperties props) { this.props = props; this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO); Map<ColumnDescriptor, ColumnWriterBase> mcolumns = new TreeMap<>(); for (ColumnDescriptor path : schema.getColumns()) { PageWriter pageWriter = pageWriteStore.getPageWriter(path); mcolumns.put(path, createColumnWriter(path, pageWriter, props)); } this.columns = unmodifiableMap(mcolumns); this.rowCountForNextSizeCheck = min(props.getMinRowCountForPageSizeCheck(), props.getPageRowCountLimit()); columnWriterProvider = new ColumnWriterProvider() { @Override public ColumnWriter getColumnWriter(ColumnDescriptor path) { return columns.get(path); } }; }
private void sizeCheck() { long minRecordToWait = Long.MAX_VALUE; int pageRowCountLimit = props.getPageRowCountLimit(); long rowCountForNextRowCountCheck = rowCount + pageRowCountLimit; for (ColumnWriterBase writer : columns.values()) {
LOG.info("Max row count for page size check is: {}", props.getMaxRowCountForPageSizeCheck()); LOG.info("Truncate length for column indexes is: {}", props.getColumnIndexTruncateLength()); LOG.info("Page row count limit to {}", props.getPageRowCountLimit());