@Override ColumnWriterBase createColumnWriter(ColumnDescriptor path, PageWriter pageWriter, ParquetProperties props) { return new ColumnWriterV1(path, pageWriter, props); } }
@Override public long getAllocatedSize() { Collection<ColumnWriterV1> values = columns.values(); long total = 0; for (ColumnWriterV1 memColumn : values) { total += memColumn.allocatedSize(); } return total; }
public void close() { Collection<ColumnWriterV1> values = columns.values(); for (ColumnWriterV1 memColumn : values) { memColumn.close(); } }
@Override public void write(float value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeFloat(value); updateStatistics(value); accountForValueWritten(); }
@Override public void writeNull(int repetitionLevel, int definitionLevel) { if (DEBUG) log(null, repetitionLevel, definitionLevel); repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); updateStatisticsNumNulls(); accountForValueWritten(); }
public String memUsageString(String indent) { StringBuilder b = new StringBuilder(indent).append(path).append(" {\n"); b.append(repetitionLevelColumn.memUsageString(indent + " r:")).append("\n"); b.append(definitionLevelColumn.memUsageString(indent + " d:")).append("\n"); b.append(dataColumn.memUsageString(indent + " data:")).append("\n"); b.append(pageWriter.memUsageString(indent + " pages:")).append("\n"); b.append(indent).append(String.format(" total: %,d/%,d", getBufferedSizeInMemory(), allocatedSize())).append("\n"); b.append(indent).append("}\n"); return b.toString(); } }
@Override public long getBufferedSize() { Collection<ColumnWriterV1> values = columns.values(); long total = 0; for (ColumnWriterV1 memColumn : values) { total += memColumn.getBufferedSizeInMemory(); } return total; }
@Override public void flush() { Collection<ColumnWriterV1> values = columns.values(); for (ColumnWriterV1 memColumn : values) { memColumn.flush(); } }
public ColumnWriterV1(ColumnDescriptor path, PageWriter pageWriter, ParquetProperties props) { this.path = path; this.pageWriter = pageWriter; this.props = props; // initial check of memory usage. So that we have enough data to make an initial prediction this.valueCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck(); resetStatistics(); this.repetitionLevelColumn = props.newRepetitionLevelWriter(path); this.definitionLevelColumn = props.newDefinitionLevelWriter(path); this.dataColumn = props.newValuesWriter(path); }
@Override public String memUsageString() { StringBuilder b = new StringBuilder("Store {\n"); Collection<ColumnWriterV1> values = columns.values(); for (ColumnWriterV1 memColumn : values) { b.append(memColumn.memUsageString(" ")); } b.append("}\n"); return b.toString(); }
@Override public void write(int value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeInteger(value); updateStatistics(value); accountForValueWritten(); }
public long maxColMemSize() { Collection<ColumnWriterV1> values = columns.values(); long max = 0; for (ColumnWriterV1 memColumn : values) { max = Math.max(max, memColumn.getBufferedSizeInMemory()); } return max; }
@Override public void close() { flush(); // Close the Values writers. repetitionLevelColumn.close(); definitionLevelColumn.close(); dataColumn.close(); }
private void writePage() { if (DEBUG) LOG.debug("write page"); try { pageWriter.writePage( concat(repetitionLevelColumn.getBytes(), definitionLevelColumn.getBytes(), dataColumn.getBytes()), valueCount, statistics, repetitionLevelColumn.getEncoding(), definitionLevelColumn.getEncoding(), dataColumn.getEncoding()); } catch (IOException e) { throw new ParquetEncodingException("could not write page for " + path, e); } repetitionLevelColumn.reset(); definitionLevelColumn.reset(); dataColumn.reset(); valueCount = 0; resetStatistics(); }
@Override public void write(Binary value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeBytes(value); updateStatistics(value); accountForValueWritten(); }
private ColumnWriterV1 newMemColumn(ColumnDescriptor path) { PageWriter pageWriter = pageWriteStore.getPageWriter(path); return new ColumnWriterV1(path, pageWriter, props); }
@Override public String toString() { StringBuilder sb = new StringBuilder(); for (Entry<ColumnDescriptor, ColumnWriterV1> entry : columns.entrySet()) { sb.append(Arrays.toString(entry.getKey().getPath())).append(": "); sb.append(entry.getValue().getBufferedSizeInMemory()).append(" bytes"); sb.append("\n"); } return sb.toString(); }
@Override public void write(double value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeDouble(value); updateStatistics(value); accountForValueWritten(); }
@Override public void write(boolean value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeBoolean(value); updateStatistics(value); accountForValueWritten(); }
@Override public void write(long value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeLong(value); updateStatistics(value); accountForValueWritten(); }