@Override public Tuple next() throws IOException { Tuple tuple; while(!context.isStopped() && (tuple = child.next()) != null) { appender.addTuple(tuple); if (maxPerFileSize > 0 && maxPerFileSize <= appender.getEstimatedOutputSize()) { appender.close(); writtenFileNum++; StatisticsUtil.aggregateTableStat(sumStats, appender.getStats()); openNewFile(writtenFileNum); } } return null; }
@Override public Tuple next() throws IOException { Tuple tuple; while(!context.isStopped() && (tuple = child.next()) != null) { appender.addTuple(tuple); if (maxPerFileSize > 0 && maxPerFileSize <= appender.getEstimatedOutputSize()) { appender.close(); writtenFileNum++; StatisticsUtil.aggregateTableStat(sumStats, appender.getStats()); openNewFile(writtenFileNum); } } return null; }
@Override public Tuple next() throws IOException { Tuple tuple; while(!context.isStopped() && (tuple = child.next()) != null) { if (prevKey == null) { appender = getNextPartitionAppender(getSubdirectory(tuple)); prevKey = new ComparableTuple(inSchema, keyIds); prevKey.set(tuple); } else if (!prevKey.equals(tuple)) { appender.close(); StatisticsUtil.aggregateTableStat(aggregatedStats, appender.getStats()); appender = getNextPartitionAppender(getSubdirectory(tuple)); prevKey.set(tuple); // reset all states for file rotating writtenFileNum = 0; } appender.addTuple(tuple); if (maxPerFileSize > 0 && maxPerFileSize <= appender.getEstimatedOutputSize()) { appender.close(); writtenFileNum++; StatisticsUtil.aggregateTableStat(aggregatedStats, appender.getStats()); openAppender(writtenFileNum); } } return null; }
@Override public Tuple next() throws IOException { Tuple tuple; while(!context.isStopped() && (tuple = child.next()) != null) { if (prevKey == null) { appender = getNextPartitionAppender(getSubdirectory(tuple)); prevKey = new ComparableTuple(inSchema, keyIds); prevKey.set(tuple); } else if (!prevKey.equals(tuple)) { appender.close(); StatisticsUtil.aggregateTableStat(aggregatedStats, appender.getStats()); appender = getNextPartitionAppender(getSubdirectory(tuple)); prevKey.set(tuple); // reset all states for file rotating writtenFileNum = 0; } appender.addTuple(tuple); if (maxPerFileSize > 0 && maxPerFileSize <= appender.getEstimatedOutputSize()) { appender.close(); writtenFileNum++; StatisticsUtil.aggregateTableStat(aggregatedStats, appender.getStats()); openAppender(writtenFileNum); } } return null; }