/** * Perform the actual writing of the given record into the backing file. */ public void write(HoodieRecord record, Optional<IndexedRecord> avroRecord, Optional<Exception> exception) { Optional recordMetadata = record.getData().getMetadata(); if (exception.isPresent() && exception.get() instanceof Throwable) { // Not throwing exception from here, since we don't want to fail the entire job for a single record writeStatus.markFailure(record, exception.get(), recordMetadata); logger.error("Error writing record " + record, exception.get()); } else { write(record, avroRecord); } }
public HoodieIOHandle(HoodieWriteConfig config, String commitTime, HoodieTable<T> hoodieTable) { this.commitTime = commitTime; this.config = config; this.fs = hoodieTable.getMetaClient().getFs(); this.hoodieTable = hoodieTable; this.hoodieTimeline = hoodieTable.getCompletedCommitTimeline(); this.schema = createHoodieWriteSchema(config); this.timer = new HoodieTimer().startTimer(); }
public HoodieIOHandle(HoodieWriteConfig config, String commitTime, HoodieTable<T> hoodieTable) { this.commitTime = commitTime; this.config = config; this.fs = hoodieTable.getMetaClient().getFs(); this.hoodieTable = hoodieTable; this.hoodieTimeline = hoodieTable.getCompletedCommitsTimeline(); this.schema = createHoodieWriteSchema(config); this.timer = new HoodieTimer().startTimer(); this.writeStatus = ReflectionUtils.loadClass(config.getWriteStatusClassName()); }
@Override protected List<WriteStatus> computeNext() { // Executor service used for launching writer thread. BoundedInMemoryExecutor<HoodieRecord<T>, HoodieInsertValueGenResult<HoodieRecord>, List<WriteStatus>> bufferedIteratorExecutor = null; try { final Schema schema = HoodieIOHandle.createHoodieWriteSchema(hoodieConfig); bufferedIteratorExecutor = new SparkBoundedInMemoryExecutor<>(hoodieConfig, inputItr, getInsertHandler(), getTransformFunction(schema)); final List<WriteStatus> result = bufferedIteratorExecutor.execute(); assert result != null && !result.isEmpty() && !bufferedIteratorExecutor.isRemaining(); return result; } catch (Exception e) { throw new HoodieException(e); } finally { if (null != bufferedIteratorExecutor) { bufferedIteratorExecutor.shutdownNow(); } } }
@Override protected List<WriteStatus> computeNext() { // Executor service used for launching writer thread. BoundedInMemoryExecutor<HoodieRecord<T>, Tuple2<HoodieRecord<T>, Optional<IndexedRecord>>, List<WriteStatus>> bufferedIteratorExecutor = null; try { final Schema schema = HoodieIOHandle.createHoodieWriteSchema(hoodieConfig); bufferedIteratorExecutor = new SparkBoundedInMemoryExecutor<>(hoodieConfig, inputItr, getInsertHandler(), getTransformFunction(schema)); final List<WriteStatus> result = bufferedIteratorExecutor.execute(); assert result != null && !result.isEmpty() && !bufferedIteratorExecutor.isRemaining(); return result; } catch (Exception e) { throw new HoodieException(e); } finally { if (null != bufferedIteratorExecutor) { bufferedIteratorExecutor.shutdownNow(); } } }