@Override public TupleEntryCollector openForWrite(FlowProcess<Properties> flowProcess, Object output) throws IOException { return new TupleEntrySchemeCollector<Properties, Object>(flowProcess, getScheme(), output); }
protected void setOutput( Output output ) { sinkCall.setOutput( wrapOutput( output ) ); }
@Override public void add( Tuple tuple ) { if( !prepared ) // this is unfortunate prepare(); super.add( tuple ); }
public TupleEntrySchemeCollector( FlowProcess<? extends Config> flowProcess, Tap tap, Scheme scheme, Output output, Supplier<String> loggableIdentifier ) { super( Fields.asDeclaration( scheme.getSinkFields() ) ); this.flowProcess = flowProcess; this.scheme = scheme; if( loggableIdentifier != null ) this.loggableIdentifier = loggableIdentifier; // only used for logging this.sinkCall = createSinkCall(); this.sinkCall.setTap( tap ); this.sinkCall.setOutgoingEntry( this.tupleEntry ); // created in super ctor if( output != null ) setOutput( output ); }
@Override protected void collect(TupleEntry tupleEntry) throws IOException { super.collect(tupleEntry); handler.endMessage(); } };
@Override public void close() { try { LOG.info("closing tap collector for: {}", tap); writer.close(reporter); } catch (IOException exception) { LOG.warn("exception closing: {}", exception); throw new TapException("exception closing HBaseTapCollector", exception); } finally { super.close(); } }
@Override protected OutputCollector<?, ?> wrapOutput( OutputCollector outputCollector ) { if( measuredOutputCollector == null ) measuredOutputCollector = new MeasuredOutputCollector( getFlowProcess(), SliceCounters.Write_Duration ); measuredOutputCollector.setOutputCollector( super.wrapOutput( outputCollector ) ); return measuredOutputCollector; } }
@Override public void prepare() { try { initialize(); } catch (IOException e) { throw new RuntimeException(e); } super.prepare(); }
@Override protected TupleEntrySchemeCollector createTupleEntrySchemeCollector( FlowProcess<? extends Configuration> flowProcess, Tap parent, String path, long sequence ) throws IOException { TapOutputCollector outputCollector = new TapOutputCollector( flowProcess, parent, path, sequence ); return new TupleEntrySchemeCollector<Configuration, OutputCollector>( flowProcess, parent, outputCollector ); }
@Override protected OutputCollector<?, ?> wrapOutput( OutputCollector outputCollector ) { if( measuredOutputCollector == null ) measuredOutputCollector = new MeasuredOutputCollector( getFlowProcess(), SliceCounters.Write_Duration ); measuredOutputCollector.setOutputCollector( super.wrapOutput( outputCollector ) ); return measuredOutputCollector; } }
@Override public void add( TupleEntry tupleEntry ) { if( !prepared ) prepare(); super.add( tupleEntry ); }
@Override public TupleEntryCollector openForWrite( FlowProcess<? extends Properties> flowProcess, OutputStream output ) throws IOException { return new TupleEntrySchemeCollector<Properties, OutputStream>( flowProcess, this, getScheme(), System.err ); }
@Override public TupleEntryCollector openForWrite( FlowProcess<? extends Properties> flowProcess, OutputStream output ) throws IOException { return new TupleEntrySchemeCollector<Properties, OutputStream>( flowProcess, this, getScheme(), System.out ); }
@Override protected TupleEntrySchemeCollector createTupleEntrySchemeCollector( FlowProcess<? extends Configuration> flowProcess, Tap parent, String path, long sequence ) throws IOException { TapOutputCollector outputCollector = new TapOutputCollector( flowProcess, parent, path, sequence ); return new TupleEntrySchemeCollector<Configuration, OutputCollector>( flowProcess, parent, outputCollector ); }
@Override public TupleEntryCollector openForWrite(FlowProcess<Properties> flowProcess, Object output) throws IOException { return new TupleEntrySchemeCollector<Properties, Object>(flowProcess, getScheme(), output); }
@Override public TupleEntryCollector openForWrite(FlowProcess<Properties> flowProcess, OutputStream output) throws IOException { if (output == null) { // wrap tuple entry to indicate when an entry has finished return new TupleEntrySchemeCollector<Properties, OutputStream>(flowProcess, getScheme(), handler, getIdentifier()) { @Override protected void collect(TupleEntry tupleEntry) throws IOException { super.collect(tupleEntry); handler.endMessage(); } }; } return new TupleEntrySchemeCollector<Properties, OutputStream>(flowProcess, getScheme(), output, getIdentifier()); }
@Override protected TupleEntrySchemeCollector createTupleEntrySchemeCollector( FlowProcess<? extends Properties> flowProcess, Tap parent, String path, long sequence ) throws IOException { if( sequence != -1 && flowProcess.getConfig() != null ) ( (LocalFlowProcess) FlowProcessWrapper.undelegate( flowProcess ) ).getConfig().setProperty( PART_NUM_PROPERTY, Long.toString( sequence ) ); if( parent instanceof TapWith ) return (TupleEntrySchemeCollector) ( (TapWith) parent ) .withChildIdentifier( path ) .withSinkMode( SinkMode.UPDATE ) .asTap().openForWrite( flowProcess ); TapFileOutputStream output = new TapFileOutputStream( parent, path, true ); // always append return new TupleEntrySchemeCollector<Properties, OutputStream>( flowProcess, parent, output ); }
@Override public TupleEntryCollector openForWrite( FlowProcess<? extends Properties> flowProcess, OutputStream output ) throws IOException { if( output == null ) output = new TapFileOutputStream( getOutputIdentifier( flowProcess ), isUpdate() ); // append if we are in update mode return new TupleEntrySchemeCollector<Properties, OutputStream>( flowProcess, this, getScheme(), output, getIdentifier() ); }