@Override public String toString(Object field) { if (field instanceof SinkCall) { return ((SinkCall) field).getOutgoingEntry().toString(); } return field.toString(); } }
static Tuple coerceToString(SinkCall<?, ?> sinkCall) { return sinkCall.getOutgoingEntry().getTuple(); } }
@SuppressWarnings({ "rawtypes" }) @Override protected Object extractField(Object target) { List<String> fieldNames = getFieldNames(); for (int i = 0; i < fieldNames.size(); i++) { if (target instanceof SinkCall) { target = ((SinkCall) target).getOutgoingEntry().getObject(fieldNames.get(i)); if (target == null) { return NOT_FOUND; } } else { return NOT_FOUND; } } return target; }
@Override public void convert(Object from, BytesArray to) { // expect a tuple holding one field - chararray or bytearray Assert.isTrue(from instanceof SinkCall, String.format("Unexpected object type, expecting [%s], given [%s]", SinkCall.class, from.getClass())); // handle common cases SinkCall sinkCall = (SinkCall) from; Tuple rawTuple = sinkCall.getOutgoingEntry().getTuple(); if (rawTuple == null || rawTuple.isEmpty()) { to.bytes("{}"); return; } Assert.isTrue(rawTuple.size() == 1, "When using JSON input, only one field is expected"); // postpone the coercion Tuple tuple = CascadingUtils.coerceToString(sinkCall); super.convert(tuple.getObject(0), to); } }
static Tuple coerceToString(SinkCall<?, ?> sinkCall) { TupleEntry entry = sinkCall.getOutgoingEntry(); Fields fields = entry.getFields(); Tuple tuple = entry.getTuple(); if (fields.hasTypes()) { Type types[] = new Type[fields.size()]; for (int index = 0; index < fields.size(); index++) { Type type = fields.getType(index); if (type instanceof CoercibleType<?>) { types[index] = String.class; } else { types[index] = type; } } tuple = entry.getCoercedTuple(types); } return tuple; } }
@Override public String toString(Object field) { if (field instanceof SinkCall) { return ((SinkCall) field).getOutgoingEntry().toString(); } return field.toString(); } }
@Override public void sink(FlowProcess<? extends JobConf> fp, SinkCall<Object[], OutputCollector> sink) throws IOException { TupleEntry tuple = sink.getOutgoingEntry(); OutputCollector outputCollector = sink.getOutput(); outputCollector.collect(null, tuple); } }
@Override public void sink( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) throws IOException { sinkCall.getContext().println( sinkCall.getOutgoingEntry().getTuple().toString() ); }
@Override public void sink(FlowProcess<JobConf> flowProcess, SinkCall<Void, OutputCollector> sinkCall) throws IOException { TupleEntry tupleEntry = sinkCall.getOutgoingEntry(); byte[] key = (byte[]) tupleEntry.getObject(0); byte[] val = (byte[]) tupleEntry.getObject(1); sinkCall.getOutput().collect(new BytesWritable(key), new BytesWritable(val)); } }
protected void writeHeader( SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { Fields fields = sinkCall.getOutgoingEntry().getFields(); Text text = (Text) sinkCall.getContext()[ 0 ]; StringBuilder line = (StringBuilder) sinkCall.getContext()[ 1 ]; Charset charset = (Charset) sinkCall.getContext()[ 2 ]; line = (StringBuilder) delimitedParser.joinFirstLine( fields, line ); text.set( line.toString().getBytes( charset ) ); sinkCall.getOutput().collect( null, text ); line.setLength( 0 ); }
protected void writeHeader( SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { Fields fields = sinkCall.getOutgoingEntry().getFields(); Text text = (Text) sinkCall.getContext()[ 0 ]; StringBuilder line = (StringBuilder) sinkCall.getContext()[ 1 ]; Charset charset = (Charset) sinkCall.getContext()[ 2 ]; line = (StringBuilder) delimitedParser.joinFirstLine( fields, line ); text.set( line.toString().getBytes( charset ) ); sinkCall.getOutput().collect( null, text ); line.setLength( 0 ); }
@Override public void sink( FlowProcess<? extends Configuration> flowProcess, SinkCall<Void, OutputCollector> sinkCall ) throws IOException { sinkCall.getOutput().collect( Tuple.NULL, sinkCall.getOutgoingEntry().getTuple() ); }
@SuppressWarnings("unchecked") @Override public void sink(FlowProcess<? extends JobConf> fp, SinkCall<Object[], OutputCollector> sc) throws IOException { TupleEntry tuple = sc.getOutgoingEntry(); if (tuple.size() != 1) { throw new RuntimeException("ParquetValueScheme expects tuples with an arity of exactly 1, but found " + tuple.getFields()); } T value = (T) tuple.getObject(0); OutputCollector output = sc.getOutput(); output.collect(null, value); }
@Override public void sink(FlowProcess<JobConf> jobConfFlowProcess, SinkCall<Void, OutputCollector> sinkCall) throws IOException { TupleEntry tupleEntry = sinkCall.getOutgoingEntry(); IntWritable partition = new IntWritable(tupleEntry.getInteger(partitionFieldName)); BytesWritable key = (BytesWritable) tupleEntry.getObject(keyFieldName); BytesWritable value = (BytesWritable) tupleEntry.getObject(valueFieldName); KeyAndPartitionWritable keyAndPartitionWritable = new KeyAndPartitionWritable(key, partition); ValueWritable valueWritable = new ValueWritable(value); sinkCall.getOutput().collect(keyAndPartitionWritable, valueWritable); } }
@Override public void sink( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { Fields found = sinkCall.getOutgoingEntry().getFields(); if( !found.equals( expectedFields ) ) throw new RuntimeException( "fields to not match, expect: " + expectedFields + ", found: " + found ); super.sink( flowProcess, sinkCall ); } }
@Override public void sinkPrepare( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { Fields found = sinkCall.getOutgoingEntry().getFields(); if( !found.equals( expectedFields ) ) throw new RuntimeException( "fields to not match, expect: " + expectedFields + ", found: " + found ); super.sinkPrepare( flowProcess, sinkCall ); }
@Override public void sinkPrepare( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { Fields found = sinkCall.getOutgoingEntry().getFields(); if( !found.equals( expectedFields ) ) throw new RuntimeException( "fields to not match, expect: " + expectedFields + ", found: " + found ); super.sinkPrepare( flowProcess, sinkCall ); }
@Override public void sinkPrepare( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) { OutputStream originalOutput = sinkCall.getOutput(); sinkCall.setContext( createOutput( originalOutput ) ); if( writeHeader && !isAppendingFile( sinkCall, originalOutput ) ) { Fields fields = sinkCall.getOutgoingEntry().getFields(); delimitedParser.joinFirstLine( fields, sinkCall.getContext() ); sinkCall.getContext().println(); } }
@Override public void sink(FlowProcess<JobConf> flowProcess, SinkCall<T, OutputCollector> sinkCall) throws IOException { OutputCollector collector = sinkCall.getOutput(); TupleEntry entry = sinkCall.getOutgoingEntry(); T writable = sinkCall.getContext(); writable.set((M) entry.getTuple().getObject(0)); collector.collect(null, writable); }
@Override public void sink(FlowProcess<? extends Configuration> flowProcess, SinkCall<T, OutputCollector> sinkCall) throws IOException { OutputCollector collector = sinkCall.getOutput(); TupleEntry entry = sinkCall.getOutgoingEntry(); T writable = sinkCall.getContext(); writable.set((M) entry.getTuple().getObject(0)); collector.collect(null, writable); }