Refine search
@Override public void convert(Object from, BytesArray to) { // expect a tuple holding one field - chararray or bytearray Assert.isTrue(from instanceof SinkCall, String.format("Unexpected object type, expecting [%s], given [%s]", SinkCall.class, from.getClass())); // handle common cases SinkCall sinkCall = (SinkCall) from; Tuple rawTuple = sinkCall.getOutgoingEntry().getTuple(); if (rawTuple == null || rawTuple.isEmpty()) { to.bytes("{}"); return; } Assert.isTrue(rawTuple.size() == 1, "When using JSON input, only one field is expected"); // postpone the coercion Tuple tuple = CascadingUtils.coerceToString(sinkCall); super.convert(tuple.getObject(0), to); } }
@SuppressWarnings("unchecked") @Override public Result write(SinkCall<Object[], ?> sinkCall, Generator generator) { Tuple tuple = CascadingUtils.coerceToString(sinkCall); // consider names (in case of aliases these are already applied) List<String> names = (List<String>) sinkCall.getContext()[SINK_CTX_ALIASES]; generator.writeBeginObject(); for (int i = 0; i < tuple.size(); i++) { String name = (i < names.size() ? names.get(i) : "tuple" + i); // filter out fields if (shouldKeep(generator.getParentPath(), name)) { generator.writeFieldName(name); Object object = tuple.getObject(i); Result result = jdkWriter.write(object, generator); if (!result.isSuccesful()) { if (object instanceof Writable) { return writableWriter.write((Writable) object, generator); } return Result.FAILED(object); } } } generator.writeEndObject(); return Result.SUCCESFUL(); }
@Test public void testGeneratorAggregator() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new TestAggregator( new Fields( "count1" ), new Fields( "ip" ), new Tuple( "first1" ), new Tuple( "first2" ) ) ); pipe = new Every( pipe, new TestAggregator( new Fields( "count2" ), new Fields( "ip" ), new Tuple( "second" ), new Tuple( "second2" ), new Tuple( "second3" ) ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "generatoraggregator" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 8 * 2 * 3, null ); }
@Test(expected = OperationException.class) public void testGetMissingFail() throws Exception { TupleEntry entry = new TupleEntry( new Fields( "json", JSONCoercibleType.TYPE ), Tuple.size( 1 ) ); entry.setObject( 0, JSONData.nested ); JSONGetFunction function = new JSONGetFunction( new Fields( "result" ), true, "/person/foobar" ); invokeFunction( function, entry, new Fields( "result" ) ); }
void runTestCount( String name, Fields argumentSelector, Fields fieldDeclaration, Fields outputSelector ) throws Exception { getPlatform().copyFromLocal( inputFileIps ); Tap source = getPlatform().getTextFile( Fields.size( 2 ), inputFileIps ); Tap sink = getPlatform().getTextFile( Fields.size( 1 ), getOutputPath( name ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, argumentSelector, new Count( fieldDeclaration ), outputSelector ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.start(); // simple test for start flow.complete(); validateLength( flow, 17 ); assertTrue( getSinkAsList( flow ).contains( new Tuple( "63.123.238.8\t2" ) ) ); }
@Test public void testSelect() { Fields selector = new Fields( "a", "d" ); TupleEntry entryA = new TupleEntry( new Fields( "a", "b" ), new Tuple( "a", "b" ) ); TupleEntry entryB = new TupleEntry( new Fields( "c", "d" ), new Tuple( "c", "d" ) ); Tuple tuple = TupleEntry.select( selector, entryA, entryB ); assertEquals( "wrong size", 2, tuple.size() ); assertEquals( "not equal: tuple.get(0)", "a", tuple.getObject( 0 ) ); assertEquals( "not equal: tuple.get(1)", "d", tuple.getObject( 1 ) ); }
private TupleEntry getEntry( Comparable lhs, Comparable rhs ) { Fields fields = new Fields( "a", "b" ); Tuple parameters = new Tuple( lhs, rhs ); return new TupleEntry( fields, parameters ); }
@Override public void doAssert( FlowProcess flowProcess, ValueAssertionCall assertionCall ) { TupleEntry input = assertionCall.getArguments(); int pos = 0; for( Object element : input.getTuple() ) { if( !value.equals( element ) ) fail( input.getFields().get( pos ), element, value, input.getTuple().print() ); pos++; } }
@Test public void testSet() { TupleEntry entryA = new TupleEntry( new Fields( "a", "b", "c" ), new Tuple( "a", "b", "c" ) ); TupleEntry entryB = new TupleEntry( new Fields( "c", "b" ), new Tuple( "C", "B" ) ); entryA.set( entryB ); Tuple tuple = entryA.getTuple(); assertEquals( "wrong size", 3, tuple.size() ); assertEquals( "not equal: tuple.get(0)", "a", tuple.getObject( 0 ) ); assertEquals( "not equal: tuple.get(1)", "B", tuple.getObject( 1 ) ); assertEquals( "not equal: tuple.get(2)", "C", tuple.getObject( 2 ) ); }
@Test public void testWildcard() { Fields aFields = Fields.ALL; Tuple aTuple = tuple.get( fields, aFields ); assertEquals( "not equal: aTuple.size()", 5, aTuple.size() ); assertEquals( "not equal: aTuple.get( 0 )", "a", aTuple.get( fields, new Fields( "one" ) ).getObject( 0 ) ); assertEquals( "not equal: aTuple.get( 1 )", "b", aTuple.get( fields, new Fields( "two" ) ).getObject( 0 ) ); }
public void testSetCoerce() { Fields fieldsA = new Fields( "a", "b", "c" ).applyTypes( String.class, String.class, String.class ); Tuple tupleA = new Tuple( "0", "1", "2" ); Fields fieldsB = new Fields( "c", "b" ).applyTypes( Integer.class, Integer.class ); Tuple tupleB = new Tuple( -2, -1 ); tupleA.set( fieldsA, fieldsB, tupleB ); assertEquals( "wrong size", 3, tupleA.size() ); assertEquals( "not equal: tuple.get(0)", "0", tupleA.getObject( 0 ) ); assertEquals( "not equal: tuple.get(1)", "-1", tupleA.getObject( 1 ) ); assertEquals( "not equal: tuple.get(2)", "-2", tupleA.getObject( 2 ) ); } }
@Test public void testPut() { Fields aFields = new Fields( "one", "five" ); tuple.put( fields, aFields, new Tuple( "ten", "eleven" ) ); assertEquals( "not equal: tuple.size()", 5, tuple.size() ); assertEquals( "not equal: tuple.get( 0 )", "ten", tuple.getObject( 0 ) ); assertEquals( "not equal: tuple.get( 0 )", "ten", tuple.get( fields, new Fields( "one" ) ).getObject( 0 ) ); assertEquals( "not equal: tuple.get( 0 )", "eleven", tuple.getObject( 4 ) ); assertEquals( "not equal: tuple.get( 0 )", "eleven", tuple.get( fields, new Fields( "five" ) ).getObject( 0 ) ); }
private void useSize( TupleEntry input, TupleEntryCollector outputCollector ) { LOG.debug( "using size: {}", size ); Tuple tuple = new Tuple( input.getTuple() ); // make clone Tuple group = tuple.remove( input.getFields(), groupFieldSelector ); for( int i = 0; i < tuple.size(); i = i + size ) { Tuple result = new Tuple( group ); result.addAll( tuple.get( Fields.offsetSelector( size, i ).getPos() ) ); outputCollector.add( result ); } }
public Builder<C> addTuple(Object... values) { values = FieldTypeValidator.validateValues(fieldMask, values); TupleEntry newTuple = new TupleEntry(fields, Tuple.size(fields.size())); newTuple.setTuple(fieldMask, new Tuple(values)); tuples.add(newTuple); return this; }
/** * Method setInteger sets the value in the given field or position. * * @param fieldName field name or position to set * @param value of type int */ public void setInteger( Comparable fieldName, int value ) { int pos = fields.getPos( asFieldName( fieldName ) ); if( pos > coercions.length - 1 ) throw new TupleException( "position value is too large: " + pos + ", positions in field: " + tuple.size() ); tuple.set( pos, coercions[ pos ].canonical( value ) ); }
private TupleEntry getEntry( String[] names, Class[] types, Object... values ) { Fields fields = new Fields( names ).applyTypes( types ); Tuple parameters = new Tuple( values ); return new TupleEntry( fields, parameters ); }
@Test public void testReplace() throws IOException { RegexReplace splitter = new RegexReplace( new Fields( "words" ), "\\s+", "-", true ); Tuple arguments = new Tuple( "foo\t bar" ); Fields resultFields = Fields.UNKNOWN; TupleListCollector collector = invokeFunction( splitter, arguments, resultFields ); assertEquals( "wrong size", 1, collector.size() ); Iterator<Tuple> iterator = collector.iterator(); Tuple tuple = iterator.next(); assertEquals( "not equal: tuple.get(0)", "foo-bar", tuple.getObject( 0 ) ); }