Refine search
static Tuple coerceToString(SinkCall<?, ?> sinkCall) { TupleEntry entry = sinkCall.getOutgoingEntry(); Fields fields = entry.getFields(); Tuple tuple = entry.getTuple(); if (fields.hasTypes()) { Type types[] = new Type[fields.size()]; for (int index = 0; index < fields.size(); index++) { Type type = fields.getType(index); if (type instanceof CoercibleType<?>) { types[index] = String.class; } else { types[index] = type; } } tuple = entry.getCoercedTuple(types); } return tuple; } }
static List<String> asStrings(Fields fields) { if (fields == null || !fields.isDefined()) { // use auto-generated name return Collections.emptyList(); } int size = fields.size(); List<String> names = new ArrayList<String>(size); for (int fieldIndex = 0; fieldIndex < size; fieldIndex++) { names.add(fields.get(fieldIndex).toString()); } return names; }
@Test public void testUnGroup() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungrouped" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num", "char" ), new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testAppend2() { Fields fieldA = new Fields( 0, 1 ); Fields fieldB = new Fields( 2 ); Fields appended = fieldA.append( fieldB ); assertEquals( "not equal: ", 3, appended.size() ); assertEquals( "not equal: ", 0, appended.get( 0 ) ); assertEquals( "not equal: ", 1, appended.get( 1 ) ); assertEquals( "not equal: ", 2, appended.get( 2 ) ); }
@Test public void testSameSourceGroupSplitCoGroup() throws Exception { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ).applyTypes( Long.TYPE, String.class ), inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath(), SinkMode.REPLACE ); Function splitter = new RegexSplitter( new Fields( "num", "char" ).applyTypes( String.class, String.class ), " " ); Pipe sourcePipe = new Each( new Pipe( "source" ), new Fields( "line" ), splitter ); sourcePipe = new GroupBy( sourcePipe, new Fields( "num" ) ); sourcePipe = new Every( sourcePipe, new Fields( "char" ), new First( new Fields( "first", String.class ) ), Fields.ALL ); Pipe lhsPipe = new Retain( new Pipe( "lhs", sourcePipe ), Fields.ALL ); Pipe rhsPipe = new Retain( new Pipe( "rhs", sourcePipe ), Fields.ALL ); Pipe splice = new CoGroup( lhsPipe, new Fields( "num" ), rhsPipe, new Fields( "num" ), Fields.size( 4 ) ); Map<Object, Object> properties = getPlatform().getProperties(); properties.put( "cascading.serialization.types.required", "true" ); Flow flow = getPlatform().getFlowConnector( properties ).connect( source, sink, splice ); flow.complete(); validateLength( flow, 5, null ); List<Tuple> values = getSinkAsList( flow ); assertTrue( values.contains( new Tuple( "1\ta\t1\ta" ) ) ); assertTrue( values.contains( new Tuple( "2\tb\t2\tb" ) ) ); }
@Test public void testSinkUnknown() throws IOException { getPlatform().copyFromLocal( inputFileCross ); Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileCross ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third" ), "\\s" ), Fields.RESULTS ); Tap sink = getPlatform().getTabDelimitedFile( Fields.UNKNOWN, getOutputPath( "unknownsinks" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 37, null ); TupleEntryIterator iterator = flow.openSink(); String line = iterator.next().getTuple().toString(); assertTrue( "not equal: wrong values: " + line, line.matches( "[0-9]\t[a-z]\t[A-Z]" ) ); iterator.close(); }
private TupleEntry getEntry( String[] names, Class[] types, Object... values ) { Fields fields = new Fields( names ).applyTypes( types ); Tuple parameters = new Tuple( values ); return new TupleEntry( fields, parameters ); }
void runTestCount( String name, Fields argumentSelector, Fields fieldDeclaration, Fields outputSelector ) throws Exception { getPlatform().copyFromLocal( inputFileIps ); Tap source = getPlatform().getTextFile( Fields.size( 2 ), inputFileIps ); Tap sink = getPlatform().getTextFile( Fields.size( 1 ), getOutputPath( name ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, argumentSelector, new Count( fieldDeclaration ), outputSelector ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.start(); // simple test for start flow.complete(); validateLength( flow, 17 ); assertTrue( getSinkAsList( flow ).contains( new Tuple( "63.123.238.8\t2" ) ) ); }
@Test public void testResolveIndexOnly2() { Fields declarationA = new Fields( 0, 1 ); Fields declarationB = new Fields( 0, 1 ); Fields selectA = new Fields( 1 ); Fields got = Fields.resolve( selectA, declarationA, declarationB ); assertEquals( "not equal: ", 1, got.size() ); assertEquals( "not equal: ", 1, got.get( 0 ) ); }
@Test public void testSelectPos() { Fields declarationA = new Fields( 0, 1 ); Fields selectA = new Fields( 0, 1 ); Fields got = declarationA.select( selectA ); assertEquals( "not equal: ", 2, got.size() ); assertEquals( "not equal: ", 0, got.get( 0 ) ); assertEquals( "not equal: ", 1, got.get( 1 ) ); }
@Test public void testTypedSelect() { Fields declarationA = new Fields( names( "a", "b" ), types( int.class, String.class ) ); Fields selectA = new Fields( "a", "b" ); Fields got = declarationA.select( selectA ); assertEquals( "not equal: ", 2, got.size() ); assertEquals( "not equal: ", int.class, got.getType( 0 ) ); assertEquals( "not equal: ", String.class, got.getType( 1 ) ); }
@Test public void testTrapNoOperation() throws Exception { getPlatform().copyFromLocal( testDelimitedProblematic ); Tap source = getPlatform().getDelimitedFile( new Fields( "id", "name" ).applyTypes( int.class, String.class ), ",", testDelimitedProblematic ); Tap sink = getPlatform().getDelimitedFile( new Fields( "id", "name" ).applyTypes( int.class, String.class ), ",", getOutputPath( getTestName() ), SinkMode.REPLACE ); Tap trap = getPlatform().getTextFile( getOutputPath( getTestName() + "_trap" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "copy" ); FlowDef flowDef = FlowDef.flowDef() .addSource( pipe, source ) .addTailSink( pipe, sink ) .addTrap( pipe, trap ); Flow flow = getPlatform().getFlowConnector().connect( flowDef ); flow.complete(); validateLength( flow.openTrap(), 1 ); }
@Test public void testTypedAppendToNone() { Fields fieldA = Fields.NONE; Fields fieldB = new Fields( names( "a" ), types( String.class ) ); Fields appended = fieldA.append( fieldB ); assertEquals( "not equal: ", 1, appended.size() ); assertEquals( "not equal: ", "a", appended.get( 0 ) ); assertEquals( "not equal: ", String.class, appended.getType( 0 ) ); }
@Override public Fields getDeclaredFields() { Fields sumName = new Fields( AverageBy.class.getPackage().getName() + "." + declaredFields.get( 0 ) + ".sum", Double.class ); Fields countName = new Fields( AverageBy.class.getPackage().getName() + "." + declaredFields.get( 0 ) + ".count", Long.class ); return sumName.append( countName ); }
@Override public void setUp() throws Exception { Logging.setLogLevel( this.getClass(), "cascading.pattern", "debug" ); super.setUp(); expectedFields = Fields.NONE .append( new Fields( "var0", double.class ) ) .append( new Fields( "var1", String.class ) ) .append( new Fields( "var2", double.class ) ); Tuple tuple = new Tuple( 0.d, "value", null ); tupleEntry = new TupleEntry( expectedFields, tuple ); }
@Test public void testTypedDiff() { Fields fieldA = new Fields( names( "a", "b" ), types( int.class, String.class ) ); Fields fieldB = new Fields( "a" ); Fields diff = fieldA.subtract( fieldB ); assertEquals( "not equal: ", 1, diff.size() ); assertEquals( "not equal: ", String.class, diff.getType( 0 ) ); }
/** * Method appendNew appends the given TupleEntry instance to this instance. * * @param entry of type TupleEntry * @return TupleEntry */ public TupleEntry appendNew( TupleEntry entry ) { Fields appendedFields = fields.append( entry.fields.isUnknown() ? Fields.size( entry.tuple.size() ) : entry.fields ); Tuple appendedTuple = tuple.append( entry.tuple ); return new TupleEntry( appendedFields, appendedTuple ); }
private static Fields toFields( DDLColumn[] ddlColumns ) { Fields fields = Fields.NONE; for( DDLColumn ddlColumn : ddlColumns ) fields = fields.append( new Fields( ddlColumn.name ).applyTypes( ddlColumn.type ) ); return fields; } }
public void testResolveWithTypes() { Fields selector = new Fields( "ip" ); Fields fields[] = new Fields[]{new Fields( "ip" ).applyTypes( String.class ), new Fields( "offset", "line" ).applyTypes( String.class, String.class )}; Fields resolved = Fields.resolve( selector, fields ); assertTrue( resolved.hasTypes() ); }