@ConstructorProperties({"argumentSelectors", "filters"}) protected Logic( Fields[] argumentSelectors, Filter[] filters ) { this.argumentSelectors = argumentSelectors; this.filters = filters; verify(); this.numArgs = getFieldsSize(); }
/** * Creates a new Sample that permits percent Tuples to pass. * * @param fraction of type double */ @ConstructorProperties({"fraction"}) public Sample( double fraction ) { this.seed = makeSeed(); this.fraction = fraction; }
@Override public boolean isRemove( FlowProcess flowProcess, FilterCall<Context> filterCall ) { return filterCall.getContext().increment(); }
@Test public void testNot() { Fields[] fields = new Fields[]{new Fields( 0 ), new Fields( 1 )}; Filter[] filters = new Filter[]{new BooleanFilter( true ), new BooleanFilter( true )}; Filter filter = new Not( new Or( fields, filters ) ); assertFalse( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( true ), new BooleanFilter( false )}; filter = new Not( new Or( fields, filters ) ); assertFalse( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( false ), new BooleanFilter( true )}; filter = new Not( new Or( fields, filters ) ); assertFalse( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( false ), new BooleanFilter( false )}; filter = new Not( new Or( fields, filters ) ); assertTrue( invokeFilter( filter, new Tuple( 1, 2 ) ) ); }
@Test public void testAnd() { Fields[] fields = new Fields[]{new Fields( 0 ), new Fields( 1 )}; Filter[] filters = new Filter[]{new BooleanFilter( true ), new BooleanFilter( true )}; Filter filter = new And( fields, filters ); assertTrue( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( true ), new BooleanFilter( false )}; filter = new And( fields, filters ); assertFalse( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( false ), new BooleanFilter( true )}; filter = new And( fields, filters ); assertFalse( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( false ), new BooleanFilter( false )}; filter = new And( fields, filters ); assertFalse( invokeFilter( filter, new Tuple( 1, 2 ) ) ); }
@Test public void testXor() { Fields[] fields = new Fields[]{new Fields( 0 ), new Fields( 1 )}; Filter[] filters = new Filter[]{new BooleanFilter( true ), new BooleanFilter( true )}; Filter filter = new Xor( fields[ 0 ], filters[ 0 ], fields[ 1 ], filters[ 1 ] ); assertFalse( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( true ), new BooleanFilter( false )}; filter = new Xor( fields[ 0 ], filters[ 0 ], fields[ 1 ], filters[ 1 ] ); assertTrue( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( false ), new BooleanFilter( true )}; filter = new Xor( fields[ 0 ], filters[ 0 ], fields[ 1 ], filters[ 1 ] ); assertTrue( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( false ), new BooleanFilter( false )}; filter = new Xor( fields[ 0 ], filters[ 0 ], fields[ 1 ], filters[ 1 ] ); assertFalse( invokeFilter( filter, new Tuple( 1, 2 ) ) ); }
@Test public void testComplexLogicXor() throws Exception { copyFromLocal( inputFileLhs ); Tap source = getPlatform().getDelimitedFile( new Fields( "num", "char" ), " ", inputFileLhs ); Pipe pipe = new Pipe( "test" ); Filter filter = new Not( new Xor( new Fields( "num" ), new RegexFilter( "1", true, true ), new Fields( "char" ), new RegexFilter( "a", true, true ) ) ); // compounding the filter for the Fields.ALL case. pipe = new Each( pipe, filter ); pipe = new Each( pipe, new Fields( "num", "char" ), filter ); Tap sink = getPlatform().getDelimitedFile( Fields.ALL, " ", getOutputPath( "/regression/complexlogicxor" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 3, 2, Pattern.compile( "(1\t.)|(.\ta)" ) ); }
private void performLimitTest( int limit, int tasks, int values ) { Filter filter = new Limit( limit ); int count = 0; for( int i = 0; i < tasks; i++ ) { FlowProcess process = new TestFlowProcess( tasks, i ); filter.prepare( process, operationCall ); operationCall.setArguments( getEntry( new Tuple( 1 ) ) ); for( int j = 0; j < values; j++ ) { if( !filter.isRemove( process, operationCall ) ) count++; } } String message = String.format( "limit:%d tasks:%d values:%d", limit, tasks, values ); assertEquals( message, Math.min( limit, values * tasks ), count ); } }
@Test public void testNull() { Filter filter = new FilterNull(); assertFalse( invokeFilter( filter, new Tuple( 1 ) ) ); assertTrue( invokeFilter( filter, new Tuple( (Comparable) null ) ) ); assertFalse( invokeFilter( filter, new Tuple( "0", 1 ) ) ); assertTrue( invokeFilter( filter, new Tuple( "0", null ) ) ); assertTrue( invokeFilter( filter, new Tuple( null, null ) ) ); }
@Test public void testNotNull() { Filter filter = new FilterNotNull(); assertTrue( invokeFilter( filter, new Tuple( 1 ) ) ); assertFalse( invokeFilter( filter, new Tuple( (Comparable) null ) ) ); assertTrue( invokeFilter( filter, new Tuple( "0", 1 ) ) ); assertTrue( invokeFilter( filter, new Tuple( "0", null ) ) ); assertFalse( invokeFilter( filter, new Tuple( null, null ) ) ); }
private void performSampleTest( double sample, int values ) { Filter filter = new Sample( sample ); int count = 0; filter.prepare( null, operationCall ); operationCall.setArguments( getEntry( new Tuple( 1 ) ) ); for( int j = 0; j < values; j++ ) { if( !filter.isRemove( null, operationCall ) ) count++; } String message = String.format( "sample:%f values:%d", sample, values ); assertEquals( message, sample, (double) count / values, 1 ); } }
@Override public void prepare( FlowProcess flowProcess, OperationCall operationCall ) { Context context = new Context(); context.argumentEntries = getArgumentEntries(); context.calls = new ConcreteCall[ filters.length ]; for( int i = 0; i < filters.length; i++ ) { Filter filter = filters[ i ]; context.calls[ i ] = new ConcreteCall( (ConcreteCall) operationCall ); context.calls[ i ].setArguments( context.argumentEntries[ i ] ); context.calls[ i ].setArgumentFields( context.argumentEntries[ i ].getFields() ); filter.prepare( flowProcess, context.calls[ i ] ); } operationCall.setContext( context ); }
@Test public void testPartialDuplicates() { Filter filter = new Unique.FilterPartialDuplicates( 2 ); Tuple[] tuples = new Tuple[]{new Tuple( 1 ), // false - first time new Tuple( 1 ), // true - second time new Tuple( (Comparable) null ), // false new Tuple( (Comparable) null ), // true - make lots, its a LRU new Tuple( (Comparable) null ), // true new Tuple( (Comparable) null ), // true new Tuple( 1 ), // true - holds two, so still cached new Tuple( 2 ), // false - force least recently seen out new Tuple( 1 )}; // false boolean[] expected = new boolean[]{false, true, false, true, true, true, true, false, false}; boolean[] results = invokeFilter( filter, tuples ); for( int i = 0; i < results.length; i++ ) assertEquals( "failed on: " + i, expected[ i ], results[ i ] ); } }
@ConstructorProperties({"lhsArgumentsSelector", "lhsFilter", "rhsArgumentSelector", "rhsFilter"}) protected Logic( Fields lhsArgumentSelector, Filter lhsFilter, Fields rhsArgumentSelector, Filter rhsFilter ) { this( Fields.fields( lhsArgumentSelector, rhsArgumentSelector ), filters( lhsFilter, rhsFilter ) ); }
@Override public boolean isRemove( FlowProcess flowProcess, FilterCall filterCall ) { if( filter.isRemove( flowProcess, filterCall ) ) throw new StopDataNotificationException( "data stopped on filter: " + getFilterString() ); return false; }
@Test public void testSample() { for( double i = 0; i < 1; i = i + .01 ) performSampleTest( i, 100000 ); }
@Test public void testLimit() { int limit = 20; int tasks = 20; int values = 10; for( int currentLimit = 0; currentLimit < limit; currentLimit++ ) { for( int currentTask = 1; currentTask < tasks; currentTask++ ) { for( int currentValue = 1; currentValue < values; currentValue++ ) { performLimitTest( currentLimit, currentTask, currentValue ); } } } }
@Override public void prepare( FlowProcess flowProcess, OperationCall<Context> operationCall ) { super.prepare( flowProcess, operationCall ); Context context = new Context(); operationCall.setContext( context ); int numTasks = flowProcess.getNumProcessSlices(); int taskNum = flowProcess.getCurrentSliceNum(); context.limit = (long) Math.floor( (double) limit / (double) numTasks ); long remainingLimit = limit % numTasks; // evenly divide limits across tasks context.limit += taskNum < remainingLimit ? 1 : 0; }
@Test public void testOr() { Fields[] fields = new Fields[]{new Fields( 0 ), new Fields( 1 )}; Filter[] filters = new Filter[]{new BooleanFilter( true ), new BooleanFilter( true )}; Filter filter = new Or( fields, filters ); assertTrue( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( true ), new BooleanFilter( false )}; filter = new Or( fields, filters ); assertTrue( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( false ), new BooleanFilter( true )}; filter = new Or( fields, filters ); assertTrue( invokeFilter( filter, new Tuple( 1, 2 ) ) ); filters = new Filter[]{new BooleanFilter( false ), new BooleanFilter( false )}; filter = new Or( fields, filters ); assertFalse( invokeFilter( filter, new Tuple( 1, 2 ) ) ); }
@ConstructorProperties({"filters"}) protected Logic( Filter... filters ) { this.filters = filters; if( filters == null ) throw new IllegalArgumentException( "given filters array must not be null" ); this.argumentSelectors = new Fields[ filters.length ]; Arrays.fill( this.argumentSelectors, Fields.ALL ); verify(); this.numArgs = getFieldsSize(); }