@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for CoGroup may only be preserved on key fields. DualInputSemanticProperties origProps = ((DualInputOperator<?, ?, ?, ?>) getOperator()).getSemanticProperties(); DualInputSemanticProperties filteredProps = new DualInputSemanticProperties(); FieldSet readSet1 = origProps.getReadFields(0); FieldSet readSet2 = origProps.getReadFields(1); if(readSet1 != null) { filteredProps.addReadFields(0, readSet1); } if(readSet2 != null) { filteredProps.addReadFields(1, readSet2); } // preserve only key fields (first input) for(int f : this.keys1) { FieldSet targets = origProps.getForwardingTargetFields(0, f); for(int t : targets) { filteredProps.addForwardedField(0, f, t); } } // preserve only key fields (second input) for(int f : this.keys2) { FieldSet targets = origProps.getForwardingTargetFields(1, f); for(int t : targets) { filteredProps.addForwardedField(1, f, t); } } return filteredProps; }
/** * Adds, to the existing information, a field that is forwarded directly * from the source record(s) in the first input to the destination * record(s). * * @param input the input of the source field * @param sourceField the position in the source record * @param targetField the position in the destination record */ public void addForwardedField(int input, int sourceField, int targetField) { Map<Integer, FieldSet> fieldMapping; if (input != 0 && input != 1) { throw new IndexOutOfBoundsException(); } else if (input == 0) { fieldMapping = this.fieldMapping1; } else { fieldMapping = this.fieldMapping2; } if(isTargetFieldPresent(targetField, fieldMapping)) { throw new InvalidSemanticAnnotationException("Target field "+targetField+" was added twice to input "+input); } FieldSet targetFields = fieldMapping.get(sourceField); if (targetFields != null) { fieldMapping.put(sourceField, targetFields.addField(targetField)); } else { fieldMapping.put(sourceField, new FieldSet(targetField)); } }
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class) public void testAddForwardedFieldsTargetTwice2() { DualInputSemanticProperties sp = new DualInputSemanticProperties(); sp.addForwardedField(1, 0, 2); sp.addForwardedField(1, 1, 2); }
DualInputSemanticProperties sp = new DualInputSemanticProperties(); sp.addForwardedField(0, 0,1); sp.addForwardedField(0, 1,4); sp.addForwardedField(0, 2,3); sp.addForwardedField(0, 3,2); assertEquals(0, sp.getForwardingSourceField(0, 1)); assertEquals(1, sp.getForwardingSourceField(0, 4)); assertEquals(2, sp.getForwardingSourceField(0, 3)); assertEquals(3, sp.getForwardingSourceField(0, 2)); assertTrue(sp.getForwardingSourceField(0, 0) < 0); assertTrue(sp.getForwardingSourceField(0, 5) < 0); sp = new DualInputSemanticProperties(); sp.addForwardedField(0, 0,0); sp.addForwardedField(0, 0,4); sp.addForwardedField(0, 1,1); sp.addForwardedField(0, 1,2); sp.addForwardedField(0, 1,3); assertEquals(0, sp.getForwardingSourceField(0, 0)); assertEquals(0, sp.getForwardingSourceField(0, 4)); assertEquals(1, sp.getForwardingSourceField(0, 1)); assertEquals(1, sp.getForwardingSourceField(0, 2)); assertEquals(1, sp.getForwardingSourceField(0, 3)); assertTrue(sp.getForwardingSourceField(0, 5) < 0); sp = new DualInputSemanticProperties(); sp.addForwardedField(1, 0,1);
@Override @Internal public DualInputSemanticProperties getSemanticProperties() { if (this.udfSemantics == null || analyzedUdfSemantics) { DualInputSemanticProperties props = extractSemanticAnnotationsFromUdf(getFunction().getClass()); if (props != null) { setSemanticProperties(props); } } if (this.udfSemantics == null) { setSemanticProperties(new DualInputSemanticProperties()); } return this.udfSemantics; }
@Test public void testGetReadSet() { // first input DualInputSemanticProperties sp = new DualInputSemanticProperties(); sp.addReadFields(0, new FieldSet(0, 1)); assertEquals(2, sp.getReadFields(0).size()); assertTrue(sp.getReadFields(0).contains(0)); assertTrue(sp.getReadFields(0).contains(1)); sp.addReadFields(0, new FieldSet(3)); assertEquals(3, sp.getReadFields(0).size()); assertTrue(sp.getReadFields(0).contains(0)); assertTrue(sp.getReadFields(0).contains(1)); assertTrue(sp.getReadFields(0).contains(3)); // second input sp = new DualInputSemanticProperties(); sp.addReadFields(1, new FieldSet(0, 1)); assertEquals(2, sp.getReadFields(1).size()); assertTrue(sp.getReadFields(1).contains(0)); assertTrue(sp.getReadFields(1).contains(1)); sp.addReadFields(1, new FieldSet(3)); assertEquals(3, sp.getReadFields(1).size()); assertTrue(sp.getReadFields(1).contains(0)); assertTrue(sp.getReadFields(1).contains(1)); assertTrue(sp.getReadFields(1).contains(3)); }
DualInputSemanticProperties sp = new DualInputSemanticProperties(); sp.addForwardedField(0, 0,1); sp.addForwardedField(0, 1,4); sp.addForwardedField(0, 2,3); sp.addForwardedField(0, 3,2); assertEquals(1, sp.getForwardingTargetFields(0, 0).size()); assertEquals(1, sp.getForwardingTargetFields(0, 1).size()); assertEquals(1, sp.getForwardingTargetFields(0, 2).size()); assertEquals(1, sp.getForwardingTargetFields(0, 3).size()); assertTrue(sp.getForwardingTargetFields(0, 0).contains(1)); assertTrue(sp.getForwardingTargetFields(0, 1).contains(4)); assertTrue(sp.getForwardingTargetFields(0, 2).contains(3)); assertTrue(sp.getForwardingTargetFields(0, 3).contains(2)); assertNotNull(sp.getForwardingTargetFields(0, 4)); assertEquals(0, sp.getForwardingTargetFields(0, 4).size()); sp = new DualInputSemanticProperties(); sp.addForwardedField(0, 0,0); sp.addForwardedField(0, 0,4); sp.addForwardedField(0, 1,1); sp.addForwardedField(0, 1,2); sp.addForwardedField(0, 1,3); assertEquals(2, sp.getForwardingTargetFields(0, 0).size()); assertEquals(3, sp.getForwardingTargetFields(0, 1).size()); assertTrue(sp.getForwardingTargetFields(0, 0).contains(0)); assertTrue(sp.getForwardingTargetFields(0, 0).contains(4)); assertTrue(sp.getForwardingTargetFields(0, 1).contains(1));
@Test public void testForwardedReadDual() { String[] forwardedFieldsFirst = { "f1->f2; f2->f3" }; String[] forwardedFieldsSecond = { "f1->f1; f2->f0" }; String[] readFieldsFirst = {"0;2"}; String[] readFieldsSecond = {"1"}; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, readFieldsFirst, readFieldsSecond, fourIntTupleType, fourIntTupleType, fourIntTupleType); assertTrue(dsp.getForwardingTargetFields(0, 1).contains(2)); assertTrue(dsp.getForwardingTargetFields(0, 2).contains(3)); assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1)); assertTrue(dsp.getForwardingTargetFields(1, 2).contains(0)); assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0); assertTrue(dsp.getForwardingTargetFields(1, 3).size() == 0); assertTrue(dsp.getReadFields(0).size() == 2); assertTrue(dsp.getReadFields(0).contains(0)); assertTrue(dsp.getReadFields(0).contains(2)); assertTrue(dsp.getReadFields(1).size() == 1); assertTrue(dsp.getReadFields(1).contains(1)); }
@Test public void testGetSemanticProperties() { DualInputSemanticProperties origProps = new DualInputSemanticProperties(); origProps.addForwardedField(0, 0, 1); origProps.addForwardedField(0, 2, 2); origProps.addForwardedField(0, 3, 4); origProps.addForwardedField(0, 6, 0); origProps.addReadFields(0, new FieldSet(0, 2, 4, 7)); origProps.addForwardedField(1, 1, 2); origProps.addForwardedField(1, 2, 8); origProps.addForwardedField(1, 3, 7); origProps.addForwardedField(1, 6, 6); origProps.addReadFields(1, new FieldSet(1, 3, 4));
assertEquals(1, props.getForwardingTargetFields(0, 0).size()); assertNotNull(props.getForwardingTargetFields(0, 1)); assertNotNull(props.getForwardingTargetFields(0, 2)); assertNotNull(props.getForwardingTargetFields(0, 3)); assertEquals(1, props.getForwardingTargetFields(0, 4).size()); assertEquals(1, props.getForwardingTargetFields(0, 5).size()); assertNotNull(props.getForwardingTargetFields(0, 6)); assertEquals(0, props.getForwardingTargetFields(0, 1).size()); assertEquals(0, props.getForwardingTargetFields(0, 2).size()); assertEquals(0, props.getForwardingTargetFields(0, 3).size()); assertEquals(0, props.getForwardingTargetFields(0, 6).size()); assertNotNull(props.getForwardingTargetFields(1, 0)); assertEquals(1, props.getForwardingTargetFields(1, 1).size()); assertEquals(1, props.getForwardingTargetFields(1, 2).size()); assertEquals(1, props.getForwardingTargetFields(1, 3).size()); assertNotNull(props.getForwardingTargetFields(1, 4)); assertNotNull(props.getForwardingTargetFields(1, 5)); assertEquals(1, props.getForwardingTargetFields(1, 6).size()); assertEquals(0, props.getForwardingTargetFields(1, 0).size()); assertEquals(0, props.getForwardingTargetFields(1, 4).size()); assertEquals(0, props.getForwardingTargetFields(1, 5).size()); assertTrue(props.getForwardingTargetFields(0, 4).contains(0)); assertTrue(props.getForwardingTargetFields(0, 5).contains(1)); assertTrue(props.getForwardingTargetFields(0, 0).contains(2)); assertTrue(props.getForwardingTargetFields(1, 1).contains(3)); assertTrue(props.getForwardingTargetFields(1, 2).contains(4)); assertTrue(props.getForwardingTargetFields(1, 3).contains(5));
String[] nonForwardedFieldsFirst = { "f1;f2" }; String[] nonForwardedFieldsSecond = { "f0" }; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, nonForwardedFieldsFirst, nonForwardedFieldsSecond, null, null, threeIntTupleType, threeIntTupleType, threeIntTupleType); assertTrue(dsp.getForwardingTargetFields(0, 0).contains(0)); assertTrue(dsp.getForwardingTargetFields(0, 1).size() == 0); assertTrue(dsp.getForwardingTargetFields(0, 2).size() == 0); assertTrue(dsp.getForwardingTargetFields(1, 0).size() == 0); assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1)); assertTrue(dsp.getForwardingTargetFields(1, 2).contains(2)); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, nonForwardedFieldsFirst, null, null, null, threeIntTupleType, fiveIntTupleType, threeIntTupleType); assertTrue(dsp.getForwardingTargetFields(0, 0).contains(0)); assertTrue(dsp.getForwardingTargetFields(0, 1).size() == 0); assertTrue(dsp.getForwardingTargetFields(0, 2).contains(2)); assertTrue(dsp.getForwardingTargetFields(1, 0).size() == 0); assertTrue(dsp.getForwardingTargetFields(1, 1).size() == 0); assertTrue(dsp.getForwardingTargetFields(1, 2).size() == 0); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, nonForwardedFieldsSecond, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType); assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0);
((SingleInputSemanticProperties) sp).addForwardedField(i, i); } else if (sp instanceof DualInputSemanticProperties) { ((DualInputSemanticProperties) sp).addForwardedField(input, i, i); int sourceField = inFFDs.get(i).getPosition(); int targetField = outFFDs.get(i).getPosition(); ((DualInputSemanticProperties) sp).addForwardedField(input, sourceField, targetField); ((SingleInputSemanticProperties) sp).addForwardedField(sourcePos, targetPos); } else if (sp instanceof DualInputSemanticProperties) { ((DualInputSemanticProperties) sp).addForwardedField(input, sourcePos, targetPos);
String[] readFieldsFirst = {"f1;f2"}; String[] readFieldsSecond = {"f0"}; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, null, readFieldsFirst, readFieldsSecond, threeIntTupleType, threeIntTupleType, threeIntTupleType); assertTrue(dsp.getReadFields(0).size() == 2); assertTrue(dsp.getReadFields(0).contains(1)); assertTrue(dsp.getReadFields(0).contains(2)); assertTrue(dsp.getReadFields(1).size() == 1); assertTrue(dsp.getReadFields(1).contains(0)); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, null, readFieldsFirst, readFieldsSecond, nestedTupleType, pojoType, threeIntTupleType); assertTrue(dsp.getReadFields(0).size() == 4); assertTrue(dsp.getReadFields(0).contains(0)); assertTrue(dsp.getReadFields(0).contains(1)); assertTrue(dsp.getReadFields(0).contains(2)); assertTrue(dsp.getReadFields(0).contains(4)); assertTrue(dsp.getReadFields(1).size() == 2); assertTrue(dsp.getReadFields(1).contains(0)); assertTrue(dsp.getReadFields(1).contains(3)); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, null, readFieldsFirst, readFieldsSecond, nestedPojoType, pojoInTupleType, threeIntTupleType);
@Test public void testBinaryForwardedInLine2() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new ReadSetJoin<Long>()) .withForwardedFieldsFirst("0->1; 1->2").withForwardedFieldsSecond("1->0") .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput(); DualInputSemanticProperties semantics = join.getSemanticProperties(); assertNotNull(semantics.getForwardingTargetFields(1, 0)); assertEquals(1, semantics.getForwardingTargetFields(0, 0).size()); assertEquals(1, semantics.getForwardingTargetFields(0, 1).size()); assertEquals(1, semantics.getForwardingTargetFields(1, 1).size()); assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1)); assertTrue(semantics.getForwardingTargetFields(0, 1).contains(2)); assertTrue(semantics.getForwardingTargetFields(1, 1).contains(0)); assertNotNull(semantics.getReadFields(0)); assertNotNull(semantics.getReadFields(1)); assertEquals(1, semantics.getReadFields(0).size()); assertEquals(1, semantics.getReadFields(1).size()); assertTrue(semantics.getReadFields(0).contains(1)); assertTrue(semantics.getReadFields(1).contains(0)); assertEquals(0, semantics.getForwardingTargetFields(1, 0).size()); }
((SingleInputSemanticProperties) sp).addReadFields(readFields); } else if (sp instanceof DualInputSemanticProperties) { ((DualInputSemanticProperties) sp).addReadFields(input, readFields);
@Test public void testBinaryReadFieldsAnnotation() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new ReadSetJoin<Long>()) .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput(); DualInputSemanticProperties semantics = join.getSemanticProperties(); assertNotNull(semantics.getReadFields(0)); assertNotNull(semantics.getReadFields(1)); assertEquals(1, semantics.getReadFields(0).size()); assertEquals(1, semantics.getReadFields(1).size()); assertTrue(semantics.getReadFields(0).contains(1)); assertTrue(semantics.getReadFields(1).contains(0)); }
@Test(expected = InvalidSemanticAnnotationException.class) public void testNonForwardedDualInvalidTypes2() { String[] nonForwardedFieldsSecond = { "f1" }; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, null, nonForwardedFieldsSecond, null, null, threeIntTupleType, pojoInTupleType, threeIntTupleType); }
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class) public void testAddForwardedFieldsTargetTwice1() { DualInputSemanticProperties sp = new DualInputSemanticProperties(); sp.addForwardedField(0, 0, 2); sp.addForwardedField(0, 1, 2); }
assertEquals(1, props.getForwardingTargetFields(0, 0).size()); assertNotNull(props.getForwardingTargetFields(0, 1)); assertNotNull(props.getForwardingTargetFields(0, 2)); assertNotNull(props.getForwardingTargetFields(0, 3)); assertEquals(1, props.getForwardingTargetFields(0, 4).size()); assertEquals(1, props.getForwardingTargetFields(0, 5).size()); assertNotNull(props.getForwardingTargetFields(0, 6)); assertEquals(0, props.getForwardingTargetFields(0, 1).size()); assertEquals(0, props.getForwardingTargetFields(0, 2).size()); assertEquals(0, props.getForwardingTargetFields(0, 3).size()); assertEquals(0, props.getForwardingTargetFields(0, 6).size()); assertNotNull(props.getForwardingTargetFields(1, 0)); assertEquals(1, props.getForwardingTargetFields(1, 1).size()); assertEquals(1, props.getForwardingTargetFields(1, 2).size()); assertEquals(1, props.getForwardingTargetFields(1, 3).size()); assertNotNull(props.getForwardingTargetFields(1, 4)); assertNotNull(props.getForwardingTargetFields(1, 5)); assertEquals(1, props.getForwardingTargetFields(1, 6).size()); assertEquals(0, props.getForwardingTargetFields(1, 0).size()); assertEquals(0, props.getForwardingTargetFields(1, 4).size()); assertEquals(0, props.getForwardingTargetFields(1, 5).size()); assertTrue(props.getForwardingTargetFields(0, 4).contains(0)); assertTrue(props.getForwardingTargetFields(0, 5).contains(1)); assertTrue(props.getForwardingTargetFields(0, 0).contains(2)); assertTrue(props.getForwardingTargetFields(1, 1).contains(3)); assertTrue(props.getForwardingTargetFields(1, 2).contains(4)); assertTrue(props.getForwardingTargetFields(1, 3).contains(5));
String[] forwardedFieldsFirst = { "f1->f2; f2->f3" }; String[] forwardedFieldsSecond = { "f1->f1; f2->f0" }; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, null, null, fourIntTupleType, fourIntTupleType, fourIntTupleType); assertTrue(dsp.getForwardingTargetFields(0, 1).contains(2)); assertTrue(dsp.getForwardingTargetFields(0, 2).contains(3)); assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1)); assertTrue(dsp.getForwardingTargetFields(1, 2).contains(0)); assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0); assertTrue(dsp.getForwardingTargetFields(1, 3).size() == 0); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, null, null, fourIntTupleType, pojoType, pojoInTupleType); assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0); assertTrue(dsp.getForwardingTargetFields(0, 1).contains(0)); assertTrue(dsp.getForwardingTargetFields(0, 2).size() == 0); assertTrue(dsp.getForwardingTargetFields(0, 3).contains(1)); assertTrue(dsp.getForwardingTargetFields(1, 0).contains(2)); assertTrue(dsp.getForwardingTargetFields(1, 1).contains(3)); assertTrue(dsp.getForwardingTargetFields(1, 2).contains(4)); assertTrue(dsp.getForwardingTargetFields(1, 3).contains(5)); dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, null, null, deepNestedTupleType, pojoType, nestedPojoType);