@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for GroupCombine may only be preserved on key fields. SingleInputSemanticProperties origProps = ((SingleInputOperator<?,?,?>) getOperator()).getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } // only add forward field information for key fields if(this.keys != null) { for (int f : this.keys) { FieldSet targets = origProps.getForwardingTargetFields(0, f); for (int t : targets) { filteredProps.addForwardedField(f, t); } } } return filteredProps; }
/** * Adds, to the existing information, a field that is forwarded directly * from the source record(s) to the destination record(s). * * @param sourceField the position in the source record(s) * @param targetField the position in the destination record(s) */ public void addForwardedField(int sourceField, int targetField) { if(isTargetFieldPresent(targetField)) { throw new InvalidSemanticAnnotationException("Target field "+targetField+" was added twice."); } FieldSet targetFields = fieldMapping.get(sourceField); if (targetFields != null) { fieldMapping.put(sourceField, targetFields.addField(targetField)); } else { fieldMapping.put(sourceField, new FieldSet(targetField)); } }
@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for MapPartition may not be preserved. SingleInputSemanticProperties origProps = ((SingleInputOperator<?,?,?>) getOperator()).getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } return filteredProps; }
@Test(expected = IndexOutOfBoundsException.class) public void testGetTargetFieldInvalidIndex() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); sp.addForwardedField(0,0); sp.getForwardingTargetFields(1, 0); }
@Test public void testForwardedMixedOneString() { String[] forwardedFields = {"f2;f3;f0->f4;f4->f0"}; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, forwardedFields, null, null, fiveIntTupleType, fiveIntTupleType); assertTrue(sp.getForwardingTargetFields(0, 2).contains(2)); assertTrue(sp.getForwardingTargetFields(0, 3).contains(3)); assertTrue(sp.getForwardingTargetFields(0, 0).contains(4)); assertTrue(sp.getForwardingTargetFields(0, 4).contains(0)); }
@Test(expected = InvalidSemanticAnnotationException.class) public void testAddForwardedFieldsTargetTwice() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); sp.addForwardedField(0, 2); sp.addForwardedField(1, 2); }
@Test public void testGetSemanticProperties() { SingleInputSemanticProperties origProps = new SingleInputSemanticProperties(); origProps.addForwardedField(0, 1); origProps.addForwardedField(2, 2); origProps.addReadFields(new FieldSet(0, 2, 4, 7)); MapPartitionOperatorBase<?,?,?> op = mock(MapPartitionOperatorBase.class); when(op.getSemanticProperties()).thenReturn(origProps); when(op.getKeyColumns(0)).thenReturn(new int[]{}); MapPartitionNode node = new MapPartitionNode(op); SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering(); assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0); assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 0); assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0); assertTrue(filteredProps.getForwardingSourceField(0, 2) < 0); assertTrue(filteredProps.getReadFields(0).size() == 4); assertTrue(filteredProps.getReadFields(0).contains(0)); assertTrue(filteredProps.getReadFields(0).contains(2)); assertTrue(filteredProps.getReadFields(0).contains(4)); assertTrue(filteredProps.getReadFields(0).contains(7)); }
@Test public void testReadFieldsBasic() { String[] readFields = { "*" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, intType, intType); FieldSet fs = sp.getReadFields(0); assertTrue(fs.size() == 1); assertTrue(fs.contains(0)); sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, intType, fiveIntTupleType); fs = sp.getReadFields(0); assertTrue(fs.size() == 1); assertTrue(fs.contains(0)); }
@Test(expected = InvalidSemanticAnnotationException.class) public void testForwardedSameTargetTwice() { String[] forwardedFields = {"f0->f2; f1->f2"}; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, forwardedFields, null, null, fiveIntTupleType, fiveIntTupleType); }
@Test public void testForwardedRead() { String[] forwardedFields = { "f0->f0;f1->f2" }; String[] readFields = {"f0; f2"}; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, forwardedFields, null, readFields, threeIntTupleType, fiveIntTupleType); assertTrue(sp.getForwardingTargetFields(0, 0).contains(0)); assertTrue(sp.getForwardingTargetFields(0, 1).contains(2)); assertTrue(sp.getReadFields(0).size() == 2); assertTrue(sp.getReadFields(0).contains(0)); assertTrue(sp.getReadFields(0).contains(2)); }
@Test(expected = IndexOutOfBoundsException.class) public void testGetSourceFieldInvalidIndex() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); sp.addForwardedField(0,0); sp.getForwardingSourceField(1, 0); }
@Test(expected = IndexOutOfBoundsException.class) public void testAllForwardedSingleInputSemPropsInvalidIndex2() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties.AllFieldsForwardedProperties(); sp.getForwardingTargetFields(1, 0); } }
((SingleInputSemanticProperties) sp).addForwardedField(i, i); } else if (sp instanceof DualInputSemanticProperties) { ((DualInputSemanticProperties) sp).addForwardedField(input, i, i); int sourceField = inFFDs.get(i).getPosition(); int targetField = outFFDs.get(i).getPosition(); ((SingleInputSemanticProperties) sp).addForwardedField(sourceField, targetField); int targetPos = outFFDs.get(i).getPosition(); if (sp instanceof SingleInputSemanticProperties) { ((SingleInputSemanticProperties) sp).addForwardedField(sourcePos, targetPos); } else if (sp instanceof DualInputSemanticProperties) { ((DualInputSemanticProperties) sp).addForwardedField(input, sourcePos, targetPos);
((SingleInputSemanticProperties) sp).addReadFields(readFields); } else if (sp instanceof DualInputSemanticProperties) { ((DualInputSemanticProperties) sp).addReadFields(input, readFields);
@Test public void testAllForwardedSingleInputSemProps() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties.AllFieldsForwardedProperties(); assertEquals(1, sp.getForwardingTargetFields(0, 0).size()); assertEquals(1, sp.getForwardingTargetFields(0, 1).size()); assertEquals(1, sp.getForwardingTargetFields(0, 123).size()); assertTrue(sp.getForwardingTargetFields(0, 0).contains(0)); assertTrue(sp.getForwardingTargetFields(0, 1).contains(1)); assertTrue(sp.getForwardingTargetFields(0, 123).contains(123)); assertEquals(0, sp.getForwardingSourceField(0, 0)); assertEquals(2, sp.getForwardingSourceField(0, 2)); assertEquals(123, sp.getForwardingSourceField(0, 123)); }
@Test public void testUnaryFunctionReadFieldsAnnotation() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L)); input.map(new ReadSetMapper<Tuple3<Long, Long, Long>>()).output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput(); SingleInputSemanticProperties semantics = mapper.getSemanticProperties(); FieldSet read = semantics.getReadFields(0); assertNotNull(read); assertEquals(2, read.size()); assertTrue(read.contains(0)); assertTrue(read.contains(2)); }
@Test public void testNonForwardedNone() { String[] nonForwardedFields = { "" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, nonForwardedFields, null, threeIntTupleType, threeIntTupleType); assertTrue(sp.getForwardingTargetFields(0, 0).contains(0)); assertTrue(sp.getForwardingTargetFields(0, 1).contains(1)); assertTrue(sp.getForwardingTargetFields(0, 2).contains(2)); }
@Test(expected = IndexOutOfBoundsException.class) public void testGetReadFieldsInvalidIndex() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); sp.addReadFields(new FieldSet(0, 1)); sp.getReadFields(1); }
public static SingleInputSemanticProperties createProjectionPropertiesSingle(int[] fields, CompositeType<?> inType) { Character.isJavaIdentifierStart(1); SingleInputSemanticProperties ssp = new SingleInputSemanticProperties(); int[] sourceOffsets = new int[inType.getArity()]; sourceOffsets[0] = 0; for (int i = 1; i < inType.getArity(); i++) { sourceOffsets[i] = inType.getTypeAt(i - 1).getTotalFields() + sourceOffsets[i - 1]; } int targetOffset = 0; for (int i = 0; i < fields.length; i++) { int sourceOffset = sourceOffsets[fields[i]]; int numFieldsToCopy = inType.getTypeAt(fields[i]).getTotalFields(); for (int j = 0; j < numFieldsToCopy; j++) { ssp.addForwardedField(sourceOffset + j, targetOffset + j); } targetOffset += numFieldsToCopy; } return ssp; }
@Test public void testGetSemanticProperties() { SingleInputSemanticProperties origProps = new SingleInputSemanticProperties(); origProps.addForwardedField(0, 1); origProps.addForwardedField(2, 2); origProps.addForwardedField(3, 4); origProps.addForwardedField(6, 0); origProps.addReadFields(new FieldSet(0, 2, 4, 7));