@Test(expected = InvalidSemanticAnnotationException.class) public void testNonForwardedDualInvalidTypes1() { String[] nonForwardedFieldsFirst = { "f1" }; DualInputSemanticProperties dsp = new DualInputSemanticProperties(); SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, nonForwardedFieldsFirst, null, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType); }
@Test(expected = InvalidSemanticAnnotationException.class) public void testNonForwardedInvalidTypes3() { String[] nonForwardedFields = { "f1; f2" }; SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, nonForwardedFields, null, threeIntTupleType, fiveIntTupleType); }
@Override @Internal public SingleInputSemanticProperties getSemanticProperties() { SingleInputSemanticProperties props = super.getSemanticProperties(); // offset semantic information by extracted key fields if (props != null && this.grouper != null && this.grouper.keys instanceof SelectorFunctionKeys) { int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields(); if (this.grouper instanceof SortedGrouping) { offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields(); } props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset); } return props; }
parseForwardedFields(result, forwardedFirst, inType1, outType, 0, skipIncompatibleTypes); } else if (hasNonForwardedFirstAnnotation) { parseNonForwardedFields(result, nonForwardedFirst, inType1, outType, 0, skipIncompatibleTypes); parseForwardedFields(result, forwardedSecond, inType2, outType, 1, skipIncompatibleTypes); } else if (hasNonForwardedSecondAnnotation) { parseNonForwardedFields(result, nonForwardedSecond, inType2, outType, 1, skipIncompatibleTypes); parseReadFields(result, readFieldsFirst, inType1, 0); parseReadFields(result, readFieldsSecond, inType2, 1);
ff2Array = new String[]{ff2}; SemanticPropUtil.getSemanticPropsDualFromString((DualInputSemanticProperties) sp, ff1Array, ff2Array, null, null, null, null, in1Type, in2Type, outType, true); ffArray = new String[]{ff}; SemanticPropUtil.getSemanticPropsSingleFromString((SingleInputSemanticProperties) sp, ffArray, null, null, in1Type, outType, true);
if (!areFieldsCompatible(sourceStr, inType, targetStr, outType, !skipIncompatibleTypes)) { if (skipIncompatibleTypes) { continue; List<FlatFieldDescriptor> inFFDs = getFlatFields(sourceStr, inType); List<FlatFieldDescriptor> outFFDs = getFlatFields(targetStr, outType); if (sp instanceof SingleInputSemanticProperties) { for (int i = 0; i < inFFDs.size(); i++) { try { if (!areFieldsCompatible(fieldStr, inType, fieldStr, outType, !skipIncompatibleTypes)) { if (skipIncompatibleTypes) { continue; List<FlatFieldDescriptor> inFFDs = getFlatFields(fieldStr, inType); List<FlatFieldDescriptor> outFFDs = getFlatFields(fieldStr, outType); for (int i = 0; i < inFFDs.size(); i++) { int sourcePos = inFFDs.get(i).getPosition();
@Override protected DualInputSemanticProperties extractSemanticAnnotationsFromUdf(Class<?> udfClass) { // we do not extract the annotation, we construct the properties from the projection# return SemanticPropUtil.createProjectionPropertiesDual(getFunction().getFields(), getFunction().getIsFromFirst(), getInput1Type(), getInput2Type()); }
@Override public DualInputSemanticProperties getSemanticProperties() { DualInputSemanticProperties props = super.getSemanticProperties(); // offset semantic information by extracted key fields if (props != null && (this.keys1 instanceof SelectorFunctionKeys || this.keys2 instanceof SelectorFunctionKeys)) { int numFields1 = this.getInput1Type().getTotalFields(); int numFields2 = this.getInput2Type().getTotalFields(); int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ? ((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0; int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ? ((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0; props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2); } return props; }
@Override protected org.apache.flink.api.common.operators.base.MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "Projection " + Arrays.toString(fields); // create operator PlanProjectOperator<IN, OUT> ppo = new PlanProjectOperator<IN, OUT>(fields, name, getInputType(), getResultType(), context.getConfig()); // set input ppo.setInput(input); // set parallelism ppo.setParallelism(this.getParallelism()); ppo.setSemanticProperties(SemanticPropUtil.createProjectionPropertiesSingle(fields, (CompositeType<?>) getInputType())); return ppo; }
protected DualInputSemanticProperties extractSemanticAnnotationsFromUdf(Class<?> udfClass) { Set<Annotation> annotations = FunctionAnnotation.readDualForwardAnnotations(udfClass); return SemanticPropUtil.getSemanticPropsDual(annotations, getInput1Type(), getInput2Type(), getResultType()); }
protected SingleInputSemanticProperties extractSemanticAnnotations(Class<?> udfClass) { Set<Annotation> annotations = FunctionAnnotation.readSingleForwardAnnotations(udfClass); return SemanticPropUtil.getSemanticPropsSingle(annotations, getInputType(), getResultType()); }
private static boolean areFieldsCompatible(String sourceField, TypeInformation<?> inType, String targetField, TypeInformation<?> outType, boolean throwException) { try { // get source type information TypeInformation<?> sourceType = getExpressionTypeInformation(sourceField, inType); // get target type information TypeInformation<?> targetType = getExpressionTypeInformation(targetField, outType); return sourceType.equals(targetType); } catch (InvalidFieldReferenceException e) { if (throwException) { throw e; } else { return false; } } }
parseForwardedFields(result, forwardedFirst, inType1, outType, 0, skipIncompatibleTypes); } else if (hasNonForwardedFirstAnnotation) { parseNonForwardedFields(result, nonForwardedFirst, inType1, outType, 0, skipIncompatibleTypes); parseForwardedFields(result, forwardedSecond, inType2, outType, 1, skipIncompatibleTypes); } else if (hasNonForwardedSecondAnnotation) { parseNonForwardedFields(result, nonForwardedSecond, inType2, outType, 1, skipIncompatibleTypes); parseReadFields(result, readFieldsFirst, inType1, 0); parseReadFields(result, readFieldsSecond, inType2, 1);
ff2Array = new String[]{ff2}; SemanticPropUtil.getSemanticPropsDualFromString((DualInputSemanticProperties) sp, ff1Array, ff2Array, null, null, null, null, in1Type, in2Type, outType, true); ffArray = new String[]{ff}; SemanticPropUtil.getSemanticPropsSingleFromString((SingleInputSemanticProperties) sp, ffArray, null, null, in1Type, outType, true);
@Override protected DualInputSemanticProperties extractSemanticAnnotationsFromUdf(Class<?> udfClass) { // we do not extract anything, but construct the properties from the projection return SemanticPropUtil.createProjectionPropertiesDual(getFunction().getFields(), getFunction().getIsFromFirst(), getInput1Type(), getInput2Type()); } }
@Override @Internal public DualInputSemanticProperties getSemanticProperties() { DualInputSemanticProperties props = super.getSemanticProperties(); // offset semantic information by extracted key fields if (props != null && (this.keys1 instanceof SelectorFunctionKeys || this.keys2 instanceof SelectorFunctionKeys)) { int numFields1 = this.getInput1Type().getTotalFields(); int numFields2 = this.getInput2Type().getTotalFields(); int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ? ((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0; int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ? ((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0; props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2); } return props; }
if (!areFieldsCompatible(sourceStr, inType, targetStr, outType, !skipIncompatibleTypes)) { if (skipIncompatibleTypes) { continue; List<FlatFieldDescriptor> inFFDs = getFlatFields(sourceStr, inType); List<FlatFieldDescriptor> outFFDs = getFlatFields(targetStr, outType); if (sp instanceof SingleInputSemanticProperties) { for (int i = 0; i < inFFDs.size(); i++) { try { if (!areFieldsCompatible(fieldStr, inType, fieldStr, outType, !skipIncompatibleTypes)) { if (skipIncompatibleTypes) { continue; List<FlatFieldDescriptor> inFFDs = getFlatFields(fieldStr, inType); List<FlatFieldDescriptor> outFFDs = getFlatFields(fieldStr, outType); for (int i = 0; i < inFFDs.size(); i++) { int sourcePos = inFFDs.get(i).getPosition();
SingleInputSemanticProperties sp = SemanticPropUtil.createProjectionPropertiesSingle(pMap, (CompositeType<?>) fiveIntTupleType); assertTrue(sp.getForwardingTargetFields(0, 0).contains(1)); assertTrue(sp.getForwardingTargetFields(0, 3).contains(0)); sp = SemanticPropUtil.createProjectionPropertiesSingle(pMap, (CompositeType<?>) fiveIntTupleType); assertTrue(sp.getForwardingTargetFields(0, 1).size() == 2); assertTrue(sp.getForwardingTargetFields(0, 1).contains(2)); sp = SemanticPropUtil.createProjectionPropertiesSingle(pMap, (CompositeType<?>) nestedTupleType); assertTrue(sp.getForwardingTargetFields(0, 4).contains(0)); assertTrue(sp.getForwardingTargetFields(0, 0).contains(1)); sp = SemanticPropUtil.createProjectionPropertiesSingle(pMap, (CompositeType<?>) deepNestedTupleType); assertTrue(sp.getForwardingTargetFields(0, 6).contains(0)); assertTrue(sp.getForwardingTargetFields(0, 0).contains(1)); sp = SemanticPropUtil.createProjectionPropertiesSingle(pMap, (CompositeType<?>) pojoInTupleType); assertTrue(sp.getForwardingTargetFields(0, 2).contains(0)); assertTrue(sp.getForwardingTargetFields(0, 3).contains(1));
@SuppressWarnings({ "rawtypes", "unchecked" }) public static void compareAnalyzerResultWithAnnotationsDualInputWithKeys(Class<?> baseClass, Class<?> clazz, TypeInformation<?> in1Type, TypeInformation<?> in2Type, TypeInformation<?> outType, String[] keys1, String[] keys2) { // expected final Set<Annotation> annotations = FunctionAnnotation.readDualForwardAnnotations(clazz); final DualInputSemanticProperties expected = SemanticPropUtil.getSemanticPropsDual(annotations, in1Type, in2Type, outType); // actual final UdfAnalyzer ua = new UdfAnalyzer(baseClass, clazz, "operator", in1Type, in2Type, outType, (keys1 == null) ? null : new Keys.ExpressionKeys(keys1, in1Type), (keys2 == null) ? null : new Keys.ExpressionKeys( keys2, in2Type), true); ua.analyze(); final DualInputSemanticProperties actual = (DualInputSemanticProperties) ua.getSemanticProperties(); assertEquals(expected.toString(), actual.toString()); }
@SuppressWarnings({ "rawtypes", "unchecked" }) public static void compareAnalyzerResultWithAnnotationsSingleInputWithKeys(Class<?> baseClass, Class<?> clazz, TypeInformation<?> inType, TypeInformation<?> outType, String[] keys) { // expected final Set<Annotation> annotations = FunctionAnnotation.readSingleForwardAnnotations(clazz); SingleInputSemanticProperties expected = SemanticPropUtil.getSemanticPropsSingle(annotations, inType, outType); if (expected == null) { expected = new SingleInputSemanticProperties(); } // actual final UdfAnalyzer ua = new UdfAnalyzer(baseClass, clazz, "operator", inType, null, outType, (keys == null) ? null : new Keys.ExpressionKeys(keys, inType), null, true); ua.analyze(); final SingleInputSemanticProperties actual = (SingleInputSemanticProperties) ua.getSemanticProperties(); assertEquals(expected.toString(), actual.toString()); }