@Test public void isInCollectionCheckExceptionMessage() { List<Row> rows = Arrays.asList( RowFactory.create(1, Arrays.asList(1)), RowFactory.create(2, Arrays.asList(2)), RowFactory.create(3, Arrays.asList(3))); StructType schema = createStructType(Arrays.asList( createStructField("a", IntegerType, false), createStructField("b", createArrayType(IntegerType, false), false))); Dataset<Row> df = spark.createDataFrame(rows, schema); try { df.filter(df.col("a").isInCollection(Arrays.asList(new Column("b")))); Assert.fail("Expected org.apache.spark.sql.AnalysisException"); } catch (Exception e) { Arrays.asList("cannot resolve", "due to data type mismatch: Arguments must be same type but were") .forEach(s -> Assert.assertTrue( e.getMessage().toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))); } } }
@Test public void isInCollectionCheckExceptionMessage() { List<Row> rows = Arrays.asList( RowFactory.create(1, Arrays.asList(1)), RowFactory.create(2, Arrays.asList(2)), RowFactory.create(3, Arrays.asList(3))); StructType schema = createStructType(Arrays.asList( createStructField("a", IntegerType, false), createStructField("b", createArrayType(IntegerType, false), false))); Dataset<Row> df = spark.createDataFrame(rows, schema); try { df.filter(df.col("a").isInCollection(Arrays.asList(new Column("b")))); Assert.fail("Expected org.apache.spark.sql.AnalysisException"); } catch (Exception e) { Arrays.asList("cannot resolve", "due to data type mismatch: Arguments must be same type but were") .forEach(s -> Assert.assertTrue( e.getMessage().toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))); } } }
public static Column[] toColumnArray(List<String> columnList) { Column[] columnArray = new Column[columnList.size()]; for (int i = 0; i < columnList.size(); i++) { columnArray[i] = new Column(columnList.get(i)); } return columnArray; }