private boolean isVisible(final Row e, final String visibility, final Authorisations auths) throws VisibilityParseException { if (e.getAs(visibility) != null) { final VisibilityEvaluator visibilityEvaluator = new VisibilityEvaluator(auths); final ElementVisibility elementVisibility = new ElementVisibility((String) e.getAs(visibility)); return visibilityEvaluator.evaluate(elementVisibility); } else { return true; } } }
@Override public Row getPrecedingTime(Row row) { Date date; try { date = format.parse(row.<String>getAs(field.name())); } catch (ParseException e) { throw new RuntimeException(e); } String precedingDateString = format.format(new Date(date.getTime() - 1)); return new RowWithSchema(getSchema(), precedingDateString); }
@Override public LabeledPoint call(Row arg0) throws Exception { return new LabeledPoint(arg0.getDouble(1),arg0.getAs(0)); } }
@Override public Row getPrecedingTime(Row row) { Date date; try { date = format.parse(row.<String>getAs(field.name())); } catch (ParseException e) { throw new RuntimeException(e); } String precedingDateString = format.format(precedingDate(date)); return new RowWithSchema(getSchema(), precedingDateString); }
Seq<Integer> result = first.getAs(1); Assert.assertEquals(bean.getB().length, result.length()); for (int i = 0; i < result.length(); i++) { bean.getC().get("hello"), Ints.toArray(JavaConverters.seqAsJavaListConverter(outputBuffer).asJava())); Seq<String> d = first.getAs(3); Assert.assertEquals(bean.getD().size(), d.length()); for (int i = 0; i < d.length(); i++) {
@Override public Row getPrecedingTime(Row row) { return new RowWithSchema(getSchema(), row.<Long>getAs(field.name()) - 1); }
Seq<Integer> result = first.getAs(1); Assert.assertEquals(bean.getB().length, result.length()); for (int i = 0; i < result.length(); i++) { bean.getC().get("hello"), Ints.toArray(JavaConverters.seqAsJavaListConverter(outputBuffer).asJava())); Seq<String> d = first.getAs(3); Assert.assertEquals(bean.getD().size(), d.length()); for (int i = 0; i < d.length(); i++) {
Seq<Integer> result = first.getAs(1); Assert.assertEquals(bean.getB().length, result.length()); for (int i = 0; i < result.length(); i++) { bean.getC().get("hello"), Ints.toArray(JavaConverters.seqAsJavaListConverter(outputBuffer).asJava())); Seq<String> d = first.getAs(3); Assert.assertEquals(bean.getD().size(), d.length()); for (int i = 0; i < d.length(); i++) {
@Override public Row getPrecedingTime(Row row) { Timestamp time = row.<Timestamp>getAs(field.name()); return new RowWithSchema(getSchema(), getPrecedingTimestamp(time)); }
@Test public void testCopyTime() { Row row = new RowWithSchema(schemaWithTMs, 1000L, null); Row copied = PlannerUtils.copyTime(row, firstTM, row, secondTM); assertEquals(copied.<Long>getAs("second"), (Long)1000L); }
@Test public void testSummarizer() { dataset.select(col("features")); Row result = dataset .select(Summarizer.metrics("mean", "max", "count").summary(col("features"))) .first().getStruct(0); Vector meanVec = result.getAs("mean"); Vector maxVec = result.getAs("max"); long count = result.getAs("count"); assertEquals(2L, count); assertArrayEquals(new double[]{2.0, 3.0}, meanVec.toArray(), 0.0); assertArrayEquals(new double[]{3.0, 4.0}, maxVec.toArray(), 0.0); } }
@Test public void verifyLibSVMDF() { Dataset<Row> dataset = spark.read().format("libsvm").option("vectorType", "dense") .load(path); Assert.assertEquals("label", dataset.columns()[0]); Assert.assertEquals("features", dataset.columns()[1]); Row r = dataset.first(); Assert.assertEquals(1.0, r.getDouble(0), 1e-15); DenseVector v = r.getAs(1); Assert.assertEquals(Vectors.dense(1.0, 0.0, 2.0, 0.0, 3.0, 0.0), v); } }
@Test public void verifyLibSVMDF() { Dataset<Row> dataset = spark.read().format("libsvm").option("vectorType", "dense") .load(path); Assert.assertEquals("label", dataset.columns()[0]); Assert.assertEquals("features", dataset.columns()[1]); Row r = dataset.first(); Assert.assertEquals(1.0, r.getDouble(0), 1e-15); DenseVector v = r.getAs(1); Assert.assertEquals(Vectors.dense(1.0, 0.0, 2.0, 0.0, 3.0, 0.0), v); } }
@Test public void verifyLibSVMDF() { Dataset<Row> dataset = spark.read().format("libsvm").option("vectorType", "dense") .load(path); Assert.assertEquals("label", dataset.columns()[0]); Assert.assertEquals("features", dataset.columns()[1]); Row r = dataset.first(); Assert.assertEquals(1.0, r.getDouble(0), 1e-15); DenseVector v = r.getAs(1); Assert.assertEquals(Vectors.dense(1.0, 0.0, 2.0, 0.0, 3.0, 0.0), v); } }