@Override public Dataset<T> where(final Column condition) { final boolean userTriggered = initializeFunction(condition); final Dataset<T> result = from(super.where(condition)); this.setIsUserTriggered(userTriggered); return result; }
@Override public Dataset<T> where(final String conditionExpr) { final boolean userTriggered = initializeFunction(conditionExpr); final Dataset<T> result = from(super.where(conditionExpr)); this.setIsUserTriggered(userTriggered); return result; }
Dataset<Row> decisionDataset = keyDataset.where(whereClause);
/** * Returns the values for the given URI and version. * * @param uri the uri of the value set for which we get values * @param version the version of the value set for which we get values * @return a dataset of values for the given URI and version. */ public Dataset<Value> getValues(String uri, String version) { return this.values.where(col("valueseturi").equalTo(lit(uri)) .and(col("valuesetversion").equalTo(lit(version)))); }
/** * Returns the values for the given URI and version. * * @param uri the uri of the value set for which we get values * @param version the version of the value set for which we get values * @return a dataset of values for the given URI and version. */ public Dataset<Value> getValues(String uri, String version) { return this.values.where(col("valueseturi").equalTo(lit(uri)) .and(col("valuesetversion").equalTo(lit(version)))); }
/** * Returns the mappings for the given URI and version. * * @param uri the uri of the concept map for which we get mappings * @param version the version of the concept map for which we get mappings * @return a dataset of mappings for the given URI and version. */ public Dataset<Mapping> getMappings(String uri, String version) { return this.mappings.where(functions.col("conceptmapuri").equalTo(lit(uri)) .and(functions.col("conceptmapversion").equalTo(lit(version)))); }
/** * Returns the mappings for the given URI and version. * * @param uri the uri of the concept map for which we get mappings * @param version the version of the concept map for which we get mappings * @return a dataset of mappings for the given URI and version. */ public Dataset<Mapping> getMappings(String uri, String version) { return this.mappings.where(functions.col("conceptmapuri").equalTo(lit(uri)) .and(functions.col("conceptmapversion").equalTo(lit(version)))); }
ds.where(col("i").gt(2)).select(col("english"), col("french")).show();
/** * Reads a Snomed relationship file and converts it to a {@link HierarchicalElement} dataset. * * @param spark the Spark session * @param snomedRelationshipPath path to the SNOMED relationship file * @return a dataset of{@link HierarchicalElement} representing the hierarchical relationship. */ public static Dataset<HierarchicalElement> readRelationshipFile(SparkSession spark, String snomedRelationshipPath) { return spark.read() .option("header", true) .option("delimiter", "\t") .csv(snomedRelationshipPath) .where(col("typeId").equalTo(lit(SNOMED_ISA_RELATIONSHIP_ID))) .where(col("active").equalTo(lit("1"))) .select(col("destinationId"), col("sourceId")) .where(col("destinationId").isNotNull() .and(col("destinationId").notEqual(lit("")))) .where(col("sourceId").isNotNull() .and(col("sourceId").notEqual(lit("")))) .map((MapFunction<Row, HierarchicalElement>) row -> { HierarchicalElement element = new HierarchicalElement(); element.setAncestorSystem(SNOMED_CODE_SYSTEM_URI); element.setAncestorValue(row.getString(0)); element.setDescendantSystem(SNOMED_CODE_SYSTEM_URI); element.setDescendantValue(row.getString(1)); return element; }, Hierarchies.getHierarchicalElementEncoder()); }
tupleDS.where(col("_1").gt(2)).select(col("_2"), col("_3")).show();
/** * Reads a Snomed relationship file and converts it to a {@link HierarchicalElement} dataset. * * @param spark the Spark session * @param snomedRelationshipPath path to the SNOMED relationship file * @return a dataset of{@link HierarchicalElement} representing the hierarchical relationship. */ public static Dataset<HierarchicalElement> readRelationshipFile(SparkSession spark, String snomedRelationshipPath) { return spark.read() .option("header", true) .option("delimiter", "\t") .csv(snomedRelationshipPath) .where(col("typeId").equalTo(lit(SNOMED_ISA_RELATIONSHIP_ID))) .where(col("active").equalTo(lit("1"))) .select(col("destinationId"), col("sourceId")) .where(col("destinationId").isNotNull() .and(col("destinationId").notEqual(lit("")))) .where(col("sourceId").isNotNull() .and(col("sourceId").notEqual(lit("")))) .map((MapFunction<Row, HierarchicalElement>) row -> { HierarchicalElement element = new HierarchicalElement(); element.setAncestorSystem(SNOMED_CODE_SYSTEM_URI); element.setAncestorValue(row.getString(0)); element.setDescendantSystem(SNOMED_CODE_SYSTEM_URI); element.setDescendantValue(row.getString(1)); return element; }, Hierarchies.getHierarchicalElementEncoder()); }
double bestThreshold = fMeasure.where(fMeasure.col("F-Measure").equalTo(maxFMeasure)) .select("threshold").head().getDouble(0); lrModel.setThreshold(bestThreshold);
/** * Reads the LOINC mutliaxial hierarchy file and converts it to a {@link HierarchicalElement} * dataset. * * @param spark the Spark session * @param loincHierarchyPath path to the multiaxial hierarchy CSV * @return a dataset of {@link HierarchicalElement} representing the hierarchical relationship. */ public static Dataset<HierarchicalElement> readMultiaxialHierarchyFile(SparkSession spark, String loincHierarchyPath) { return spark.read() .option("header", true) .csv(loincHierarchyPath) .select(col("IMMEDIATE_PARENT"), col("CODE")) .where(col("IMMEDIATE_PARENT").isNotNull() .and(col("IMMEDIATE_PARENT").notEqual(lit("")))) .where(col("CODE").isNotNull() .and(col("CODE").notEqual(lit("")))) .map((MapFunction<Row, HierarchicalElement>) row -> { HierarchicalElement element = new HierarchicalElement(); element.setAncestorSystem(LOINC_CODE_SYSTEM_URI); element.setAncestorValue(row.getString(0)); element.setDescendantSystem(LOINC_CODE_SYSTEM_URI); element.setDescendantValue(row.getString(1)); return element; }, Hierarchies.getHierarchicalElementEncoder()); }
/** * Reads the LOINC mutliaxial hierarchy file and converts it to a {@link HierarchicalElement} * dataset. * * @param spark the Spark session * @param loincHierarchyPath path to the multiaxial hierarchy CSV * @return a dataset of {@link HierarchicalElement} representing the hierarchical relationship. */ public static Dataset<HierarchicalElement> readMultiaxialHierarchyFile(SparkSession spark, String loincHierarchyPath) { return spark.read() .option("header", true) .csv(loincHierarchyPath) .select(col("IMMEDIATE_PARENT"), col("CODE")) .where(col("IMMEDIATE_PARENT").isNotNull() .and(col("IMMEDIATE_PARENT").notEqual(lit("")))) .where(col("CODE").isNotNull() .and(col("CODE").notEqual(lit("")))) .map((MapFunction<Row, HierarchicalElement>) row -> { HierarchicalElement element = new HierarchicalElement(); element.setAncestorSystem(LOINC_CODE_SYSTEM_URI); element.setAncestorValue(row.getString(0)); element.setDescendantSystem(LOINC_CODE_SYSTEM_URI); element.setDescendantValue(row.getString(1)); return element; }, Hierarchies.getHierarchicalElementEncoder()); }