/** * Returns the values for the given URI and version. * * @param uri the uri of the value set for which we get values * @param version the version of the value set for which we get values * @return a dataset of values for the given URI and version. */ public Dataset<Value> getValues(String uri, String version) { return this.values.where(col("valueseturi").equalTo(lit(uri)) .and(col("valuesetversion").equalTo(lit(version)))); }
/** * Returns the mappings for the given URI and version. * * @param uri the uri of the concept map for which we get mappings * @param version the version of the concept map for which we get mappings * @return a dataset of mappings for the given URI and version. */ public Dataset<Mapping> getMappings(String uri, String version) { return this.mappings.where(functions.col("conceptmapuri").equalTo(lit(uri)) .and(functions.col("conceptmapversion").equalTo(lit(version)))); }
/** * Returns the values for the given URI and version. * * @param uri the uri of the value set for which we get values * @param version the version of the value set for which we get values * @return a dataset of values for the given URI and version. */ public Dataset<Value> getValues(String uri, String version) { return this.values.where(col("valueseturi").equalTo(lit(uri)) .and(col("valuesetversion").equalTo(lit(version)))); }
/** * Returns the mappings for the given URI and version. * * @param uri the uri of the concept map for which we get mappings * @param version the version of the concept map for which we get mappings * @return a dataset of mappings for the given URI and version. */ public Dataset<Mapping> getMappings(String uri, String version) { return this.mappings.where(functions.col("conceptmapuri").equalTo(lit(uri)) .and(functions.col("conceptmapversion").equalTo(lit(version)))); }
/** * Returns the value set with the given uri and version, or null if there is no such value set. * * @param uri the uri of the value set to return * @param version the version of the value set to return * @return the specified value set. */ public T getValueSet(String uri, String version) { // Load the value sets, which may contain zero items if the value set does not exist // Typecast necessary to placate the Java compiler calling this Scala function T[] valueSets = (T[]) this.valueSets.filter( col("url").equalTo(lit(uri)) .and(col("version").equalTo(lit(version)))) .head(1); if (valueSets.length == 0) { return null; } else { T valueSet = valueSets[0]; Dataset<Value> filteredValues = getValues(uri, version); addToValueSet(valueSet, filteredValues); return valueSet; } }
/** * Returns the concept map with the given uri and version, or null if there is no such map. * * @param uri the uri of the map to return * @param version the version of the map to return * @return the specified concept map. */ public T getConceptMap(String uri, String version) { // Load the concept maps, which may contain zero items // if the map does not exist. // Typecast necessary to placate the Java compiler calling this Scala function. T[] maps = (T[]) this.conceptMaps.filter( functions.col("url").equalTo(lit(uri)) .and(functions.col("version").equalTo(lit(version)))) .head(1); if (maps.length == 0) { return null; } else { T map = maps[0]; Dataset<Mapping> filteredMappings = getMappings(uri, version); addToConceptMap(map, filteredMappings); return map; } }
/** * Returns the value set with the given uri and version, or null if there is no such value set. * * @param uri the uri of the value set to return * @param version the version of the value set to return * @return the specified value set. */ public T getValueSet(String uri, String version) { // Load the value sets, which may contain zero items if the value set does not exist // Typecast necessary to placate the Java compiler calling this Scala function T[] valueSets = (T[]) this.valueSets.filter( col("url").equalTo(lit(uri)) .and(col("version").equalTo(lit(version)))) .head(1); if (valueSets.length == 0) { return null; } else { T valueSet = valueSets[0]; Dataset<Value> filteredValues = getValues(uri, version); addToValueSet(valueSet, filteredValues); return valueSet; } }
/** * Returns the concept map with the given uri and version, or null if there is no such map. * * @param uri the uri of the map to return * @param version the version of the map to return * @return the specified concept map. */ public T getConceptMap(String uri, String version) { // Load the concept maps, which may contain zero items // if the map does not exist. // Typecast necessary to placate the Java compiler calling this Scala function. T[] maps = (T[]) this.conceptMaps.filter( functions.col("url").equalTo(lit(uri)) .and(functions.col("version").equalTo(lit(version)))) .head(1); if (maps.length == 0) { return null; } else { T map = maps[0]; Dataset<Mapping> filteredMappings = getMappings(uri, version); addToConceptMap(map, filteredMappings); return map; } }
referencesToLoad, col("present.valueSetUri").equalTo(col("toload.valueSetUri")) .and(col("present.valueSetVersion").equalTo(col("toload.valueSetVersion")))) .select("referenceName", "system", "value") .collectAsList(); ancestorsToLoad, col("present.uri").equalTo(col("toload.uri")) .and(col("present.version").equalTo(col("toload.version"))) .and(col("present.ancestorSystem").equalTo(col("toload.ancestorSystem"))) .and(col("present.ancestorValue").equalTo(col("toload.ancestorValue")))) .select("referenceName", "descendantSystem", "descendantValue") .collectAsList();
/** * Returns all concept maps that are disjoint with concept maps stored in the default database and * adds them to our collection. The directory may be anything readable from a Spark path, * including local filesystems, HDFS, S3, or others. * * @param path a path from which disjoint concept maps will be loaded * @param database the database to check concept maps against * @return an instance of ConceptMaps that includes content from that directory that is disjoint * with content already contained in the default database. */ public C withDisjointMapsFromDirectory(String path, String database) { Dataset<UrlAndVersion> currentMembers = this.spark .sql("SELECT url, version FROM " + database + "." + CONCEPT_MAP_TABLE) .as(URL_AND_VERSION_ENCODER) .alias("current"); Dataset<T> maps = conceptMapsDatasetFromDirectory(path) .alias("new") .join(currentMembers, col("new.url").equalTo(col("current.url")) .and(col("new.version").equalTo(col("current.version"))), "leftanti") .as(conceptMapEncoder); return withConceptMaps(maps); }
/** * Reads the LOINC mutliaxial hierarchy file and converts it to a {@link HierarchicalElement} * dataset. * * @param spark the Spark session * @param loincHierarchyPath path to the multiaxial hierarchy CSV * @return a dataset of {@link HierarchicalElement} representing the hierarchical relationship. */ public static Dataset<HierarchicalElement> readMultiaxialHierarchyFile(SparkSession spark, String loincHierarchyPath) { return spark.read() .option("header", true) .csv(loincHierarchyPath) .select(col("IMMEDIATE_PARENT"), col("CODE")) .where(col("IMMEDIATE_PARENT").isNotNull() .and(col("IMMEDIATE_PARENT").notEqual(lit("")))) .where(col("CODE").isNotNull() .and(col("CODE").notEqual(lit("")))) .map((MapFunction<Row, HierarchicalElement>) row -> { HierarchicalElement element = new HierarchicalElement(); element.setAncestorSystem(LOINC_CODE_SYSTEM_URI); element.setAncestorValue(row.getString(0)); element.setDescendantSystem(LOINC_CODE_SYSTEM_URI); element.setDescendantValue(row.getString(1)); return element; }, Hierarchies.getHierarchicalElementEncoder()); }
/** * Returns all concept maps that are disjoint with concept maps stored in the default database and * adds them to our collection. The directory may be anything readable from a Spark path, * including local filesystems, HDFS, S3, or others. * * @param path a path from which disjoint concept maps will be loaded * @param database the database to check concept maps against * @return an instance of ConceptMaps that includes content from that directory that is disjoint * with content already contained in the default database. */ public C withDisjointMapsFromDirectory(String path, String database) { Dataset<UrlAndVersion> currentMembers = this.spark .sql("SELECT url, version FROM " + database + "." + CONCEPT_MAP_TABLE) .as(URL_AND_VERSION_ENCODER) .alias("current"); Dataset<T> maps = conceptMapsDatasetFromDirectory(path) .alias("new") .join(currentMembers, col("new.url").equalTo(col("current.url")) .and(col("new.version").equalTo(col("current.version"))), "leftanti") .as(conceptMapEncoder); return withConceptMaps(maps); }
referencesToLoad, col("present.valueSetUri").equalTo(col("toload.valueSetUri")) .and(col("present.valueSetVersion").equalTo(col("toload.valueSetVersion")))) .select("referenceName", "system", "value") .collectAsList(); ancestorsToLoad, col("present.uri").equalTo(col("toload.uri")) .and(col("present.version").equalTo(col("toload.version"))) .and(col("present.ancestorSystem").equalTo(col("toload.ancestorSystem"))) .and(col("present.ancestorValue").equalTo(col("toload.ancestorValue")))) .select("referenceName", "descendantSystem", "descendantValue") .collectAsList();
/** * Reads the LOINC mutliaxial hierarchy file and converts it to a {@link HierarchicalElement} * dataset. * * @param spark the Spark session * @param loincHierarchyPath path to the multiaxial hierarchy CSV * @return a dataset of {@link HierarchicalElement} representing the hierarchical relationship. */ public static Dataset<HierarchicalElement> readMultiaxialHierarchyFile(SparkSession spark, String loincHierarchyPath) { return spark.read() .option("header", true) .csv(loincHierarchyPath) .select(col("IMMEDIATE_PARENT"), col("CODE")) .where(col("IMMEDIATE_PARENT").isNotNull() .and(col("IMMEDIATE_PARENT").notEqual(lit("")))) .where(col("CODE").isNotNull() .and(col("CODE").notEqual(lit("")))) .map((MapFunction<Row, HierarchicalElement>) row -> { HierarchicalElement element = new HierarchicalElement(); element.setAncestorSystem(LOINC_CODE_SYSTEM_URI); element.setAncestorValue(row.getString(0)); element.setDescendantSystem(LOINC_CODE_SYSTEM_URI); element.setDescendantValue(row.getString(1)); return element; }, Hierarchies.getHierarchicalElementEncoder()); }
/** * Returns all value sets that are disjoint with value sets stored in the given database and * adds them to our collection. The directory may be anything readable from a Spark path, * including local filesystems, HDFS, S3, or others. * * @param path a path from which disjoint value sets will be loaded * @param database the database to check value sets against * @return an instance of ValueSets that includes content from that directory that is disjoint * with content already contained in the given database. */ public C withDisjointValueSetsFromDirectory(String path, String database) { Dataset<UrlAndVersion> currentMembers = this.spark.table(database + "." + VALUE_SETS_TABLE) .select("url", "version") .distinct() .as(URL_AND_VERSION_ENCODER) .alias("current"); Dataset<T> valueSets = valueSetDatasetFromDirectory(path) .alias("new") .join(currentMembers, col("new.url").equalTo(col("current.url")) .and(col("new.version").equalTo(col("current.version"))), "leftanti") .as(valueSetEncoder); return withValueSets(valueSets); }
/** * Returns all value sets that are disjoint with value sets stored in the given database and * adds them to our collection. The directory may be anything readable from a Spark path, * including local filesystems, HDFS, S3, or others. * * @param path a path from which disjoint value sets will be loaded * @param database the database to check value sets against * @return an instance of ValueSets that includes content from that directory that is disjoint * with content already contained in the given database. */ public C withDisjointValueSetsFromDirectory(String path, String database) { Dataset<UrlAndVersion> currentMembers = this.spark.table(database + "." + VALUE_SETS_TABLE) .select("url", "version") .distinct() .as(URL_AND_VERSION_ENCODER) .alias("current"); Dataset<T> valueSets = valueSetDatasetFromDirectory(path) .alias("new") .join(currentMembers, col("new.url").equalTo(col("current.url")) .and(col("new.version").equalTo(col("current.version"))), "leftanti") .as(valueSetEncoder); return withValueSets(valueSets); }
/** * Reads a Snomed relationship file and converts it to a {@link HierarchicalElement} dataset. * * @param spark the Spark session * @param snomedRelationshipPath path to the SNOMED relationship file * @return a dataset of{@link HierarchicalElement} representing the hierarchical relationship. */ public static Dataset<HierarchicalElement> readRelationshipFile(SparkSession spark, String snomedRelationshipPath) { return spark.read() .option("header", true) .option("delimiter", "\t") .csv(snomedRelationshipPath) .where(col("typeId").equalTo(lit(SNOMED_ISA_RELATIONSHIP_ID))) .where(col("active").equalTo(lit("1"))) .select(col("destinationId"), col("sourceId")) .where(col("destinationId").isNotNull() .and(col("destinationId").notEqual(lit("")))) .where(col("sourceId").isNotNull() .and(col("sourceId").notEqual(lit("")))) .map((MapFunction<Row, HierarchicalElement>) row -> { HierarchicalElement element = new HierarchicalElement(); element.setAncestorSystem(SNOMED_CODE_SYSTEM_URI); element.setAncestorValue(row.getString(0)); element.setDescendantSystem(SNOMED_CODE_SYSTEM_URI); element.setDescendantValue(row.getString(1)); return element; }, Hierarchies.getHierarchicalElementEncoder()); }
/** * Reads a Snomed relationship file and converts it to a {@link HierarchicalElement} dataset. * * @param spark the Spark session * @param snomedRelationshipPath path to the SNOMED relationship file * @return a dataset of{@link HierarchicalElement} representing the hierarchical relationship. */ public static Dataset<HierarchicalElement> readRelationshipFile(SparkSession spark, String snomedRelationshipPath) { return spark.read() .option("header", true) .option("delimiter", "\t") .csv(snomedRelationshipPath) .where(col("typeId").equalTo(lit(SNOMED_ISA_RELATIONSHIP_ID))) .where(col("active").equalTo(lit("1"))) .select(col("destinationId"), col("sourceId")) .where(col("destinationId").isNotNull() .and(col("destinationId").notEqual(lit("")))) .where(col("sourceId").isNotNull() .and(col("sourceId").notEqual(lit("")))) .map((MapFunction<Row, HierarchicalElement>) row -> { HierarchicalElement element = new HierarchicalElement(); element.setAncestorSystem(SNOMED_CODE_SYSTEM_URI); element.setAncestorValue(row.getString(0)); element.setDescendantSystem(SNOMED_CODE_SYSTEM_URI); element.setDescendantValue(row.getString(1)); return element; }, Hierarchies.getHierarchicalElementEncoder()); }