@Override public Dataset<Row> read() throws Exception { Properties properties = new Properties(); properties.put("user", username); properties.put("password", password); return Contexts.getSparkSession().read().jdbc(url, tableName, properties); }
public static void main(String[] args) { //JDBC connection properties final Properties connectionProperties = new Properties(); connectionProperties.put("user", MYSQL_USERNAME); connectionProperties.put("password", MYSQL_PWD); final String dbTable = "(select emp_no, concat_ws(' ', first_name, last_name) as full_name from employees) as employees_name"; //Load MySQL query result as Dataset Dataset<Row> jdbcDF = sparkSession.read() .jdbc(MYSQL_CONNECTION_URL, dbTable, "emp_no", 10001, 499999, 10, connectionProperties); List<Row> employeeFullNameRows = jdbcDF.collectAsList(); for (Row employeeFullNameRow : employeeFullNameRows) { LOGGER.info(employeeFullNameRow); } } }
@Override public Dataset<Row> jdbc(final String url, final String table, final java.util.Properties properties) { final boolean userTriggered = initializeFunction(url, table, properties); final Dataset<Row> result = Dataset.from(super.jdbc(url, table, properties)); this.setIsUserTriggered(userTriggered); return result; }
@Override public Dataset<Row> jdbc(final String url, final String table, final String[] predicates, final java.util.Properties connectionProperties) { final boolean userTriggered = initializeFunction(url, table, predicates, connectionProperties); final Dataset<Row> result = Dataset.from(super.jdbc(url, table, predicates, connectionProperties)); this.setIsUserTriggered(userTriggered); return result; }
@Override public Dataset<Row> jdbc(final String url, final String table, final String columnName, final long lowerBound, final long upperBound, final int numPartitions, final java.util.Properties connectionProperties) { final boolean userTriggered = initializeFunction( url, table, columnName, lowerBound, upperBound, numPartitions, connectionProperties); final Dataset<Row> result = Dataset.from(super.jdbc( url, table, columnName, lowerBound, upperBound, numPartitions, connectionProperties)); this.setIsUserTriggered(userTriggered); return result; }
private void start() { SparkSession spark = SparkSession.builder() .appName("Dataset from MySQL JDBC Connection") .master("local") .getOrCreate(); java.util.Properties props = new Properties(); props.put("user", "root"); props.put("password", "password"); props.put("useSSL", "false"); Dataset<Row> df = spark.read().jdbc( "jdbc:mysql://localhost:3306/sakila?serverTimezone=EST", "actor", props); df = df.orderBy(df.col("last_name")); df.show(); } }