@Test public void testSaveModeAPI() { spark .range(10) .write() .format("org.apache.spark.sql.test") .mode(SaveMode.ErrorIfExists) .save(); }
@Test public void saveAndLoad() { Map<String, String> options = new HashMap<>(); options.put("path", path.toString()); df.write().mode(SaveMode.ErrorIfExists).format("json").options(options).save(); Dataset<Row> loadedDF = spark.read().format("json").options(options).load(); checkAnswer(loadedDF, df.collectAsList()); }
@Test public void saveAndLoad() { Map<String, String> options = new HashMap<>(); options.put("path", path.toString()); df.write().mode(SaveMode.ErrorIfExists).format("json").options(options).save(); Dataset<Row> loadedDF = spark.read().format("json").options(options).load(); checkAnswer(loadedDF, df.collectAsList()); }
@Test public void saveAndLoad() { Map<String, String> options = new HashMap<>(); options.put("path", path.toString()); df.write().mode(SaveMode.ErrorIfExists).format("json").options(options).save(); Dataset<Row> loadedDF = spark.read().format("json").options(options).load(); checkAnswer(loadedDF, df.collectAsList()); }
@Test public void testSaveModeAPI() { spark .range(10) .write() .format("org.apache.spark.sql.test") .mode(SaveMode.ErrorIfExists) .save(); }
@Test public void saveAndLoadWithSchema() { Map<String, String> options = new HashMap<>(); options.put("path", path.toString()); df.write().format("json").mode(SaveMode.ErrorIfExists).options(options).save(); List<StructField> fields = new ArrayList<>(); fields.add(DataTypes.createStructField("b", DataTypes.StringType, true)); StructType schema = DataTypes.createStructType(fields); Dataset<Row> loadedDF = spark.read().format("json").schema(schema).options(options).load(); checkAnswer(loadedDF, spark.sql("SELECT b FROM jsonTable").collectAsList()); } }
@Test public void saveAndLoadWithSchema() { Map<String, String> options = new HashMap<>(); options.put("path", path.toString()); df.write().format("json").mode(SaveMode.ErrorIfExists).options(options).save(); List<StructField> fields = new ArrayList<>(); fields.add(DataTypes.createStructField("b", DataTypes.StringType, true)); StructType schema = DataTypes.createStructType(fields); Dataset<Row> loadedDF = spark.read().format("json").schema(schema).options(options).load(); checkAnswer(loadedDF, spark.sql("SELECT b FROM jsonTable").collectAsList()); } }
@Test public void testFormatAPI() { spark .read() .format("org.apache.spark.sql.test") .load() .write() .format("org.apache.spark.sql.test") .save(); }
@Test public void testFormatAPI() { spark .read() .format("org.apache.spark.sql.test") .load() .write() .format("org.apache.spark.sql.test") .save(); }
@Test public void testTextAPI() { spark.read().text(); spark.read().text(input); spark.read().text(input, input, input); spark.read().text(new String[]{input, input}) .write().text(output); }
@Test public void testTextAPI() { spark.read().text(); spark.read().text(input); spark.read().text(input, input, input); spark.read().text(new String[]{input, input}) .write().text(output); }
/** * This only tests whether API compiles, but does not run it as orc() * cannot be run without Hive classes. */ public void testOrcAPI() { spark.read().schema(schema).orc(); spark.read().schema(schema).orc(input); spark.read().schema(schema).orc(input, input, input); spark.read().schema(schema).orc(new String[]{input, input}) .write().orc(output); } }
/** * This only tests whether API compiles, but does not run it as orc() * cannot be run without Hive classes. */ public void testOrcAPI() { spark.read().schema(schema).orc(); spark.read().schema(schema).orc(input); spark.read().schema(schema).orc(input, input, input); spark.read().schema(schema).orc(new String[]{input, input}) .write().orc(output); } }
@Test public void testParquetAPI() { spark.read().schema(schema).parquet(); spark.read().schema(schema).parquet(input); spark.read().schema(schema).parquet(input, input, input); spark.read().schema(schema).parquet(new String[] { input, input }) .write().parquet(output); }
@Test public void testCsvAPI() { spark.read().schema(schema).csv(); spark.read().schema(schema).csv(input); spark.read().schema(schema).csv(input, input, input); spark.read().schema(schema).csv(new String[]{input, input}) .write().csv(output); }
@Test public void testCsvAPI() { spark.read().schema(schema).csv(); spark.read().schema(schema).csv(input); spark.read().schema(schema).csv(input, input, input); spark.read().schema(schema).csv(new String[]{input, input}) .write().csv(output); }
@Test public void testJsonAPI() { spark.read().schema(schema).json(); spark.read().schema(schema).json(input); spark.read().schema(schema).json(input, input, input); spark.read().schema(schema).json(new String[]{input, input}) .write().json(output); }
@Test public void testParquetAPI() { spark.read().schema(schema).parquet(); spark.read().schema(schema).parquet(input); spark.read().schema(schema).parquet(input, input, input); spark.read().schema(schema).parquet(new String[] { input, input }) .write().parquet(output); }
@Test public void testJsonAPI() { spark.read().schema(schema).json(); spark.read().schema(schema).json(input); spark.read().schema(schema).json(input, input, input); spark.read().schema(schema).json(new String[]{input, input}) .write().json(output); }
@Test public void testJsonAPI() { spark.read().schema(schema).json(); spark.read().schema(schema).json(input); spark.read().schema(schema).json(input, input, input); spark.read().schema(schema).json(new String[]{input, input}) .write().json(output); }