private StructType createSchema() { StructType intervalType = new StructType(new StructField[] { new StructField("startTime", DataTypes.LongType, true, Metadata.empty()), new StructField("endTime", DataTypes.LongType, true, Metadata.empty()) }); DataType intervalsType = new ArrayType(intervalType, false); return new StructType(new StructField[] { new StructField("id", DataTypes.IntegerType, true, Metadata.empty()), new StructField("intervals", intervalsType, true, Metadata.empty()) }); }
map2, complexMap2}); StructType schema = new StructType() .add("a", BooleanType, false) .add("b", IntegerType, false)
map2, complexMap2}); StructType schema = new StructType() .add("a", BooleanType, false) .add("b", IntegerType, false)
map2, complexMap2}); StructType schema = new StructType() .add("a", BooleanType, false) .add("b", IntegerType, false)
StructType rfSchema = new StructType(new StructField[]{ new StructField("word", DataTypes.StringType, false, Metadata.empty()), new StructField("neighbour", DataTypes.StringType, false, Metadata.empty()),
StructType partitionColumns, InternalRow partitionValues) { StructType batchSchema = new StructType(); for (StructField f: sparkSchema.fields()) { batchSchema = batchSchema.add(f);
StructType partitionColumns, InternalRow partitionValues) { StructType batchSchema = new StructType(); for (StructField f: sparkSchema.fields()) { batchSchema = batchSchema.add(f);
public static StructType getStructTypeFromCells(Cells cells) { List<StructField> fields = new ArrayList<>(); for(Cell cell:cells.getCells()) { StructField field = getStructFieldFromCell(cell); fields.add(field); } return new StructType(fields.toArray(new StructField[0])); }
public static StructType getStructTypeFromCells(Cells cells) { List<StructField> fields = new ArrayList<>(); for(Cell cell:cells.getCells()) { StructField field = getStructFieldFromCell(cell); fields.add(field); } return new StructType(fields.toArray(new StructField[0])); }
private static StructType parseSchema(List<? extends ConfigObject> fieldsConfig) { StructField[] fields = new StructField[fieldsConfig.size()]; for (int i = 0; i < fieldsConfig.size(); i++) { fields[i] = parseField(fieldsConfig.get(i).toConfig()); } return new StructType(fields); }
StructType batchSchema = new StructType(); for (StructField f: sparkSchema.fields()) { batchSchema = batchSchema.add(f);
OuterScopes.addOuterScope(this); StructType schema = new StructType() .add("f", new StructType() .add("a", StringType, true) .add("b", IntegerType, true), true);
public StructType add(String name, DataType dataType, Metadata metadata) { return new StructType(on( new ArrayList<>(fields.size() + 1), ArrayList::addAll, fields, ArrayList::add, new StructField(name, dataType, metadata))); }
OuterScopes.addOuterScope(this); StructType schema = new StructType() .add("f", new StructType() .add("a", StringType, true) .add("b", IntegerType, true), true);
OuterScopes.addOuterScope(this); StructType schema = new StructType() .add("f", new StructType() .add("a", StringType, true) .add("b", IntegerType, true), true);
public static Row remove(Row row, String fieldName) { List<StructField> removedFields = Lists.newArrayList(row.schema().fields()); removedFields.remove(row.fieldIndex(fieldName)); StructType removedSchema = new StructType(removedFields.toArray(new StructField[removedFields.size()])); List<Object> removedValues = Lists.newArrayList(RowUtils.valuesFor(row)); removedValues.remove(row.fieldIndex(fieldName)); return new RowWithSchema(removedSchema, removedValues.toArray()); }
@Override public DataType sqlType() { return new StructType( new StructField[] {new StructField("wkb", DataTypes.BinaryType, true, Metadata.empty())}); }
assert(requiredFields.length == requestedColIds.length); StructType resultSchema = new StructType(requiredFields); for (StructField f : partitionSchema.fields()) { resultSchema = resultSchema.add(f);
assert(requiredFields.length == requestedColIds.length); StructType resultSchema = new StructType(requiredFields); for (StructField f : partitionSchema.fields()) { resultSchema = resultSchema.add(f);
@Override public DataFrame transform(DataFrame dataset) { JavaRDD<Row> output = dataset.javaRDD().map(new DecodeFunction()); StructType schema = new StructType(new StructField[]{ new StructField("sentence", DataTypes.StringType, false, Metadata.empty()), new StructField("prediction", DataTypes.StringType, false, Metadata.empty()) }); return dataset.sqlContext().createDataFrame(output, schema); }