@Override public WriterCommitMessage commit() throws IOException { closeCurrent(); return new TaskCommit(completedFiles); }
private PartitionKey(PartitionKey toCopy) { this.spec = toCopy.spec; this.size = toCopy.size; this.partitionTuple = new Object[toCopy.partitionTuple.length]; this.transforms = toCopy.transforms; this.accessors = toCopy.accessors; for (int i = 0; i < partitionTuple.length; i += 1) { this.partitionTuple[i] = defensiveCopyIfNeeded(toCopy.partitionTuple[i]); } }
static TestTable create(File temp, String name, Schema schema, PartitionSpec spec) { TestTableOperations ops = new TestTableOperations(name); if (ops.current() != null) { throw new AlreadyExistsException("Table %s already exists at location: %s", name, temp); } ops.commit(null, TableMetadata.newTableMetadata(ops, schema, spec, temp.toString())); return new TestTable(ops, name); }
@Override public DataReader<UnsafeRow> createDataReader() { return new TaskDataReader(task, lazyTableSchema(), lazyExpectedSchema(), conf.value()); }
private static Accessor<InternalRow> newAccessor(int p, boolean isOptional, Types.StructType type, Accessor<InternalRow> accessor) { int size = type.fields().size(); if (isOptional) { // the wrapped position handles null layers return new WrappedPositionAccessor(p, size, accessor); } else if (accessor instanceof PositionAccessor) { return new Position2Accessor(p, size, (PositionAccessor) accessor); } else if (accessor instanceof Position2Accessor) { return new Position3Accessor(p, size, (Position2Accessor) accessor); } else { return new WrappedPositionAccessor(p, size, accessor); } }
private static Accessor<InternalRow> newAccessor(int p, Type type) { switch (type.typeId()) { case STRING: return new StringAccessor(p, convert(type)); case DECIMAL: return new DecimalAccessor(p, convert(type)); default: return new PositionAccessor(p, convert(type)); } }
@Override protected void writeAndValidate(Schema schema) throws IOException { File location = createTableFolder(); Table table = createTable(schema, location); writeAndValidateWithLocations(table, location, new File(location, "data")); }
static TestTable load(String name) { TestTableOperations ops = new TestTableOperations(name); return new TestTable(ops, name); }
@Override public DataWriterFactory<InternalRow> createInternalRowWriterFactory() { return new WriterFactory(table.spec(), format, dataLocation(), table.properties(), conf); }
PartitionedWriter(PartitionSpec spec, FileFormat format, Configuration conf, AppenderFactory<InternalRow> factory, Function<PartitionKey, Path> outputPathFunc) { this.spec = spec; this.format = format; this.conf = conf; this.factory = factory; this.outputPathFunc = outputPathFunc; this.key = new PartitionKey(spec); }
@Override public Object get(InternalRow row) { InternalRow inner = row.getStruct(p, size); if (inner != null) { return accessor.get(inner); } return null; } }
@Override public StructType readSchema() { return lazyType(); }
@Override public FileIO io() { return new LocalFileIO(); }
private static List<Row> read(String table, String expr) { return read(table, expr, "*"); }
private StructType lazyType() { if (type == null) { this.type = convert(lazySchema()); } return type; }
private void pushFilters(DataSourceReader reader, com.netflix.iceberg.expressions.Expression... filters) { Expression[] expressions = new Expression[filters.length]; for (int i = 0; i < filters.length; i += 1) { expressions[i] = SparkExpressions.convert(filters[i], SCHEMA); } pushFilters(reader, expressions); }
PartitionKey copy() { return new PartitionKey(this); }
@SuppressWarnings("unchecked") void partition(InternalRow row) { for (int i = 0; i < partitionTuple.length; i += 1) { Transform<Object, Object> transform = transforms[i]; partitionTuple[i] = transform.apply(accessors[i].get(row)); } }