@Override public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException { delegate.allocate(vectorMap); }
@Override public Void run() throws Exception { checkNotNull(reader).setup(mutator); return null; } });
dataset.setPath(file.getPath().toString()); try(RecordReader reader = new AdditionalColumnsRecordReader(((EasyFormatPlugin)formatPlugin).getRecordReader(operatorContext, dfs, dataset, GroupScan.ALL_COLUMNS), explorer.getImplicitFieldsForSample(selection))) { reader.setup(mutator); Map<String, ValueVector> fieldVectorMap = new HashMap<>(); for (VectorWrapper<?> vw : mutator.getContainer()) { fieldVectorMap.put(vw.getField().getName(), vw.getValueVector()); reader.allocate(fieldVectorMap); reader.next(); mutator.getContainer().buildSchema(BatchSchema.SelectionVectorMode.NONE); return mutator.getContainer().getSchema();
@Override public SchemaChangeMutator getSchemaChangeMutator() { return current.getSchemaChangeMutator(); }
@Override public SchemaChangeMutator getSchemaChangeMutator() { return inner.getSchemaChangeMutator(); }
@Override public void setup(OutputMutator output) throws ExecutionSetupException { inner.setup(output); for(Populator p : populators){ p.setup(output); } }
@Override public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException { current.allocate(vectorMap); }
@Override public int next() { // need to make sure number of rows in batch is the same for all the readers int count = -1; for (RecordReader recordReader : delegates) { int n = recordReader.next(); if (count == -1) { count = n; } else { if (count != n) { throw new IllegalStateException(String.format("Inconsistent row count. Reader %s returned %d while " + "previous reader returned %d", recordReader.toString(), n, count)); } } } return count; }
@Override public SchemaChangeMutator getSchemaChangeMutator() { return delegate.getSchemaChangeMutator(); }
@Override public void setup(OutputMutator output) throws ExecutionSetupException { this.outputMutator = output; inner.setup(mutator); newSchema(); }
@Override public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException { inner.allocate(vectorMap); for(Populator p : populators){ p.allocate(); } }
@Override public int next() { final int count = inner.next(); try { for (Populator p : populators) { p.populate(count); } } catch (Throwable t) { throw userExceptionWithDiagnosticInfo(t, count); } return count; }
private void checkAndLearnSchema(){ if (mutator.isSchemaChanged()) { outgoing.buildSchema(SelectionVectorMode.NONE); final BatchSchema newSchema = mutator.transformFunction.apply(outgoing.getSchema()); if (config.mayLearnSchema() && tableSchemaPath != null) { try { schemaUpdater.observedSchemaChange(tableSchemaPath, config.getSchema(), newSchema, currentReader.getSchemaChangeMutator()); } catch(Exception ex){ throw UserException.schemaChangeError(ex) .addContext("Original schema", config.getSchema()) .addContext("New schema", newSchema) .message("Schema change detected but unable to learn schema, query failed. A full table scan may be necessary to fully learn the schema.") .build(logger); } throw UserException.schemaChangeError() .addContext("Original Schema", config.getSchema().toString()) .addContext("New Schema", newSchema.toString()) .message("New schema found and recorded. Please reattempt the query. Multiple attempts may be necessary to fully learn the schema.").build(logger); } else { // TODO: change error if we can't update. No reason to re-run if we didn't update. throw UserException.schemaChangeError().message("Schema change detected but unable to learn schema, query failed. Original: %s, New: %s.", config.getSchema(), newSchema).build(logger); } } }
private void nextReader() { AutoCloseables.closeNoChecked(current); current = null; currentIndex++; if(currentIndex == footer.getBlocks().size()) { return; } current = new ParquetRowiseReader( context, footer, currentIndex, status.getPath().toString(), GroupScan.ALL_COLUMNS, fs, schemaHelper, streamProvider ); try { current.setup(output); } catch (ExecutionSetupException e) { throw Throwables.propagate(e); } }
@Override public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException { super.allocate(vectorMap); inner.allocate(mutator.getFieldVectorMap()); }