public TempFileReader(List<Type> types, OrcDataSource dataSource) { this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); try { OrcReader orcReader = new OrcReader( dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(16, MEGABYTE)); Map<Integer, Type> includedColumns = new HashMap<>(); for (int i = 0; i < types.size(); i++) { includedColumns.put(i, types.get(i)); } reader = orcReader.createRecordReader( includedColumns, OrcPredicate.TRUE, UTC, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); } catch (IOException e) { throw new PrestoException(HIVE_WRITER_DATA_ERROR, "Failed to read temporary data"); } }
static void validateFile( OrcWriteValidation writeValidation, OrcDataSource input, List<Type> types, DateTimeZone hiveStorageTimeZone, OrcEncoding orcEncoding) throws OrcCorruptionException { ImmutableMap.Builder<Integer, Type> readTypes = ImmutableMap.builder(); for (int columnIndex = 0; columnIndex < types.size(); columnIndex++) { readTypes.put(columnIndex, types.get(columnIndex)); } try { OrcReader orcReader = new OrcReader(input, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(16, MEGABYTE), Optional.of(writeValidation)); try (OrcRecordReader orcRecordReader = orcReader.createRecordReader(readTypes.build(), OrcPredicate.TRUE, hiveStorageTimeZone, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE)) { while (orcRecordReader.nextBatch() >= 0) { // ignored } } } catch (IOException e) { throw new OrcCorruptionException(e, input.getId(), "Validation failed"); } } }
private RowBlock read(TempFile tempFile, Type readerType) throws IOException { DataSize dataSize = new DataSize(1, MEGABYTE); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), dataSize, dataSize, dataSize, true); OrcReader orcReader = new OrcReader(orcDataSource, ORC, dataSize, dataSize, dataSize, dataSize); Map<Integer, Type> includedColumns = new HashMap<>(); includedColumns.put(0, readerType); OrcRecordReader recordReader = orcReader.createRecordReader(includedColumns, OrcPredicate.TRUE, UTC, newSimpleAggregatedMemoryContext(), OrcReader.INITIAL_BATCH_SIZE); recordReader.nextBatch(); RowBlock block = (RowBlock) recordReader.readBlock(readerType, 0); recordReader.close(); return block; }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(booleanWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, BOOLEAN), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(tinyIntWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, TINYINT), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(timestampNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, TIMESTAMP), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(tinyIntNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, TINYINT), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(decimalNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DECIMAL_TYPE), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(floatNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, REAL), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(doubleNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DOUBLE), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(doubleWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DOUBLE), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(bigintWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, BIGINT), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(floatWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, REAL), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(bigintNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, BIGINT), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(varcharWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, VARCHAR), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(timestampWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, TIMESTAMP), OrcPredicate.TRUE, UTC, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(dataPath, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DECIMAL_TYPE), OrcPredicate.TRUE, DateTimeZone.UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(booleanNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, BOOLEAN), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(decimalWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DECIMAL_TYPE), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(varcharNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, VARCHAR), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }