public OrcScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { super(conf, schema, meta, fragment); this.path = this.fragment.getPath(); this.fileSystem = this.path.getFileSystem(conf); }
/** * Returns the name of the file. */ @Override public String toString() { return fragment.getPath().toString(); }
private void initChannel() throws IOException { FileSystem fs = FileScanner.getFileSystem((TajoConf) conf, fragment.getPath()); if (fs instanceof LocalFileSystem) { File file; try { if (fragment.getPath().toUri().getScheme() != null) { file = new File(fragment.getPath().toUri()); } else { file = new File(fragment.getPath().toString()); } } catch (IllegalArgumentException iae) { throw new IOException(iae); } channel = new LocalFileInputChannel(new FileInputStream(file)); } else { channel = new FSDataInputChannel(fs.open(fragment.getPath())); } // initial set position if (fragment.getStartKey() > 0) { channel.seek(fragment.getStartKey()); } filePosition = fragment.getStartKey(); endOffset = fragment.getStartKey() + fragment.getLength(); if (LOG.isDebugEnabled()) { LOG.debug("RawFileScanner open:" + fragment.getPath() + ", offset :" + fragment.getStartKey() + ", fragment length :" + fragment.getLength()); } }
@Override public void init() throws IOException { reader.init(); if (targets == null) { targets = schema.toArray(); } reset(); super.init(); if (LOG.isDebugEnabled()) { LOG.debug("DelimitedTextFileScanner open:" + fragment.getPath() + "," + startOffset + "," + endOffset); } }
fs = FileScanner.getFileSystem((TajoConf) conf, fragment.getPath()); fis = fs.open(fragment.getPath()); File file; try { if (fragment.getPath().toUri().getScheme() != null) { file = new File(fragment.getPath().toUri()); } else { file = new File(fragment.getPath().toString()); BufferPool.directBuffer((int) Math.min(bufferSize, fragment.getLength()))); } else { fis = fs.open(fragment.getPath()); fis.seek(startOffset); is = fis;
byte vers = in.readByte(); if (vers != ORIGINAL_MAGIC_VERSION_WITH_METADATA) { throw new IOException(fragment.getPath() + " is a version " + vers + " SequenceFile instead of an RCFile."); } else { if (!Arrays.equals(magic, MAGIC)) { throw new IOException(fragment.getPath() + " not a RCFile and has magic of " + new String(magic)); if (!keyCls.equals(KeyBuffer.class) || !valCls.equals(ValueBuffer.class)) { throw new IOException(fragment.getPath() + " not a RCFile"); throw new IOException(fragment.getPath() + " not a RCFile", e); throw new IOException(fragment.getPath() + " not a RCFile.");
/** * Initializes the AvroScanner. */ @Override public void init() throws IOException { if (targets == null) { targets = schema.toArray(); } prepareProjection(targets); outTuple = new VTuple(projectionMap.length); Schema avroSchema = AvroUtil.getAvroSchema(meta, conf); avroFields = avroSchema.getFields(); DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(avroSchema); SeekableInput input = new FsInput(fragment.getPath(), conf); dataFileReader = new DataFileReader<>(input, datumReader); super.init(); }
public DelimitedLineReader(Configuration conf, final AbstractFileFragment fragment, int bufferSize) throws IOException { this.fragment = fragment; this.conf = conf; this.factory = new CompressionCodecFactory(conf); this.codec = factory.getCodec(fragment.getPath()); this.bufferSize = bufferSize; if (this.codec instanceof SplittableCompressionCodec) { // bzip2 does not support multi-thread model throw new TajoRuntimeException( new NotImplementedException(this.getClass() + " does not support " + this.codec.getDefaultExtension())); } }
FileSystem fs = fragment.getPath().getFileSystem(conf); end = fs.getFileStatus(fragment.getPath()).getLen(); in = openFile(fs, fragment.getPath(), 4096); if (LOG.isDebugEnabled()) { LOG.debug("RCFile open:" + fragment.getPath() + "," + start + "," + (endOffset - startOffset) + "," + fs.getFileStatus(fragment.getPath()).getLen());
public ExampleHttpJsonLineReader(Configuration conf, AbstractFileFragment fragment, int bufferSize) { this.conf = conf; this.fragment = (ExampleHttpFileFragment) fragment; this.bufferSize = bufferSize; CompressionCodecFactory factory = new CompressionCodecFactory(conf); codec = factory.getCodec(fragment.getPath()); if (this.codec instanceof SplittableCompressionCodec) { // bzip2 does not support multi-thread model throw new TajoRuntimeException(new UnsupportedException(codec.getDefaultExtension())); } }
/** * Initializes the ParquetScanner. This method initializes the * TajoParquetReader. */ @Override public void init() throws IOException { if (targets == null) { targets = schema.toArray(); } reader = new TajoParquetReader(conf, fragment.getPath(), schema, SchemaBuilder.builder().addAll(targets).build()); totalRowCount = reader.getTotalRowCount(); currentRowCount = 0; closed = false; super.init(); }
this.path = fragment.getPath(); this.codec = codec; this.types = types;