/** * Returns the name of the file. */ @Override public String toString() { return fragment.getPath().toString(); }
public OrcScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { super(conf, schema, meta, fragment); this.path = this.fragment.getPath(); this.fileSystem = this.path.getFileSystem(conf); }
/** * Returns the name of the file. */ @Override public String toString() { return fragment.getPath().toString(); }
public FileScannerV2(final Configuration conf, final TableMeta meta, final Schema schema, final FileFragment fragment) throws IOException { this.conf = conf; this.meta = meta; this.schema = schema; this.fragment = fragment; this.columnNum = this.schema.size(); this.fs = fragment.getPath().getFileSystem(conf); }
private List<Path> fragmentToPath(List<FragmentProto> tableFragments) { List<Path> list = new ArrayList<>(); for (FragmentProto proto : tableFragments) { FileFragment fragment = FragmentConvertor.convert(getConf(), proto); list.add(fragment.getPath()); } return list; }
public static String getIndexFileName(FragmentProto fragmentProto) { FileFragment fileFragment = FragmentConvertor.convert(FileFragment.class, fragmentProto); StringBuilder sb = new StringBuilder(); sb.append(fileFragment.getPath().getName()).append(fileFragment.getStartKey()).append(fileFragment.getLength()); return sb.toString(); } }
public static String getIndexFileName(Configuration conf, FragmentProto fragmentProto) { FileFragment fileFragment = FragmentConvertor.convert(conf, fragmentProto); StringBuilder sb = new StringBuilder(); sb.append(fileFragment.getPath().getName()).append(fileFragment.getStartKey()).append(fileFragment.getLength()); return sb.toString(); } }
private List<Path> fragmentToPath(List<FragmentProto> tableFragments) { List<Path> list = TUtil.newList(); for (FragmentProto proto : tableFragments) { FileFragment fragment = FragmentConvertor.convert(FileFragment.class, proto); list.add(fragment.getPath()); } return list; }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ tableName +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getEndKey() + "}" ; }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ inputSourceId +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getLength() + "}" ; } }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ tableName +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getLength() + "}" ; }
@Override public void init() throws IOException { reader.init(); if (targets == null) { targets = schema.toArray(); } reset(); super.init(); if (LOG.isDebugEnabled()) { LOG.debug("DelimitedTextFileScanner open:" + fragment.getPath() + "," + startOffset + "," + endOffset); } }
public CSVFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final FileFragment fragment) throws IOException { super(conf, meta, schema, fragment); factory = new CompressionCodecFactory(conf); codec = factory.getCodec(this.fragment.getPath()); if (isCompress() && !(codec instanceof SplittableCompressionCodec)) { splittable = false; } }
public DelimitedLineReader(Configuration conf, final FileFragment fragment, int bufferSize) throws IOException { this.fragment = fragment; this.conf = conf; this.factory = new CompressionCodecFactory(conf); this.codec = factory.getCodec(fragment.getPath()); this.bufferSize = bufferSize; if (this.codec instanceof SplittableCompressionCodec) { // bzip2 does not support multi-thread model throw new TajoRuntimeException( new NotImplementedException(this.getClass() + " does not support " + this.codec.getDefaultExtension())); } }
/** * Initializes the ParquetScanner. This method initializes the * TajoParquetReader. */ @Override public void init() throws IOException { if (targets == null) { targets = schema.toArray(); } reader = new TajoParquetReader(fragment.getPath(), schema, new Schema(targets)); super.init(); }
@Test public final void testGetAndSetFields() { FileFragment fragment1 = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }
@Test public final void testGetProtoAndRestore() { FileFragment fragment = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); FileFragment fragment1 = FragmentConvertor.convert(conf, BuiltinFragmentKinds.FILE, FragmentConvertor.toFragmentProto(conf, fragment)); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }