@Override public void reset() throws IOException { // reset the buffer buffer.clear(); forceFillBuffer = true; filePosition = fragment.getStartKey(); recordCount = 0; channel.position(filePosition); eos = false; }
@Override public void reset() throws IOException { // clear the buffer buffer.clear(); // reload initial buffer channel.position(fragment.getStartKey()); numBytesRead = channel.read(buffer); buffer.flip(); eof = false; }
public RCFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); conf.setInt("io.file.buffer.size", 4096); //TODO remove startOffset = this.fragment.getStartKey(); endOffset = startOffset + this.fragment.getLength(); start = 0; }
@Override public void reset() throws IOException { // reload initial buffer filePosition = fragment.getStartKey(); recordCount = 0; seek(filePosition); eos = false; }
public RCFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final FileFragment fragment) throws IOException { super(conf, schema, meta, fragment); conf.setInt("io.file.buffer.size", 4096); //TODO remove startOffset = fragment.getStartKey(); endOffset = startOffset + fragment.getEndKey(); start = 0; }
public static String getIndexFileName(Configuration conf, FragmentProto fragmentProto) { FileFragment fileFragment = FragmentConvertor.convert(conf, fragmentProto); StringBuilder sb = new StringBuilder(); sb.append(fileFragment.getPath().getName()).append(fileFragment.getStartKey()).append(fileFragment.getLength()); return sb.toString(); } }
public static String getIndexFileName(FragmentProto fragmentProto) { FileFragment fileFragment = FragmentConvertor.convert(FileFragment.class, fragmentProto); StringBuilder sb = new StringBuilder(); sb.append(fileFragment.getPath().getName()).append(fileFragment.getStartKey()).append(fileFragment.getLength()); return sb.toString(); } }
public String getId() { return fragment.getPath().getName() + ":" + fragment.getStartKey() + ":" + fragment.getEndKey() + "_" + System.currentTimeMillis(); }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ tableName +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getEndKey() + "}" ; }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ inputSourceId +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getLength() + "}" ; } }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ tableName +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getLength() + "}" ; }
public RowFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); SYNC_INTERVAL = conf.getInt(ConfVars.ROWFILE_SYNC_INTERVAL.varname, ConfVars.ROWFILE_SYNC_INTERVAL.defaultIntVal) * SYNC_SIZE; nullFlags = new BitArray(schema.size()); tupleHeaderSize = nullFlags.bytesLength() + (2 * Short.SIZE / 8); this.start = this.fragment.getStartKey(); this.end = this.start + this.fragment.getLength(); }
public RowFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final FileFragment fragment) throws IOException { super(conf, schema, meta, fragment); SYNC_INTERVAL = conf.getInt(ConfVars.ROWFILE_SYNC_INTERVAL.varname, ConfVars.ROWFILE_SYNC_INTERVAL.defaultIntVal) * SYNC_SIZE; nullFlags = new BitArray(schema.size()); tupleHeaderSize = nullFlags.bytesLength() + (2 * Short.SIZE / 8); this.start = fragment.getStartKey(); this.end = this.start + fragment.getEndKey(); }
@Override public boolean equals(Object o) { if (o instanceof FileFragment) { FileFragment t = (FileFragment) o; if (getPath().equals(t.getPath()) && TUtil.checkEquals(t.getStartKey(), this.getStartKey()) && TUtil.checkEquals(t.getLength(), this.getLength())) { return true; } } return false; }
@Override public boolean equals(Object o) { if (o instanceof FileFragment) { FileFragment t = (FileFragment) o; if (getPath().equals(t.getPath()) && TUtil.checkEquals(t.getStartKey(), this.getStartKey()) && TUtil.checkEquals(t.getLength(), this.getLength())) { return true; } } return false; }
@Override public boolean equals(Object o) { if (o instanceof FileFragment) { FileFragment t = (FileFragment) o; if (getPath().equals(t.getPath()) && TUtil.checkEquals(t.getStartKey(), this.getStartKey()) && TUtil.checkEquals(t.getEndKey(), this.getEndKey())) { return true; } } return false; }
@Override public TableStats getInputStats() { if(tableStats != null){ tableStats.setNumRows(recordCount); tableStats.setReadBytes(filePosition - fragment.getStartKey()); // actual read bytes (scan + rescan * n) tableStats.setNumBytes(fragment.getLength()); } return tableStats; }
@Override public void close() throws IOException { if (tableStats != null) { tableStats.setReadBytes(filePosition - fragment.getStartKey()); tableStats.setNumRows(recordCount); } if(tupleBuffer != null) { tupleBuffer.release(); tupleBuffer = null; } reader = null; IOUtils.cleanup(LOG, channel); }
@Test public final void testGetAndSetFields() { FileFragment fragment1 = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }
@Test public final void testGetProtoAndRestore() { FileFragment fragment = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); FileFragment fragment1 = FragmentConvertor.convert(conf, BuiltinFragmentKinds.FILE, FragmentConvertor.toFragmentProto(conf, fragment)); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }