@Override public float getProgress() { if(eos) { return 1.0f; } long readBytes = filePosition - startOffset; if (readBytes == 0) { return 0.0f; } else { return Math.min(1.0f, ((float) readBytes / fragment.getLength())); } } }
public RCFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); conf.setInt("io.file.buffer.size", 4096); //TODO remove startOffset = this.fragment.getStartKey(); endOffset = startOffset + this.fragment.getLength(); start = 0; }
@Override public float getProgress() { if(!inited) return 0.0f; if(eos) { return 1.0f; } long readBytes = filePosition - fragment.getStartKey(); if (readBytes == 0) { return 0.0f; } else { return Math.min(1.0f, ((float) readBytes / fragment.getLength())); } } }
@Override public void seek(long offset) throws IOException { eos = false; filePosition = channel.position(); // do not fill the buffer if the offset is already included in the buffer. if(!forceFillBuffer && filePosition > offset && offset > filePosition - buffer.limit()){ buffer.position((int)(offset - (filePosition - buffer.limit()))); } else { if(offset < startOffset || offset > startOffset + fragment.getLength()){ throw new IndexOutOfBoundsException(String.format("range(%d, %d), offset: %d", startOffset, startOffset + fragment.getLength(), offset)); } channel.position(offset); filePosition = offset; buffer.clear(); forceFillBuffer = true; fillBuffer(); } }
public static String getIndexFileName(FragmentProto fragmentProto) { FileFragment fileFragment = FragmentConvertor.convert(FileFragment.class, fragmentProto); StringBuilder sb = new StringBuilder(); sb.append(fileFragment.getPath().getName()).append(fileFragment.getStartKey()).append(fileFragment.getLength()); return sb.toString(); } }
public static String getIndexFileName(Configuration conf, FragmentProto fragmentProto) { FileFragment fileFragment = FragmentConvertor.convert(conf, fragmentProto); StringBuilder sb = new StringBuilder(); sb.append(fileFragment.getPath().getName()).append(fileFragment.getStartKey()).append(fileFragment.getLength()); return sb.toString(); } }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ inputSourceId +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getLength() + "}" ; } }
@Override public float getProgress() { if(!inited) return super.getProgress(); try { if(!more) { return 1.0f; } long filePos = getPosition(); if (startOffset == filePos) { return 0.0f; } else { //if scanner read the header, filePos moved to zero return Math.min(1.0f, (float)(Math.max(filePos - startOffset, 0)) / (float)(fragment.getLength())); } } catch (IOException e) { LOG.error(e.getMessage(), e); return 0.0f; } }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ tableName +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getLength() + "}" ; }
public RowFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); SYNC_INTERVAL = conf.getInt(ConfVars.ROWFILE_SYNC_INTERVAL.varname, ConfVars.ROWFILE_SYNC_INTERVAL.defaultIntVal) * SYNC_SIZE; nullFlags = new BitArray(schema.size()); tupleHeaderSize = nullFlags.bytesLength() + (2 * Short.SIZE / 8); this.start = this.fragment.getStartKey(); this.end = this.start + this.fragment.getLength(); }
@Override public TableStats getInputStats() { if(tableStats != null){ tableStats.setNumRows(recordCount); tableStats.setReadBytes(totalReadBytes); // actual read bytes (scan + rescan * n) tableStats.setNumBytes(fragment.getLength()); } return tableStats; }
@Override public TableStats getInputStats() { if(tableStats != null){ tableStats.setNumRows(recordCount); tableStats.setReadBytes(filePosition - fragment.getStartKey()); // actual read bytes (scan + rescan * n) tableStats.setNumBytes(fragment.getLength()); } return tableStats; }
@Override public TableStats getInputStats() { if (tableStats != null && reader != null) { tableStats.setReadBytes(reader.getReadBytes()); //Actual Processed Bytes. (decompressed bytes + overhead) tableStats.setNumRows(recordCount); tableStats.setNumBytes(fragment.getLength()); } return tableStats; }
@Override public void close() throws IOException { super.close(); if (result != null) { result.close(); } if (finalOutputFiles != null) { for (Chunk chunk : finalOutputFiles) { if (!chunk.isMemory()) { FileFragment frag = chunk.getFragment(); File tmpFile = new File(localFS.makeQualified(frag.getPath()).toUri()); if (frag.getStartKey() == 0 && frag.getLength() == tmpFile.length()) { localFS.delete(frag.getPath(), true); if(LOG.isDebugEnabled()) { debug(LOG, "Delete file: " + frag); } } } } } if(inMemoryTable != null) { inMemoryTable.release(); inMemoryTable = null; } if(executorService != null){ executorService.shutdown(); executorService = null; } plan = null; }
@Override public void close() throws IOException { super.close(); if (result != null) { result.close(); } if (finalOutputFiles != null) { for (Chunk chunk : finalOutputFiles) { if (!chunk.isMemory()) { FileFragment frag = chunk.getFragment(); File tmpFile = new File(localFS.makeQualified(frag.getPath()).toUri()); if (frag.getStartKey() == 0 && frag.getLength() == tmpFile.length()) { localFS.delete(frag.getPath(), true); if(LOG.isDebugEnabled()) { debug(LOG, "Delete file: " + frag); } } } } } if(inMemoryTable != null) { inMemoryTable.release(); inMemoryTable = null; } if(executorService != null){ executorService.shutdown(); executorService = null; } plan = null; }
public void init() throws IOException { inited = true; progress = 0.0f; if (fragment != null) { tableStats.setNumBytes(fragment.getLength()); tableStats.setNumBlocks(1); } if (schema != null) { for(Column eachColumn: schema.getRootColumns()) { ColumnStats columnStats = new ColumnStats(eachColumn); tableStats.addColumnStat(columnStats); } } }
@Test public final void testGetAndSetFields() { FileFragment fragment1 = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }
@Test public final void testGetProtoAndRestore() { FileFragment fragment = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); FileFragment fragment1 = FragmentConvertor.convert(conf, BuiltinFragmentKinds.FILE, FragmentConvertor.toFragmentProto(conf, fragment)); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }