protected FileFragment makeSplit(String fragmentId, Path file, long start, long length, String[] hosts) { return new FileFragment(fragmentId, file, start, length, hosts); }
public int getDiskId() { if(fragment.getDiskIds().length <= 0) { //LOG.warn("===> No DiskId:" + fragment.getPath() + ":" + fragment.getStartKey()); return -1; } else { return fragment.getDiskIds()[0]; } }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ inputSourceId +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getLength() + "}" ; } }
@Override public String toString() { return "\"fragment\": {\"id\": \""+ tableName +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " + getEndKey() + "}" ; }
public String toString() { return fragment.getPath() + ":" + fragment.getStartKey(); }
@Test public final void testGetAndSetFields() { FileFragment fragment1 = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }
public Path getPath() { return fragment.getPath(); }
public RCFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); conf.setInt("io.file.buffer.size", 4096); //TODO remove startOffset = this.fragment.getStartKey(); endOffset = startOffset + this.fragment.getLength(); start = 0; }
@Override public FileFragmentProto serialize(FileFragment fragment) { FileFragmentProto.Builder builder = FileFragmentProto.newBuilder(); builder.setId(fragment.inputSourceId); builder.setStartOffset(fragment.startKey); builder.setLength(fragment.length); builder.setPath(fragment.getPath().toString()); if(fragment.getDiskIds() != null) { List<Integer> idList = new ArrayList<>(); for(int eachId: fragment.getDiskIds()) { idList.add(eachId); } builder.addAllDiskIds(idList); } if(fragment.hostNames != null) { builder.addAllHosts(fragment.hostNames); } return builder.build(); }
@Override public void reset() throws IOException { // reset the buffer buffer.clear(); forceFillBuffer = true; filePosition = fragment.getStartKey(); recordCount = 0; channel.position(filePosition); eos = false; }
public RCFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final FileFragment fragment) throws IOException { super(conf, schema, meta, fragment); conf.setInt("io.file.buffer.size", 4096); //TODO remove startOffset = fragment.getStartKey(); endOffset = startOffset + fragment.getEndKey(); start = 0; }
@Test public final void testCompareTo() { final int num = 10; FileFragment[] tablets = new FileFragment[num]; for (int i = num - 1; i >= 0; i--) { tablets[i] = new FileFragment("tablet1_"+i, new Path(path, "tablet0"), i * 500, (i+1) * 500); } Arrays.sort(tablets); for(int i = 0; i < num; i++) { assertEquals("tablet1_"+i, tablets[i].getInputSourceId()); } }
/** * Generate the map of host and make them into Volume Ids. * */ private Map<String, Set<Integer>> getVolumeMap(List<FileFragment> frags) { Map<String, Set<Integer>> volumeMap = new HashMap<String, Set<Integer>>(); for (FileFragment frag : frags) { String[] hosts = frag.getHosts(); int[] diskIds = frag.getDiskIds(); for (int i = 0; i < hosts.length; i++) { Set<Integer> volumeList = volumeMap.get(hosts[i]); if (volumeList == null) { volumeList = new HashSet<Integer>(); volumeMap.put(hosts[i], volumeList); } if (diskIds.length > 0 && diskIds[i] > -1) { volumeList.add(diskIds[i]); } } } return volumeMap; } /**
@Override public float getProgress() { if(eos) { return 1.0f; } long readBytes = filePosition - startOffset; if (readBytes == 0) { return 0.0f; } else { return Math.min(1.0f, ((float) readBytes / fragment.getLength())); } } }
} else if (chunk.getFragment().getInputSourceId().contains(INTERMEDIATE_FILE_PREFIX)) { localFS.delete(chunk.getFragment().getPath(), true); numDeletedFiles++;
public FileFragment(String tableName, Path uri, long start, long length, String[] hosts, int[] diskIds) { this.set(tableName, uri, start, length, hosts, diskIds); } // Non splittable
public FileFragment(FileFragmentProto proto) { init(proto); }
private void setVolumeMeta(List<Fragment> splits, final List<BlockLocation> blockLocations) throws IOException { int locationSize = blockLocations.size(); int splitSize = splits.size(); if (locationSize == 0 || splitSize == 0) return; if (locationSize != splitSize) { // splits and locations don't match up LOG.warn("Number of block locations not equal to number of splits: " + "#locations=" + locationSize + " #splits=" + splitSize); return; } DistributedFileSystem fs = (DistributedFileSystem) this.fs; int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); int blockLocationIdx = 0; Iterator<Fragment> iter = splits.iterator(); while (locationSize > blockLocationIdx) { int subSize = Math.min(locationSize - blockLocationIdx, lsLimit); List<BlockLocation> locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize); //BlockStorageLocation containing additional volume location information for each replica of each block. BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations); for (BlockStorageLocation blockStorageLocation : blockStorageLocations) { ((FileFragment)iter.next()).setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds())); blockLocationIdx++; } } LOG.info("# of splits with volumeId " + splitSize); }
/** * Get the list of Disk Ids * Unknown disk is -1. Others 0 ~ N */ public int[] getDiskIds() { if (diskIds == null) { this.diskIds = new int[getHosts().length]; Arrays.fill(this.diskIds, -1); } return diskIds; }
@Test public final void testGetProtoAndRestore() { FileFragment fragment = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); FileFragment fragment1 = FragmentConvertor.convert(conf, BuiltinFragmentKinds.FILE, FragmentConvertor.toFragmentProto(conf, fragment)); assertEquals("table1_1", fragment1.getInputSourceId()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); assertTrue(500 == fragment1.getLength()); }