/** * @param clsName Input split class name. * @param in Input stream. * @param hosts Optional hosts. * @return File block or {@code null} if it is not a {@link FileSplit} instance. * @throws IgniteCheckedException If failed. */ @Nullable public static HadoopFileBlock readFileBlock(String clsName, FSDataInputStream in, @Nullable String[] hosts) throws IgniteCheckedException { if (!FileSplit.class.getName().equals(clsName)) return null; FileSplit split = U.newInstance(FileSplit.class); try { split.readFields(in); } catch (IOException e) { throw new IgniteCheckedException(e); } if (hosts == null) hosts = EMPTY_HOSTS; return new HadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength()); } }
HadoopFileBlock fileBlock1 = new HadoopFileBlock(HOSTS, inFileUri, 0, igfs.info(inFile).length() - 1); pw.println("world3 hello3"); HadoopFileBlock fileBlock2 = new HadoopFileBlock(HOSTS, inFileUri, fileBlock1.length(), igfs.info(inFile).length() - fileBlock1.length());
/** * Sort input splits by length. * * @param splits Splits. * @return Sorted splits. */ public static List<HadoopInputSplit> sortInputSplits(Collection<HadoopInputSplit> splits) { int id = 0; TreeSet<SplitSortWrapper> sortedSplits = new TreeSet<>(); for (HadoopInputSplit split : splits) { long len = split instanceof HadoopFileBlock ? ((HadoopFileBlock)split).length() : 0; sortedSplits.add(new SplitSortWrapper(id++, split, len)); } ArrayList<HadoopInputSplit> res = new ArrayList<>(sortedSplits.size()); for (SplitSortWrapper sortedSplit : sortedSplits) res.add(sortedSplit.split); return res; }
/** * Get affinity nodes for the given input split. * <p> * Order in the returned collection *is* significant, meaning that nodes containing more data * go first. This way, the 1st nodes in the collection considered to be preferable for scheduling. * * @param split Split. * @param top Topology. * @return Affintiy nodes. * @throws IgniteCheckedException If failed. */ private Collection<UUID> affinityNodesForSplit(HadoopInputSplit split, HadoopMapReducePlanTopology top) throws IgniteCheckedException { Collection<UUID> igfsNodeIds = igfsAffinityNodesForSplit(split); if (igfsNodeIds != null) return igfsNodeIds; Map<NodeIdAndLength, UUID> res = new TreeMap<>(); for (String host : split.hosts()) { long len = split instanceof HadoopFileBlock ? ((HadoopFileBlock)split).length() : 0L; HadoopMapReducePlanGroup grp = top.groupForHost(host); if (grp != null) { for (int i = 0; i < grp.nodeCount(); i++) { UUID nodeId = grp.nodeId(i); res.put(new NodeIdAndLength(nodeId, len), nodeId); } } } return new LinkedHashSet<>(res.values()); }
/** {@inheritDoc} */ @Override public InputSplit getInputSplit() { if (inputSplit == null) { HadoopInputSplit split = ctx.taskInfo().inputSplit(); if (split == null) return null; if (split instanceof HadoopFileBlock) { HadoopFileBlock fileBlock = (HadoopFileBlock)split; inputSplit = new FileSplit(new Path(fileBlock.file()), fileBlock.start(), fileBlock.length(), null); } else { try { inputSplit = (InputSplit) ((HadoopV2TaskContext)ctx).getNativeSplit(split); } catch (IgniteCheckedException e) { throw new IllegalStateException(e); } } } return inputSplit; }
/** * @param clsName Input split class name. * @param in Input stream. * @param hosts Optional hosts. * @return File block or {@code null} if it is not a {@link FileSplit} instance. * @throws IgniteCheckedException If failed. */ public static HadoopFileBlock readFileBlock(String clsName, DataInput in, @Nullable String[] hosts) throws IgniteCheckedException { if (!FileSplit.class.getName().equals(clsName)) return null; FileSplit split = new FileSplit(); try { split.readFields(in); } catch (IOException e) { throw new IgniteCheckedException(e); } if (hosts == null) hosts = EMPTY_HOSTS; return new HadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength()); } }
HadoopFileBlock split0 = (HadoopFileBlock)split; if (IgniteFileSystem.IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) { HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority()); if (igfs != null && !igfs.isProxy(split0.file())) { IgfsPath path = new IgfsPath(split0.file()); blocks = igfs.affinity(path, split0.start(), split0.length()); ", start=" + split0.start() + ", len=" + split0.length() + ']', e);
FileSplit s = (FileSplit)nativeSplit; res.add(new HadoopFileBlock(s.getLocations(), s.getPath().toUri(), s.getStart(), s.getLength()));
/** * @param jobConf Job configuration. * @return Collection of mapped splits. * @throws IgniteCheckedException If mapping failed. */ public static Collection<HadoopInputSplit> splitJob(JobConf jobConf) throws IgniteCheckedException { try { InputFormat<?, ?> format = jobConf.getInputFormat(); assert format != null; InputSplit[] splits = format.getSplits(jobConf, 0); Collection<HadoopInputSplit> res = new ArrayList<>(splits.length); for (int i = 0; i < splits.length; i++) { InputSplit nativeSplit = splits[i]; if (nativeSplit instanceof FileSplit) { FileSplit s = (FileSplit)nativeSplit; res.add(new HadoopFileBlock(s.getLocations(), s.getPath().toUri(), s.getStart(), s.getLength())); } else res.add(HadoopUtils.wrapSplit(i, nativeSplit, nativeSplit.getLocations())); } return res; } catch (IOException e) { throw new IgniteCheckedException(e); } }
/** * Test one HDFS splits. * * @throws Exception If failed. */ @Test public void testHdfsSplitsAffinity() throws Exception { IgfsMock igfs = LocationsBuilder.create().add(0, NODE_1).add(50, NODE_2).add(100, NODE_3).buildIgfs(); final List<HadoopInputSplit> splits = new ArrayList<>(); splits.add(new HadoopFileBlock(new String[] { HOST_1 }, URI.create("hfds://" + HOST_1 + "/x"), 0, 50)); splits.add(new HadoopFileBlock(new String[] { HOST_2 }, URI.create("hfds://" + HOST_2 + "/x"), 50, 100)); splits.add(new HadoopFileBlock(new String[] { HOST_3 }, URI.create("hfds://" + HOST_3 + "/x"), 100, 37)); // The following splits belong to hosts that are out of Ignite topology at all. // This means that these splits should be assigned to any least loaded modes: splits.add(new HadoopFileBlock(new String[] { HOST_4 }, URI.create("hfds://" + HOST_4 + "/x"), 138, 2)); splits.add(new HadoopFileBlock(new String[] { HOST_5 }, URI.create("hfds://" + HOST_5 + "/x"), 140, 3)); final int expReducers = 7; HadoopPlannerMockJob job = new HadoopPlannerMockJob(splits, expReducers); IgniteHadoopWeightedMapReducePlanner planner = createPlanner(igfs); final HadoopMapReducePlan plan = planner.preparePlan(job, NODES, null); checkPlanMappers(plan, splits, NODES, true); checkPlanReducers(plan, NODES, expReducers, true); }
/** * Test HDFS splits with Replication == 3. * * @throws Exception If failed. */ @Test public void testHdfsSplitsReplication() throws Exception { IgfsMock igfs = LocationsBuilder.create().add(0, NODE_1).add(50, NODE_2).add(100, NODE_3).buildIgfs(); final List<HadoopInputSplit> splits = new ArrayList<>(); splits.add(new HadoopFileBlock(new String[] { HOST_1, HOST_2, HOST_3 }, URI.create("hfds://" + HOST_1 + "/x"), 0, 50)); splits.add(new HadoopFileBlock(new String[] { HOST_2, HOST_3, HOST_4 }, URI.create("hfds://" + HOST_2 + "/x"), 50, 100)); splits.add(new HadoopFileBlock(new String[] { HOST_3, HOST_4, HOST_5 }, URI.create("hfds://" + HOST_3 + "/x"), 100, 37)); // The following splits belong to hosts that are out of Ignite topology at all. // This means that these splits should be assigned to any least loaded modes: splits.add(new HadoopFileBlock(new String[] { HOST_4, HOST_5, HOST_1 }, URI.create("hfds://" + HOST_4 + "/x"), 138, 2)); splits.add(new HadoopFileBlock(new String[] { HOST_5, HOST_1, HOST_2 }, URI.create("hfds://" + HOST_5 + "/x"), 140, 3)); final int expReducers = 8; HadoopPlannerMockJob job = new HadoopPlannerMockJob(splits, expReducers); IgniteHadoopWeightedMapReducePlanner planner = createPlanner(igfs); final HadoopMapReducePlan plan = planner.preparePlan(job, NODES, null); checkPlanMappers(plan, splits, NODES, true); checkPlanReducers(plan, NODES, expReducers, true); }
/** * Test one IGFS split being assigned to affinity node. * * @throws Exception If failed. */ @Test public void testOneIgfsSplitAffinity() throws Exception { IgfsMock igfs = LocationsBuilder.create().add(0, NODE_1).add(50, NODE_2).add(100, NODE_3).buildIgfs(); List<HadoopInputSplit> splits = new ArrayList<>(); splits.add(new HadoopFileBlock(new String[] { HOST_1 }, URI.create("igfs://igfs@/file"), 0, 50)); final int expReducers = 4; HadoopPlannerMockJob job = new HadoopPlannerMockJob(splits, expReducers); IgniteHadoopWeightedMapReducePlanner planner = createPlanner(igfs); HadoopMapReducePlan plan = planner.preparePlan(job, NODES, null); assert plan.mappers() == 1; assert plan.mapperNodeIds().size() == 1; assert plan.mapperNodeIds().contains(ID_1); checkPlanMappers(plan, splits, NODES, false/*only 1 split*/); checkPlanReducers(plan, NODES, expReducers, false/* because of threshold behavior.*/); }
HadoopFileBlock fileBlock1 = new HadoopFileBlock(HOSTS, inFileUri, 0, l); HadoopFileBlock fileBlock2 = new HadoopFileBlock(HOSTS, inFileUri, l, fileLen - l);