private boolean ncEq(FileSplit fs1, FileSplit fs2) { return fs1.getNodeName().equals(fs2.getNodeName()); }
@Override public String toString() { return fileSplit.getNodeName() + ":" + fileSplit.getPath(); }
static void createPartitionConstraint(JobSpecification spec, IOperatorDescriptor op, FileSplit[] splits) { String[] parts = new String[splits.length]; for (int i = 0; i < splits.length; ++i) { parts[i] = splits[i].getNodeName(); } PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, op, parts); } }
private static void createPartitionConstraint(JobSpecification spec, IOperatorDescriptor op, FileSplit[] splits) { String[] parts = new String[splits.length]; for (int i = 0; i < splits.length; ++i) { parts[i] = splits[i].getNodeName(); } PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, op, parts); } }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("FileSplitDomain["); boolean fst = true; for (FileSplit fs : splits) { if (fst) { fst = false; } else { sb.append(", "); } sb.append(fs.getNodeName() + ":" + fs.getPath()); } sb.append(']'); return sb.toString(); }
public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraints( FileSplit[] splits) { IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits); String[] loc = new String[splits.length]; for (int p = 0; p < splits.length; p++) { loc[p] = splits[p].getNodeName(); } AlgebricksPartitionConstraint pc = new AlgebricksAbsolutePartitionConstraint(loc); return new Pair<>(splitProvider, pc); }
private void formResponseObject(ObjectNode jsonResponse, FileSplit[] fileSplits, ARecordType recordType, String primaryKeys, Map<String, NodeControllerInfo> nodeMap) { ArrayNode partititons = OBJECT_MAPPER.createArrayNode(); // Adds a primary key. jsonResponse.put("keys", primaryKeys); // Adds record type. jsonResponse.set("type", recordType.toJSON()); // Generates file partitions. for (FileSplit split : fileSplits) { String ipAddress = nodeMap.get(split.getNodeName()).getNetworkAddress().getAddress(); String path = split.getPath(); FilePartition partition = new FilePartition(ipAddress, path); partititons.add(partition.toObjectNode()); } // Generates the response object which contains the splits. jsonResponse.set("splits", partititons); } }
public static String getIndexPath(AsterixHyracksIntegrationUtil integrationUtil, Dataset dataset, String nodeId) throws Exception { final FileSplit[] datasetSplits = TestDataUtil.getDatasetSplits(integrationUtil, dataset); final Optional<FileSplit> nodeFileSplit = Arrays.stream(datasetSplits).filter(s -> s.getNodeName().equals(nodeId)).findFirst(); Assert.assertTrue(nodeFileSplit.isPresent()); return nodeFileSplit.get().getPath(); } }
@Override public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink, int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc) { FileSplitDataSink fsds = (FileSplitDataSink) sink; FileSplitSinkId fssi = fsds.getId(); FileSplit fs = fssi.getFileSplit(); File outFile = new File(fs.getPath()); String nodeId = fs.getNodeName(); SinkWriterRuntimeFactory runtime = new SinkWriterRuntimeFactory(printColumns, printerFactories, outFile, getWriterFactory(), inputDesc); AlgebricksPartitionConstraint apc = new AlgebricksAbsolutePartitionConstraint(new String[] { nodeId }); return new Pair<>(runtime, apc); }