/** * Add a {@link DataNode} to the {@link FlowGraph}. The method uses the {@link FlowGraphConfigurationKeys#DATA_NODE_CLASS} config * to instantiate a {@link DataNode} from the node config file. * @param change */ private void addDataNode(DiffEntry change) { if (checkFilePath(change.getNewPath(), NODE_FILE_DEPTH)) { Path nodeFilePath = new Path(this.repositoryDir, change.getNewPath()); try { Config config = loadNodeFileWithOverrides(nodeFilePath); Class dataNodeClass = Class.forName(ConfigUtils.getString(config, FlowGraphConfigurationKeys.DATA_NODE_CLASS, FlowGraphConfigurationKeys.DEFAULT_DATA_NODE_CLASS)); DataNode dataNode = (DataNode) GobblinConstructorUtils.invokeLongestConstructor(dataNodeClass, config); if (!this.flowGraph.addDataNode(dataNode)) { log.warn("Could not add DataNode {} to FlowGraph; skipping", dataNode.getId()); } else { log.info("Added Datanode {} to FlowGraph", dataNode.getId()); } } catch (Exception e) { log.warn("Could not add DataNode defined in {} due to exception {}", change.getNewPath(), e.getMessage()); } } }
/** * Sort the changes in a commit so that changes to node files appear before changes to edge files. This is done so that * node related changes are applied to the FlowGraph before edge related changes. An example where the order matters * is the case when a commit adds a new node n2 as well as adds an edge from an existing node n1 to n2. To ensure that the * addition of edge n1->n2 is successful, node n2 must exist in the graph and so needs to be added first. For deletions, * the order does not matter and ordering the changes in the commit will result in the same FlowGraph state as if the changes * were unordered. In other words, deletion of a node deletes all its incident edges from the FlowGraph. So processing an * edge deletion later results in a no-op. Note that node and edge files do not change depth in case of modifications. * * If there are multiple commits between successive polls to Git, the re-ordering of changes across commits should not * affect the final state of the FlowGraph. This is because, the order of changes for a given file type (i.e. node or edge) * is preserved. */ @Override void processGitConfigChanges() throws GitAPIException, IOException { List<DiffEntry> changes = this.gitRepo.getChanges(); Collections.sort(changes, (o1, o2) -> { Integer o1Depth = (o1.getNewPath() != null) ? (new Path(o1.getNewPath())).depth() : (new Path(o1.getOldPath())).depth(); Integer o2Depth = (o2.getNewPath() != null) ? (new Path(o2.getNewPath())).depth() : (new Path(o2.getOldPath())).depth(); return o1Depth.compareTo(o2Depth); }); processGitConfigChangesHelper(changes); //Decrements the latch count. The countdown latch is initialized to 1. So after the first time the latch is decremented, // the following operation should be a no-op. this.initComplete.countDown(); }
/** * Add a {@link FlowEdge} to the {@link FlowGraph}. The method uses the {@link FlowEdgeFactory} instance * provided by the {@link FlowGraph} to build a {@link FlowEdge} from the edge config file. * @param change */ private void addFlowEdge(DiffEntry change) { if (checkFilePath(change.getNewPath(), EDGE_FILE_DEPTH)) { Path edgeFilePath = new Path(this.repositoryDir, change.getNewPath()); try { Config edgeConfig = loadEdgeFileWithOverrides(edgeFilePath); List<SpecExecutor> specExecutors = getSpecExecutors(edgeConfig); Class flowEdgeFactoryClass = Class.forName(ConfigUtils.getString(edgeConfig, FlowGraphConfigurationKeys.FLOW_EDGE_FACTORY_CLASS, FlowGraphConfigurationKeys.DEFAULT_FLOW_EDGE_FACTORY_CLASS)); FlowEdgeFactory flowEdgeFactory = (FlowEdgeFactory) GobblinConstructorUtils.invokeLongestConstructor(flowEdgeFactoryClass, edgeConfig); if (flowCatalog.isPresent()) { FlowEdge edge = flowEdgeFactory.createFlowEdge(edgeConfig, flowCatalog.get(), specExecutors); if (!this.flowGraph.addFlowEdge(edge)) { log.warn("Could not add edge {} to FlowGraph; skipping", edge.getId()); } else { log.info("Added edge {} to FlowGraph", edge.getId()); } } else { log.warn("Could not add edge defined in {} to FlowGraph as FlowCatalog is absent", change.getNewPath()); } } catch (Exception e) { log.warn("Could not add edge defined in {} due to exception {}", change.getNewPath(), e.getMessage()); } } }
/** * Add an element (i.e., a {@link DataNode}, or a {@link FlowEdge} to * the {@link FlowGraph} for an added, updated or modified node or edge file. * @param change */ @Override public void addChange(DiffEntry change) { Path path = new Path(change.getNewPath()); if (path.depth() == NODE_FILE_DEPTH) { addDataNode(change); } else if (path.depth() == EDGE_FILE_DEPTH) { addFlowEdge(change); } }
/** * Add a {@link FlowSpec} for an added, updated, or modified flow config * @param change */ @Override public void addChange(DiffEntry change) { if (checkConfigFilePath(change.getNewPath())) { Path configFilePath = new Path(this.repositoryDir, change.getNewPath()); try { Config flowConfig = loadConfigFileWithFlowNameOverrides(configFilePath); this.flowCatalog.put(FlowSpec.builder() .withConfig(flowConfig) .withVersion(SPEC_VERSION) .withDescription(SPEC_DESCRIPTION) .build()); } catch (IOException e) { log.warn("Could not load config file: " + configFilePath); } } }
/** * Get the path associated with this file. * * @param side * which path to obtain. * @return name for this file. */ public String getPath(Side side) { return side == Side.OLD ? getOldPath() : getNewPath(); }
private boolean isAdd(List<DiffEntry> files) { String oldPath = ((FollowFilter) pathFilter).getPath(); for (DiffEntry ent : files) { if (ent.getChangeType() == ADD && ent.getNewPath().equals(oldPath)) return true; } return false; }
private List<DiffEntry> updateFollowFilter(List<DiffEntry> files) { String oldPath = ((FollowFilter) pathFilter).getPath(); for (DiffEntry ent : files) { if (isRename(ent) && ent.getNewPath().equals(oldPath)) { pathFilter = FollowFilter.create(ent.getOldPath(), diffCfg); return Collections.singletonList(ent); } } return Collections.emptyList(); }
private static void listDiff(Repository repository, Git git, String oldCommit, String newCommit) throws GitAPIException, IOException { final List<DiffEntry> diffs = git.diff() .setOldTree(prepareTreeParser(repository, oldCommit)) .setNewTree(prepareTreeParser(repository, newCommit)) .call(); System.out.println("Found: " + diffs.size() + " differences"); for (DiffEntry diff : diffs) { System.out.println("Diff: " + diff.getChangeType() + ": " + (diff.getOldPath().equals(diff.getNewPath()) ? diff.getNewPath() : diff.getOldPath() + " -> " + diff.getNewPath())); } }
private static void listDiff(Repository repository, Git git, String oldCommit, String newCommit) throws GitAPIException, IOException { final List<DiffEntry> diffs = git.diff() .setOldTree(prepareTreeParser(repository, oldCommit)) .setNewTree(prepareTreeParser(repository, newCommit)) .call(); System.out.println("Found: " + diffs.size() + " differences"); for (DiffEntry diff : diffs) { System.out.println("Diff: " + diff.getChangeType() + ": " + (diff.getOldPath().equals(diff.getNewPath()) ? diff.getNewPath() : diff.getOldPath() + " -> " + diff.getNewPath())); } }
public static ArrayMemory valueOf(DiffEntry value) { ArrayMemory memory = new ArrayMemory(); memory.refOfIndex("oldId").assign(value.getOldId() == null ? Memory.NULL : valueOf(value.getOldId().toObjectId())); memory.refOfIndex("oldPath").assign(value.getOldPath()); memory.refOfIndex("oldMode").assign(value.getOldMode() == null ? Memory.NULL : StringMemory.valueOf(value.getOldMode().toString())); memory.refOfIndex("newId").assign(value.getNewId() == null ? Memory.NULL : valueOf(value.getNewId().toObjectId())); memory.refOfIndex("newPath").assign(value.getNewPath()); memory.refOfIndex("newMode").assign(value.getNewMode() == null ? Memory.NULL : StringMemory.valueOf(value.getNewMode().toString())); memory.refOfIndex("score").assign(value.getScore()); memory.refOfIndex("changeType").assign(value.getChangeType().name()); return memory; }
private void formatOldNewPaths(ByteArrayOutputStream o, DiffEntry ent) throws IOException { if (ent.oldId.equals(ent.newId)) return; final String oldp; final String newp; switch (ent.getChangeType()) { case ADD: oldp = DiffEntry.DEV_NULL; newp = quotePath(newPrefix + ent.getNewPath()); break; case DELETE: oldp = quotePath(oldPrefix + ent.getOldPath()); newp = DiffEntry.DEV_NULL; break; default: oldp = quotePath(oldPrefix + ent.getOldPath()); newp = quotePath(newPrefix + ent.getNewPath()); break; } o.write(encode("--- " + oldp + "\n")); //$NON-NLS-1$ //$NON-NLS-2$ o.write(encode("+++ " + newp + "\n")); //$NON-NLS-1$ //$NON-NLS-2$ }
final ChangeType type = ent.getChangeType(); final String oldp = ent.getOldPath(); final String newp = ent.getNewPath(); final FileMode oldMode = ent.getOldMode(); final FileMode newMode = ent.getNewMode();
private DiffEntry findRename(RevCommit parent, RevCommit commit, PathFilter path) throws IOException { if (renameDetector == null) return null; treeWalk.setFilter(TreeFilter.ANY_DIFF); treeWalk.reset(parent.getTree(), commit.getTree()); renameDetector.reset(); renameDetector.addAll(DiffEntry.scan(treeWalk)); for (DiffEntry ent : renameDetector.compute()) { if (isRename(ent) && ent.getNewPath().equals(path.getPath())) return ent; } return null; }
/** * Breaks apart a DiffEntry into two entries, one DELETE and one ADD. * * @param entry * the DiffEntry to break apart. * @return a list containing two entries. Calling {@link #getChangeType()} * on the first entry will return ChangeType.DELETE. Calling it on * the second entry will return ChangeType.ADD. */ static List<DiffEntry> breakModify(DiffEntry entry) { DiffEntry del = new DiffEntry(); del.oldId = entry.getOldId(); del.oldMode = entry.getOldMode(); del.oldPath = entry.getOldPath(); del.newId = A_ZERO; del.newMode = FileMode.MISSING; del.newPath = DiffEntry.DEV_NULL; del.changeType = ChangeType.DELETE; del.diffAttribute = entry.diffAttribute; DiffEntry add = new DiffEntry(); add.oldId = A_ZERO; add.oldMode = FileMode.MISSING; add.oldPath = DiffEntry.DEV_NULL; add.newId = entry.getNewId(); add.newMode = entry.getNewMode(); add.newPath = entry.getNewPath(); add.changeType = ChangeType.ADD; add.diffAttribute = entry.diffAttribute; return Arrays.asList(del, add); }
private void updateFollowFilter(ObjectId[] trees, DiffConfig cfg) throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, IOException { TreeWalk tw = pathFilter; FollowFilter oldFilter = (FollowFilter) tw.getFilter(); tw.setFilter(TreeFilter.ANY_DIFF); tw.reset(trees); List<DiffEntry> files = DiffEntry.scan(tw); RenameDetector rd = new RenameDetector(tw.getObjectReader(), cfg); rd.addAll(files); files = rd.compute(); TreeFilter newFilter = oldFilter; for (DiffEntry ent : files) { if (isRename(ent) && ent.getNewPath().equals(oldFilter.getPath())) { newFilter = FollowFilter.create(ent.getOldPath(), cfg); RenameCallback callback = oldFilter.getRenameCallback(); if (callback != null) { callback.renamed(ent); // forward the callback to the new follow filter ((FollowFilter) newFilter).setRenameCallback(callback); } break; } } tw.setFilter(newFilter); }
@JavascriptInterface public String getNewPath(int index) { DiffEntry diff = mDiffEntries.get(index); String np = diff.getNewPath(); return np; }
/** * Get the path associated with this file. * * @param side * which path to obtain. * @return name for this file. */ public String getPath(Side side) { return side == Side.OLD ? getOldPath() : getNewPath(); }
private boolean isAdd(List<DiffEntry> files) { String oldPath = ((FollowFilter) pathFilter).getPath(); for (DiffEntry ent : files) { if (ent.getChangeType() == ADD && ent.getNewPath().equals(oldPath)) return true; } return false; }
TreeWalk tw = new TreeWalk(repository); tw.setRecursive(true); tw.addTree(CommitUtils.getHead(repository).getTree()); tw.addTree(new FileTreeIterator(repository)); RenameDetector rd = new RenameDetector(repository); rd.addAll(DiffEntry.scan(tw)); List<DiffEntry> lde = rd.compute(tw.getObjectReader(), null); for (DiffEntry de : lde) { if (de.getScore() >= rd.getRenameScore()) { System.out.println("file: " + de.getOldPath() + " copied/moved to: " + de.getNewPath()); } }