public static Properties getConnParams(Collection<Map.Entry<String, String>> properties) { Properties connParams = new Properties(); for (Map.Entry<String, String> entry : properties) { if(PARAMETERS.containsKey(entry.getKey()) && PARAMETERS.get(entry.getKey()).type() == CONNECTION_PARAM) { Pair<String, String> keyValue = ((ConnectionParamAction)PARAMETERS.get(entry.getKey())).doAction(entry.getValue()); connParams.put(keyValue.getFirst(), keyValue.getSecond()); } } return connParams; }
Pair<String, String> doAction(String param) { return new Pair<>(sessionKey, param); } }
public List<Pair<Long, Long>> split(long firstSplitVolume, long splitVolume) { List<Pair<Long, Long>> splits = new ArrayList<>(); if (pages == null || pages.isEmpty()) { return splits; } int pageSize = pages.size(); long currentOffset = -1; long currentBytes = 0; long realSplitVolume = firstSplitVolume > 0 ? firstSplitVolume : splitVolume; for (Pair<Long, Integer> eachPage : pages) { if (currentOffset == -1) { currentOffset = eachPage.getFirst(); } if (currentBytes > 0 && currentBytes + eachPage.getSecond() >= realSplitVolume) { splits.add(new Pair(currentOffset, currentBytes)); currentOffset = eachPage.getFirst(); currentBytes = 0; realSplitVolume = splitVolume; } currentBytes += eachPage.getSecond(); } //add last if (currentBytes > 0) { splits.add(new Pair(currentOffset, currentBytes)); } return splits; } }
@Override public Column apply(@Nullable Pair<Integer, Column> columnPair) { return columnPair.getSecond(); } }));
public List<Pair<Long, Long>> split(long firstSplitVolume, long splitVolume) { List<Pair<Long, Long>> splits = new ArrayList<Pair<Long, Long>>(); if (pages == null || pages.isEmpty()) { return splits; } int pageSize = pages.size(); long currentOffset = -1; long currentBytes = 0; long realSplitVolume = firstSplitVolume > 0 ? firstSplitVolume : splitVolume; for (int i = 0; i < pageSize; i++) { Pair<Long, Integer> eachPage = pages.get(i); if (currentOffset == -1) { currentOffset = eachPage.getFirst(); } if (currentBytes > 0 && currentBytes + eachPage.getSecond() >= realSplitVolume) { splits.add(new Pair(currentOffset, currentBytes)); currentOffset = eachPage.getFirst(); currentBytes = 0; realSplitVolume = splitVolume; } currentBytes += eachPage.getSecond(); } //add last if (currentBytes > 0) { splits.add(new Pair(currentOffset, currentBytes)); } return splits; } }
@Override public Column apply(@Nullable Pair<Integer, Column> columnPair) { return columnPair.getSecond(); } })).build();
public static Map<String, String> getSessionVars(Collection<Map.Entry<String, String>> properties) { Map<String, String> sessionVars = new HashMap<>(); for (Map.Entry<String, String> entry : properties) { if(PARAMETERS.containsKey(entry.getKey()) && PARAMETERS.get(entry.getKey()).type() == SESSION_UPDATE) { Pair<String, String> keyValue = ((SessionAction)PARAMETERS.get(entry.getKey())).doAction(entry.getValue()); sessionVars.put(keyValue.getFirst(), keyValue.getSecond()); } } return sessionVars; } }
public Pair<String, String> doAction(String param) { return new Pair<>(connParamKey, param); } }
@Test public void testPage() { Task.IntermediateEntry interm = new Task.IntermediateEntry(-1, -1, 1, null); List<Pair<Long, Integer>> pages = new ArrayList<>(); pages.add(new Pair(0L, 1441275)); pages.add(new Pair(1441275L, 1447446)); pages.add(new Pair(2888721L, 1442507)); interm.setPages(pages); long splitBytes = 3 * 1024 * 1024; List<Pair<Long, Long>> splits = interm.split(splitBytes, splitBytes); assertEquals(2, splits.size()); long[][] expected = { {0, 1441275 + 1447446}, {1441275 + 1447446, 1442507} }; for (int i = 0; i < 2; i++) { Pair<Long, Long> eachSplit = splits.get(i); assertEquals(expected[i][0], eachSplit.getFirst().longValue()); assertEquals(expected[i][1], eachSplit.getSecond().longValue()); } } }
public static void scheduleFetchesByEvenDistributedVolumes(Stage stage, Map<Integer, FetchGroupMeta> partitions, String tableName, int num) { Map<String, List<FetchProto>>[] fetchsArray = makeEvenDistributedFetchImpl(partitions, tableName, num).getSecond(); // Schedule FetchImpls for (Map<String, List<FetchProto>> eachFetches : fetchsArray) { Stage.scheduleFetches(stage, eachFetches); } }
public static Map<String, String> getSessionVars(Collection<Map.Entry<String, String>> properties) { Map<String, String> sessionVars = new HashMap<>(); for (Map.Entry<String, String> entry : properties) { if(PARAMETERS.containsKey(entry.getKey()) && PARAMETERS.get(entry.getKey()).type() == SESSION_UPDATE) { Pair<String, String> keyValue = ((SessionAction)PARAMETERS.get(entry.getKey())).doAction(entry.getValue()); sessionVars.put(keyValue.getFirst(), keyValue.getSecond()); } } return sessionVars; } }
Pair<String, String> doAction(String param) { return new Pair<>(sessionKey, param); } }