private CounterGroupKey getCounterGroup(String groupName) throws JobCounterException { if (dictionary == null) { dictionary = CounterGroupDictionary.getInstance(); } final CounterGroupKey groupKey = dictionary.getCounterGroupByName(groupName); if (groupKey == null) { throw new JobCounterException("Invalid counter group name: " + groupName); } return groupKey; }
private CounterGroupKey getCounterGroup(int groupIndex) throws JobCounterException { if (dictionary == null) { dictionary = CounterGroupDictionary.getInstance(); } return dictionary.getCounterGroupByIndex(groupIndex); }
private long getTotalHdfsOps(JobCounters counter) { long mapHdfsReadOps = counter.getCounterValue(JobCounters.CounterName.HDFS_READ_OPS); long mapHdfsWriteOps = counter.getCounterValue(JobCounters.CounterName.HDFS_WRITE_OPS); return mapHdfsReadOps + mapHdfsWriteOps; }
@Override public JobCounters deserialize(byte[] bytes) { JobCounters counters = new JobCounters(); final int length = bytes.length; if (length < 4) { final Map<String, Map<String, Long>> groupMap = counters.getCounters(); int pos = 0; final int totalGroups = Bytes.toInt(bytes, pos); final int nextGroupPos = pos + (totalCounters * 12); try { final CounterGroupKey groupKey = getCounterGroup(groupIndex); if (groupKey == null) { throw new JobCounterException("Group index " + groupIndex + " is not defined"); groupMap.put(groupKey.getName(), counterMap); for (int j = 0; j < totalCounters; ++j) { final int counterIndex = Bytes.toInt(bytes, pos); final long value = Bytes.toLong(bytes, pos); pos += 8; final CounterKey counterKey = groupKey.getCounterKeyByID(counterIndex); if (counterKey == null) { continue; counterMap.put(counterKey.getNames().get(0), value);
private int serializeGroup(byte[] buffer, int currentPos, String groupName, Map<String, Long> counterMap) throws JobCounterException { int pos = currentPos; final CounterGroupKey groupKey = getCounterGroup(groupName); if (groupKey == null) { throw new JobCounterException("Group name " + groupName + " is not defined"); } Bytes.putInt(buffer, pos, groupKey.getIndex()); pos += 4; int totalCounterNumberPos = pos; pos += 4; int totalCounters = 0; for (Map.Entry<String, Long> entry : counterMap.entrySet()) { final String counterName = entry.getKey(); final CounterKey counterKey = groupKey.getCounterKeyByName(counterName); if (counterKey == null) { continue; } final Long counterValue = entry.getValue(); Bytes.putInt(buffer, pos, counterKey.getIndex()); pos += 4; Bytes.putLong(buffer, pos, counterValue); pos += 8; ++totalCounters; } Bytes.putInt(buffer, totalCounterNumberPos, totalCounters); return pos; }
@Override public byte[] serialize(JobCounters counters) { final Map<String, Map<String, Long>> groupMap = counters.getCounters(); int totalSize = 4; for (Map<String, Long> counterMap : groupMap.values()) { final Map<String, Long> counterMap = entry.getValue(); try { nextGroupPos = pos = serializeGroup(buffer, pos, groupName, counterMap); ++totalGroups; } catch (JobCounterException ex) {
private boolean parseGroup(int groupIndex, Properties prop) { final String groupKeyBase = "counter.group" + groupIndex; final String groupNameKey = groupKeyBase + ".name"; final String groupName = prop.getProperty(groupNameKey); if (groupName == null) { return false; } final String groupDescriptionKey = groupKeyBase + ".description"; final String groupDescription = prop.getProperty(groupDescriptionKey); final CounterGroupKeyImpl groupKey = new CounterGroupKeyImpl(groupIndex, groupName, groupDescription); final ArrayList<CounterKey> counters = new ArrayList<CounterKey>(); int counterIndex = 0; while (parseCounter(groupKey, counterIndex, counters, prop)) { ++counterIndex; } groupKey.setCounterKeys(counters.toArray(new CounterKey[counters.size()])); groupKeys.add(groupKey); return true; }
public MRTaskExecutionResponse.TaskDistributionResponse getHistoryTaskDistribution(List<org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity> tasks, String counterName, String distRange) { MRTaskExecutionResponse.TaskDistributionResponse response = new MRTaskExecutionResponse.TaskDistributionResponse(); response.counterName = counterName; List<Long> distRangeList = ResourceUtils.parseDistributionList(distRange); for (int i = 0; i < distRangeList.size(); i++) { response.taskBuckets.add(new MRTaskExecutionResponse.CountUnit(distRangeList.get(i))); } JobCounters.CounterName jobCounterName = JobCounters.CounterName.valueOf(counterName.toUpperCase()); for (org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity task : tasks) { Long counterValue = task.getJobCounters().getCounterValue(jobCounterName); int pos = ResourceUtils.getDistributionPosition(distRangeList, counterValue); response.taskBuckets.get(pos).countVal++; } return response; }
public static CounterGroupDictionary getInstance() throws JobCounterException { if (instance == null) { synchronized (CounterGroupDictionary.class) { if (instance == null) { CounterGroupDictionary tmp = new CounterGroupDictionary(); tmp.initialize(); instance = tmp; } } } return instance; }
private boolean parseCounter(CounterGroupKey groupKey, int counterIndex, List<CounterKey> counters, Properties prop) { final String counterKeyBase = "counter.group" + groupKey.getIndex() + ".counter" + counterIndex; final String counterNameKey = counterKeyBase + ".names"; final String counterNamesString = prop.getProperty(counterNameKey); if (counterNamesString == null) { return false; } final String[] names = counterNamesString.split(","); final List<String> counterNames = new ArrayList<String>(); for (String name : names) { counterNames.add(name.trim()); } final String counterDescriptionKey = counterKeyBase + ".description"; final String counterDescription = prop.getProperty(counterDescriptionKey); CounterKey counter = new CounterKeyImpl(counterIndex, counterNames, counterDescription, groupKey); counters.add(counter); return true; }
public JPMEntityRepository() { entitySet.add(JobExecutionAPIEntity.class); entitySet.add(TaskExecutionAPIEntity.class); entitySet.add(TaskAttemptExecutionAPIEntity.class); entitySet.add(YarnAppAPIEntity.class); serDeserMap.put(JobConfig.class, new JobConfigSerDeser()); serDeserMap.put(JobCounters.class, new JobCountersSerDeser()); } }
private List<MRTaskExecutionResponse.SuggestionResult> getGCsuggest(double smallerRatio, double largerRatio) { if (smallerRatio <= 0) { smallerRatio = 1; } double deviation = largerRatio / smallerRatio; String suggestName = String.format(GC_RATIO_NAME_FORMAT, JobCounters.CounterName.GC_MILLISECONDS.getName(), JobCounters.CounterName.CPU_MILLISECONDS.getName()); String suggestion = null; if (deviation > threshold) { suggestion = String.format(GC_SUGGESTION_FORMAT, threshold, largerRatio, smallerRatio); } List<MRTaskExecutionResponse.SuggestionResult> suggestionResults = new ArrayList<>(); suggestionResults.add(new MRTaskExecutionResponse.SuggestionResult(suggestName, deviation, suggestion)); return suggestionResults; }
public String getGroupName() { return group.name(); } }
public CounterGroupKey getCounterGroupByName(String groupName) { for (CounterGroupKey groupKey : groupKeys) { if (groupKey.getName().equalsIgnoreCase(groupName)) { return groupKey; } } return null; }
@Override public CounterKey getCounterKeyByName(String name) { for (CounterKey counterKey : counterKeys) { for (String n : counterKey.getNames()) { if (n.equalsIgnoreCase(name)) { return counterKey; } } } return null; }
/** * The default index size is 16. * * @param attempt * @return minimal sort memory */ private long getMinimumIOSortMemory(TaskAttemptExecutionAPIEntity attempt) { long records = attempt.getJobCounters().getCounterValue(MAP_OUTPUT_RECORDS); long outputBytes = attempt.getJobCounters().getCounterValue(MAP_OUTPUT_BYTES); return outputBytes + records * 16; }
public JPAEntityRepository() { serDeserMap.put(JobCounters.class, new JobCountersSerDeser()); serDeserMap.put(JobConfig.class, new JobConfigSerDeser()); entitySet.add(JobConfigurationAPIEntity.class); entitySet.add(JobEventAPIEntity.class); entitySet.add(JobExecutionAPIEntity.class); entitySet.add(JobErrorCategoryMappingAPIEntity.class); entitySet.add(TaskAttemptExecutionAPIEntity.class); entitySet.add(TaskExecutionAPIEntity.class); entitySet.add(TaskFailureCountAPIEntity.class); entitySet.add(TaskAttemptCounterAPIEntity.class); entitySet.add(JobProcessTimeStampEntity.class); entitySet.add(JobCountEntity.class); entitySet.add(TaskAttemptErrorCategoryEntity.class); entitySet.add(JobSuggestionAPIEntity.class); entitySet.add(JobRpcAnalysisAPIEntity.class); } }
private List<MRTaskExecutionResponse.SuggestionResult> getDeviationSuggest(double avgSmaller, double avgLarger) { if (avgSmaller <= 0) { avgSmaller = 1; } double deviation = avgLarger / avgSmaller; String suggestName = String.format(DEVIATION_SUGGEST_FORMAT, counterName.getName()); String suggestion = null; if (deviation > threshold) { suggestion = String.format(DATA_SKEW_SUGGESTION_FORMAT, counterName.getName(), threshold, avgLarger, avgSmaller); } List<MRTaskExecutionResponse.SuggestionResult> suggestionResults = new ArrayList<>(); suggestionResults.add(new MRTaskExecutionResponse.SuggestionResult(suggestName, deviation, suggestion)); return suggestionResults; } }
private double getAverageSpillBytes(List<TaskExecutionAPIEntity> tasks) { if (tasks.isEmpty()) { return 0; } long totalSpillBytes = 0; for (TaskExecutionAPIEntity task : tasks) { totalSpillBytes += task.getJobCounters().getCounterValue(JobCounters.CounterName.SPLIT_RAW_BYTES); } return totalSpillBytes / tasks.size(); }
public static double[] getCounterValues(List<TaskExecutionAPIEntity> tasks, JobCounters.CounterName counterName) { List<Double> values = new ArrayList<>(); for (TaskExecutionAPIEntity task : tasks) { values.add(Double.valueOf(task.getJobCounters().getCounterValue(counterName))); } return toArray(values); }