public TaskLogProcessor(JobConf conf) { query = HiveConf.getQueryString(conf); heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats()); heuristics.put(new MapAggrMemErrorHeuristic(), new HeuristicStats()); heuristics.put(new DataCorruptErrorHeuristic(), new HeuristicStats()); for(ErrorHeuristic e : heuristics.keySet()) { e.init(query, conf); } }
public DataCorruptErrorHeuristic() { setQueryRegex(".*"); getLogRegexes().add(SPLIT_REGEX); getLogRegexes().add(EXCEPTION_REGEX); }
@Override public ErrorAndSolution getErrorAndSolution() { ErrorAndSolution es = null; if(getQueryMatches()) { Map<String, List<String>> rll = getRegexToLogLines(); if (rll.get(EXCEPTION_REGEX).size() > 0 && rll.get(SPLIT_REGEX).size() > 0) { // There should only be a single split line... String splitLogLine = rll.get(SPLIT_REGEX).get(0); // Extract only 'split: hdfs://...' Pattern p = Pattern.compile(SPLIT_REGEX, Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(splitLogLine); m.find(); String splitStr = m.group(1); es = new ErrorAndSolution( "Data file " + splitStr + " is corrupted.", "Replace file. i.e. by re-running the query that produced the " + "source table / partition."); } } reset(); return es; } }
@Override public ErrorAndSolution getErrorAndSolution() { ErrorAndSolution es = null; if(getQueryMatches()) { Map<String, List<String>> rll = getRegexToLogLines(); if (rll.get(EXCEPTION_REGEX).size() > 0 && rll.get(SPLIT_REGEX).size() > 0) { // There should only be a single split line... String splitLogLine = rll.get(SPLIT_REGEX).get(0); // Extract only 'split: hdfs://...' Pattern p = Pattern.compile(SPLIT_REGEX, Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(splitLogLine); m.find(); String splitStr = m.group(1); es = new ErrorAndSolution( "Data file " + splitStr + " is corrupted.", "Replace file. i.e. by re-running the query that produced the " + "source table / partition."); } } reset(); return es; } }
public DataCorruptErrorHeuristic() { setQueryRegex(".*"); getLogRegexes().add(SPLIT_REGEX); getLogRegexes().add(EXCEPTION_REGEX); }
public TaskLogProcessor(JobConf conf) { query = HiveConf.getQueryString(conf); heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats()); heuristics.put(new MapAggrMemErrorHeuristic(), new HeuristicStats()); heuristics.put(new DataCorruptErrorHeuristic(), new HeuristicStats()); for(ErrorHeuristic e : heuristics.keySet()) { e.init(query, conf); } }
@Override public ErrorAndSolution getErrorAndSolution() { ErrorAndSolution es = null; if(getQueryMatches()) { Map<String, List<String>> rll = getRegexToLogLines(); if (rll.get(EXCEPTION_REGEX).size() > 0 && rll.get(SPLIT_REGEX).size() > 0) { // There should only be a single split line... String splitLogLine = rll.get(SPLIT_REGEX).get(0); // Extract only 'split: hdfs://...' Pattern p = Pattern.compile(SPLIT_REGEX, Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(splitLogLine); m.find(); String splitStr = m.group(1); es = new ErrorAndSolution( "Data file " + splitStr + " is corrupted.", "Replace file. i.e. by re-running the query that produced the " + "source table / partition."); } } reset(); return es; } }
public DataCorruptErrorHeuristic() { setQueryRegex(".*"); getLogRegexes().add(SPLIT_REGEX); getLogRegexes().add(EXCEPTION_REGEX); }
public TaskLogProcessor(JobConf conf) { query = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING); heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats()); heuristics.put(new MapAggrMemErrorHeuristic(), new HeuristicStats()); heuristics.put(new DataCorruptErrorHeuristic(), new HeuristicStats()); for(ErrorHeuristic e : heuristics.keySet()) { e.init(query, conf); } }
@Override public ErrorAndSolution getErrorAndSolution() { ErrorAndSolution es = null; if(getQueryMatches()) { Map<String, List<String>> rll = getRegexToLogLines(); if (rll.get(EXCEPTION_REGEX).size() > 0 && rll.get(SPLIT_REGEX).size() > 0) { // There should only be a single split line... assert(rll.get(SPLIT_REGEX).size()==1); String splitLogLine = rll.get(SPLIT_REGEX).get(0); // Extract only 'split: hdfs://...' Pattern p = Pattern.compile(SPLIT_REGEX, Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(splitLogLine); m.find(); String splitStr = m.group(); es = new ErrorAndSolution( "Data file " + splitStr + " is corrupted.", "Replace file. i.e. by re-running the query that produced the " + "source table / partition."); } } reset(); return es; } }
public DataCorruptErrorHeuristic() { setQueryRegex(".*"); getLogRegexes().add(SPLIT_REGEX); getLogRegexes().add(EXCEPTION_REGEX); }
public TaskLogProcessor(JobConf conf) { this.conf = conf; query = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING); heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats()); heuristics.put(new MapAggrMemErrorHeuristic(), new HeuristicStats()); heuristics.put(new DataCorruptErrorHeuristic(), new HeuristicStats()); for(ErrorHeuristic e : heuristics.keySet()) { e.init(query, conf); } }