@Override public Map<String, Object> parseToMap(String input) { try { final Object compiled = fn.apply(input); if (!(compiled instanceof Map)) { throw new ParseException("JavaScript parsed value must be in {key: value} format!"); } return (Map) compiled; } catch (Exception e) { throw new ParseException(e, "Unable to parse row [%s]", input); } }
private List<String> doAggregate( AggregatorFactory[] metrics, Aggregator[] aggs, ThreadLocal<InputRow> rowContainer, InputRow row ) { List<String> parseExceptionMessages = new ArrayList<>(); rowContainer.set(row); for (int i = 0; i < aggs.length; i++) { final Aggregator agg = aggs[i]; synchronized (agg) { try { agg.aggregate(); } catch (ParseException e) { // "aggregate" can throw ParseExceptions if a selector expects something but gets something else. log.debug(e, "Encountered parse error, skipping aggregator[%s].", metrics[i].getName()); parseExceptionMessages.add(e.getMessage()); } } } rowContainer.set(null); return parseExceptionMessages; }
return null; ParseException pe = new ParseException( "Found unparseable columns in row: [%s], exceptions: [%s]", row, stringBuilder.toString() ); pe.setFromPartiallyValidRow(true); return pe;
private void handleParseException(ParseException e) { if (e.isFromPartiallyValidRow()) { buildSegmentsMeters.incrementProcessedWithError(); } else { buildSegmentsMeters.incrementUnparseable(); } if (ingestionSchema.tuningConfig.isLogParseExceptions()) { log.error(e, "Encountered parse exception:"); } if (buildSegmentsSavedParseExceptions != null) { buildSegmentsSavedParseExceptions.add(e); } if (buildSegmentsMeters.getUnparseable() + buildSegmentsMeters.getProcessedWithError() > ingestionSchema.tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task...", e); } }
private void handleParseException(ParseException pe) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (spec.getTuningConfig().isLogParseExceptions()) { log.error(pe, "Encountered parse exception: "); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > spec.getTuningConfig().getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
public static void validateFields(Iterable<String> fieldNames) { Set<String> duplicates = findDuplicates(fieldNames); if (!duplicates.isEmpty()) { throw new ParseException("Duplicate column entries found : %s", duplicates.toString()); } }
return null; ParseException pe = new ParseException( "Found unparseable columns in row: [%s], exceptions: [%s]", row, stringBuilder.toString() ); pe.setFromPartiallyValidRow(true); return pe;
private void handleParseException(ParseException pe, ConsumerRecord<byte[], byte[]> record) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (tuningConfig.isLogParseExceptions()) { log.error( pe, "Encountered parse exception on row from partition[%d] offset[%d]", record.partition(), record.offset() ); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
parseExceptionMessages.add(pe.getMessage());
@Nullable public static Long convertObjectToLong(@Nullable Object valObj, boolean reportParseExceptions) { if (valObj == null) { return null; } if (valObj instanceof Long) { return (Long) valObj; } else if (valObj instanceof Number) { return ((Number) valObj).longValue(); } else if (valObj instanceof String) { Long ret = DimensionHandlerUtils.getExactLongFromDecimalString((String) valObj); if (reportParseExceptions && ret == null) { throw new ParseException("could not convert value [%s] to long", valObj); } return ret; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } }
private void handleParseException(ParseException pe, OrderedPartitionableRecord record) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (tuningConfig.isLogParseExceptions()) { log.error( pe, "Encountered parse exception on row from partition[%s] sequence[%s]", record.getPartitionId(), record.getSequenceNumber() ); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
parseExceptionMessages.add(pe.getMessage()); parseExceptionMessages.add(e.getMessage());
@Nullable public static Double convertObjectToDouble(@Nullable Object valObj, boolean reportParseExceptions) { if (valObj == null) { return null; } if (valObj instanceof Double) { return (Double) valObj; } else if (valObj instanceof Number) { return ((Number) valObj).doubleValue(); } else if (valObj instanceof String) { Double ret = Doubles.tryParse((String) valObj); if (reportParseExceptions && ret == null) { throw new ParseException("could not convert value [%s] to double", valObj); } return ret; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } }
private void handleParseException(ParseException pe, Context context) { context.getCounter(HadoopDruidIndexerConfig.IndexJobCounters.INVALID_ROW_COUNTER).increment(1); Counter unparseableCounter = context.getCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_UNPARSEABLE_COUNTER); Counter processedWithErrorsCounter = context.getCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_PROCESSED_WITH_ERRORS_COUNTER); if (pe.isFromPartiallyValidRow()) { processedWithErrorsCounter.increment(1); } else { unparseableCounter.increment(1); } if (config.isLogParseExceptions()) { log.error(pe, "Encountered parse exception: "); } long rowsUnparseable = unparseableCounter.getValue(); long rowsProcessedWithError = processedWithErrorsCounter.getValue(); if (rowsUnparseable + rowsProcessedWithError > config.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task...", pe); } }
Assert.assertEquals( "Found unparseable columns in row: [MapBasedInputRow{timestamp=1970-01-01T00:00:00.000Z, event={string=A, float=19.0, long=asdj, double=21.0}, dimensions=[string, float, long, double]}], exceptions: [could not convert value [asdj] to long,]", result.getParseException().getMessage() ); Assert.assertEquals( "Found unparseable columns in row: [MapBasedInputRow{timestamp=1970-01-01T00:00:00.000Z, event={string=A, float=aaa, long=20, double=21.0}, dimensions=[string, float, long, double]}], exceptions: [could not convert value [aaa] to float,]", result.getParseException().getMessage() ); Assert.assertEquals( "Found unparseable columns in row: [MapBasedInputRow{timestamp=1970-01-01T00:00:00.000Z, event={string=A, float=19.0, long=20, double=}, dimensions=[string, float, long, double]}], exceptions: [could not convert value [] to double,]", result.getParseException().getMessage() );
@Nullable public static Float convertObjectToFloat(@Nullable Object valObj, boolean reportParseExceptions) { if (valObj == null) { return null; } if (valObj instanceof Float) { return (Float) valObj; } else if (valObj instanceof Number) { return ((Number) valObj).floatValue(); } else if (valObj instanceof String) { Float ret = Floats.tryParse((String) valObj); if (reportParseExceptions && ret == null) { throw new ParseException("could not convert value [%s] to float", valObj); } return ret; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } }
private void handleParseException(ParseException e) { if (e.isFromPartiallyValidRow()) { buildSegmentsMeters.incrementProcessedWithError(); } else { buildSegmentsMeters.incrementUnparseable(); } if (ingestionSchema.tuningConfig.isLogParseExceptions()) { log.error(e, "Encountered parse exception:"); } if (buildSegmentsSavedParseExceptions != null) { buildSegmentsSavedParseExceptions.add(e); } if (buildSegmentsMeters.getUnparseable() + buildSegmentsMeters.getProcessedWithError() > ingestionSchema.tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task...", e); } }
private List<String> doAggregate( AggregatorFactory[] metrics, Aggregator[] aggs, ThreadLocal<InputRow> rowContainer, InputRow row ) { List<String> parseExceptionMessages = new ArrayList<>(); rowContainer.set(row); for (int i = 0; i < aggs.length; i++) { final Aggregator agg = aggs[i]; synchronized (agg) { try { agg.aggregate(); } catch (ParseException e) { // "aggregate" can throw ParseExceptions if a selector expects something but gets something else. log.debug(e, "Encountered parse error, skipping aggregator[%s].", metrics[i].getName()); parseExceptionMessages.add(e.getMessage()); } } } rowContainer.set(null); return parseExceptionMessages; }
/** * @param input JSON string. The root must be a JSON object, not an array. * e.g., {"valid": "true"} and {"valid":[1,2,3]} are supported * but [{"invalid": "true"}] and [1,2,3] are not. * * @return A map of field names and values */ @Override public Map<String, Object> parseToMap(String input) { try { JsonNode document = mapper.readValue(input, JsonNode.class); return flattener.flatten(document); } catch (Exception e) { throw new ParseException(e, "Unable to parse row [%s]", input); } } }
private void handleParseException(ParseException pe) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (spec.getTuningConfig().isLogParseExceptions()) { log.error(pe, "Encountered parse exception: "); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > spec.getTuningConfig().getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }