private void handleParseException(ParseException e) { if (e.isFromPartiallyValidRow()) { buildSegmentsMeters.incrementProcessedWithError(); } else { buildSegmentsMeters.incrementUnparseable(); } if (ingestionSchema.tuningConfig.isLogParseExceptions()) { log.error(e, "Encountered parse exception:"); } if (buildSegmentsSavedParseExceptions != null) { buildSegmentsSavedParseExceptions.add(e); } if (buildSegmentsMeters.getUnparseable() + buildSegmentsMeters.getProcessedWithError() > ingestionSchema.tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task...", e); } }
private void handleParseException(ParseException pe) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (spec.getTuningConfig().isLogParseExceptions()) { log.error(pe, "Encountered parse exception: "); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > spec.getTuningConfig().getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
private void handleParseException(ParseException pe, Context context) { context.getCounter(HadoopDruidIndexerConfig.IndexJobCounters.INVALID_ROW_COUNTER).increment(1); Counter unparseableCounter = context.getCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_UNPARSEABLE_COUNTER); Counter processedWithErrorsCounter = context.getCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_PROCESSED_WITH_ERRORS_COUNTER); if (pe.isFromPartiallyValidRow()) { processedWithErrorsCounter.increment(1); } else { unparseableCounter.increment(1); } if (config.isLogParseExceptions()) { log.error(pe, "Encountered parse exception: "); } long rowsUnparseable = unparseableCounter.getValue(); long rowsProcessedWithError = processedWithErrorsCounter.getValue(); if (rowsUnparseable + rowsProcessedWithError > config.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task...", pe); } }
private void handleParseException(ParseException pe, ConsumerRecord<byte[], byte[]> record) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (tuningConfig.isLogParseExceptions()) { log.error( pe, "Encountered parse exception on row from partition[%d] offset[%d]", record.partition(), record.offset() ); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
private void handleParseException(ParseException pe, OrderedPartitionableRecord record) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (tuningConfig.isLogParseExceptions()) { log.error( pe, "Encountered parse exception on row from partition[%s] sequence[%s]", record.getPartitionId(), record.getSequenceNumber() ); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
private void handleParseException(ParseException e) { if (e.isFromPartiallyValidRow()) { buildSegmentsMeters.incrementProcessedWithError(); } else { buildSegmentsMeters.incrementUnparseable(); } if (ingestionSchema.tuningConfig.isLogParseExceptions()) { log.error(e, "Encountered parse exception:"); } if (buildSegmentsSavedParseExceptions != null) { buildSegmentsSavedParseExceptions.add(e); } if (buildSegmentsMeters.getUnparseable() + buildSegmentsMeters.getProcessedWithError() > ingestionSchema.tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task...", e); } }
private void handleParseException(ParseException pe) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (spec.getTuningConfig().isLogParseExceptions()) { log.error(pe, "Encountered parse exception: "); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > spec.getTuningConfig().getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
private void handleParseException(ParseException pe, Context context) { context.getCounter(HadoopDruidIndexerConfig.IndexJobCounters.INVALID_ROW_COUNTER).increment(1); Counter unparseableCounter = context.getCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_UNPARSEABLE_COUNTER); Counter processedWithErrorsCounter = context.getCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_PROCESSED_WITH_ERRORS_COUNTER); if (pe.isFromPartiallyValidRow()) { processedWithErrorsCounter.increment(1); } else { unparseableCounter.increment(1); } if (config.isLogParseExceptions()) { log.error(pe, "Encountered parse exception: "); } long rowsUnparseable = unparseableCounter.getValue(); long rowsProcessedWithError = processedWithErrorsCounter.getValue(); if (rowsUnparseable + rowsProcessedWithError > config.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task...", pe); } }