@Override public Map<String, Object> parseToMap(String input) { try { final Object compiled = fn.apply(input); if (!(compiled instanceof Map)) { throw new ParseException("JavaScript parsed value must be in {key: value} format!"); } return (Map) compiled; } catch (Exception e) { throw new ParseException(e, "Unable to parse row [%s]", input); } }
public static void validateFields(Iterable<String> fieldNames) { Set<String> duplicates = findDuplicates(fieldNames); if (!duplicates.isEmpty()) { throw new ParseException("Duplicate column entries found : %s", duplicates.toString()); } }
@Nullable public static Long convertObjectToLong(@Nullable Object valObj, boolean reportParseExceptions) { if (valObj == null) { return null; } if (valObj instanceof Long) { return (Long) valObj; } else if (valObj instanceof Number) { return ((Number) valObj).longValue(); } else if (valObj instanceof String) { Long ret = DimensionHandlerUtils.getExactLongFromDecimalString((String) valObj); if (reportParseExceptions && ret == null) { throw new ParseException("could not convert value [%s] to long", valObj); } return ret; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } }
@Nullable public static Double convertObjectToDouble(@Nullable Object valObj, boolean reportParseExceptions) { if (valObj == null) { return null; } if (valObj instanceof Double) { return (Double) valObj; } else if (valObj instanceof Number) { return ((Number) valObj).doubleValue(); } else if (valObj instanceof String) { Double ret = Doubles.tryParse((String) valObj); if (reportParseExceptions && ret == null) { throw new ParseException("could not convert value [%s] to double", valObj); } return ret; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } }
@Nullable public static Float convertObjectToFloat(@Nullable Object valObj, boolean reportParseExceptions) { if (valObj == null) { return null; } if (valObj instanceof Float) { return (Float) valObj; } else if (valObj instanceof Number) { return ((Number) valObj).floatValue(); } else if (valObj instanceof String) { Float ret = Floats.tryParse((String) valObj); if (reportParseExceptions && ret == null) { throw new ParseException("could not convert value [%s] to float", valObj); } return ret; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } }
/** * @param input JSON string. The root must be a JSON object, not an array. * e.g., {"valid": "true"} and {"valid":[1,2,3]} are supported * but [{"invalid": "true"}] and [1,2,3] are not. * * @return A map of field names and values */ @Override public Map<String, Object> parseToMap(String input) { try { JsonNode document = mapper.readValue(input, JsonNode.class); return flattener.flatten(document); } catch (Exception e) { throw new ParseException(e, "Unable to parse row [%s]", input); } } }
@Override public GenericRecord parse(ByteBuffer bytes) { try (ByteBufferInputStream inputStream = new ByteBufferInputStream(Collections.singletonList(bytes))) { return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null)); } catch (EOFException eof) { // waiting for avro v1.9.0 (#AVRO-813) throw new ParseException( eof, "Avro's unnecessary EOFException, detail: [%s]", "https://issues.apache.org/jira/browse/AVRO-813" ); } catch (Exception e) { throw new ParseException(e, "Fail to decode avro message!"); } } }
throw new ParseException("record must have at least 5 bytes carrying version and schemaId"); throw new ParseException("found record of arbitrary version [%s]", version); Schema schemaObj = schemaObjs.get(schemaId); if (schemaObj == null) { throw new ParseException("Failed to find schema for id [%s]", schemaId); throw new ParseException( eof, "Avro's unnecessary EOFException, detail: [%s]", throw new ParseException(e, "Fail to decode avro message with schemaId [%s].", schemaId);
@Override public GenericRecord parse(ByteBuffer bytes) { try { bytes.get(); // ignore first \0 byte int id = bytes.getInt(); // extract schema registry id int length = bytes.limit() - 1 - 4; int offset = bytes.position() + bytes.arrayOffset(); Schema schema = registry.getByID(id); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); return reader.read(null, DecoderFactory.get().binaryDecoder(bytes.array(), offset, length, null)); } catch (Exception e) { throw new ParseException(e, "Fail to decode avro message!"); } }
private Map<String, Object> buildStringKeyMap(ByteBuffer input) { int payloadSize = input.remaining(); if (chars == null || chars.remaining() < payloadSize) { chars = CharBuffer.allocate(payloadSize); } final CoderResult coderResult = charset.newDecoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE) .decode(input, chars, true); Map<String, Object> theMap; if (coderResult.isUnderflow()) { chars.flip(); try { theMap = parseString(chars.toString()); } finally { chars.clear(); } } else { throw new ParseException("Failed with CoderResult[%s]", coderResult); } return theMap; }
@Override public Map<String, Object> parseToMap(String input) { try { final Matcher matcher = compiled.matcher(input); if (!matcher.matches()) { throw new ParseException("Incorrect Regex: %s . No match found.", pattern); } List<String> values = new ArrayList<>(); for (int i = 1; i <= matcher.groupCount(); i++) { values.add(matcher.group(i)); } if (fieldNames == null) { setFieldNames(ParserUtils.generateFieldNames(values.size())); } return Utils.zipMapPartial(fieldNames, Iterables.transform(values, valueFunction)); } catch (Exception e) { throw new ParseException(e, "Unable to parse row [%s]", input); } }
public void setFieldNames(final String header) { try { setFieldNames(parseLine(header)); } catch (Exception e) { throw new ParseException(e, "Unable to parse header [%s]", header); } }
@Override public GenericRecord parse(ByteBuffer bytes) { Pair<SUBJECT, ID> subjectAndId = subjectAndIdConverter.getSubjectAndId(bytes); Schema schema = typedRepository.getSchema(subjectAndId.lhs, subjectAndId.rhs); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); try (ByteBufferInputStream inputStream = new ByteBufferInputStream(Collections.singletonList(bytes))) { return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null)); } catch (EOFException eof) { // waiting for avro v1.9.0 (#AVRO-813) throw new ParseException( eof, "Avro's unnecessary EOFException, detail: [%s]", "https://issues.apache.org/jira/browse/AVRO-813" ); } catch (IOException e) { throw new ParseException(e, "Fail to decode avro message!"); } }
@Override public List<InputRow> parseBatch(Map<String, Object> theMap) { final List<String> dimensions; if (!this.dimensions.isEmpty()) { dimensions = this.dimensions; } else { dimensions = Lists.newArrayList( Sets.difference(theMap.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) ); } final DateTime timestamp; try { timestamp = parseSpec.getTimestampSpec().extractTimestamp(theMap); if (timestamp == null) { final String input = theMap.toString(); throw new NullPointerException( StringUtils.format( "Null timestamp in input: %s", input.length() < 100 ? input : input.substring(0, 100) + "..." ) ); } } catch (Exception e) { throw new ParseException(e, "Unparseable timestamp found! Event: %s", theMap); } return ImmutableList.of(new MapBasedInputRow(timestamp, dimensions, theMap)); }
@Override public Map<String, Object> parseToMap(final String input) { if (!supportSkipHeaderRows && (hasHeaderRow || maxSkipHeaderRows > 0)) { throw new UnsupportedOperationException("hasHeaderRow or maxSkipHeaderRows is not supported. " + "Please check the indexTask supports these options."); } try { List<String> values = parseLine(input); if (skippedHeaderRows < maxSkipHeaderRows) { skippedHeaderRows++; return null; } if (hasHeaderRow && !hasParsedHeader) { if (fieldNames == null) { setFieldNames(values); } hasParsedHeader = true; return null; } if (fieldNames == null) { setFieldNames(ParserUtils.generateFieldNames(values.size())); } return Utils.zipMapPartial(fieldNames, Iterables.transform(values, valueFunction)); } catch (Exception e) { throw new ParseException(e, "Unable to parse row [%s]", input); } }
@Override public InputRow nextRow() { synchronized (this) { final InputRow row = parser.parseBatch(queue.removeFirst().orElse(null)).get(0); if (row != null && row.getRaw(FAIL_DIM) != null) { throw new ParseException(FAIL_DIM); } return row; } }
@Override public InputRow nextRow() { synchronized (this) { final InputRow row = parser.parseBatch(queue.removeFirst().orElse(null)).get(0); if (row != null && row.getRaw(FAIL_DIM) != null) { throw new ParseException(FAIL_DIM); } return row; } }
throw new ParseException("Unable to parse line."); throw new ParseException("Multiple lines present; unable to parse more than one per record."); throw new ParseException("Metric not whitelisted.");
public boolean withinMinMaxRecordTime(final InputRow row) { final boolean beforeMinimumMessageTime = ioConfig.getMinimumMessageTime().isPresent() && ioConfig.getMinimumMessageTime().get().isAfter(row.getTimestamp()); final boolean afterMaximumMessageTime = ioConfig.getMaximumMessageTime().isPresent() && ioConfig.getMaximumMessageTime().get().isBefore(row.getTimestamp()); if (!Intervals.ETERNITY.contains(row.getTimestamp())) { final String errorMsg = StringUtils.format( "Encountered row with timestamp that cannot be represented as a long: [%s]", row ); throw new ParseException(errorMsg); } if (log.isDebugEnabled()) { if (beforeMinimumMessageTime) { log.debug( "CurrentTimeStamp[%s] is before MinimumMessageTime[%s]", row.getTimestamp(), ioConfig.getMinimumMessageTime().get() ); } else if (afterMaximumMessageTime) { log.debug( "CurrentTimeStamp[%s] is after MaximumMessageTime[%s]", row.getTimestamp(), ioConfig.getMaximumMessageTime().get() ); } } return !beforeMinimumMessageTime && !afterMaximumMessageTime; }
throw new ParseException(e, "Protobuf message could not be parsed");