public static boolean isArrayTypeCompatible(final Object value, final DataType elementDataType) { return value != null // Either an object array or a String to be converted to byte[] && (value instanceof Object[] || (value instanceof String && RecordFieldType.BYTE.getDataType().equals(elementDataType))); }
public void addSchemaField(final String fieldName, final RecordFieldType type, boolean isNullable) { fields.add(new RecordField(fieldName, type.getDataType(), isNullable)); }
private FieldValue missingChild(final FieldValue parent) { final RecordField field = new RecordField(childName, RecordFieldType.CHOICE.getChoiceDataType(RecordFieldType.STRING.getDataType(), RecordFieldType.RECORD.getDataType())); return new StandardFieldValue(null, field, parent); }
static RecordSchema createRecordSchema(final Grok grok) { final List<RecordField> fields = new ArrayList<>(); String grokExpression = grok.getOriginalGrokPattern(); populateSchemaFieldNames(grok, grokExpression, fields); fields.add(new RecordField(GrokRecordReader.STACK_TRACE_COLUMN_NAME, RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(GrokRecordReader.RAW_MESSAGE_NAME, RecordFieldType.STRING.getDataType(), true)); final RecordSchema schema = new SimpleRecordSchema(fields); return schema; }
private List<RecordField> getRecordFields() { if (this.recordFields != null) { return this.recordFields; } // Use a SortedMap keyed by index of the field so that we can get a List of field names in the correct order final SortedMap<Integer, String> sortedMap = new TreeMap<>(); for (final Map.Entry<String, Integer> entry : csvParser.getHeaderMap().entrySet()) { sortedMap.put(entry.getValue(), entry.getKey()); } final List<RecordField> fields = new ArrayList<>(); final List<String> rawFieldNames = new ArrayList<>(sortedMap.values()); for (final String rawFieldName : rawFieldNames) { final Optional<RecordField> option = schema.getField(rawFieldName); if (option.isPresent()) { fields.add(option.get()); } else { fields.add(new RecordField(rawFieldName, RecordFieldType.STRING.getDataType())); } } this.recordFields = fields; return fields; }
static RecordSchema createRecordSchema() { final List<RecordField> fields = new ArrayList<>(); fields.add(new RecordField(SyslogAttributes.PRIORITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.SEVERITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.FACILITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.VERSION.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.TIMESTAMP.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.HOSTNAME.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.BODY.key(), RecordFieldType.STRING.getDataType(), true)); SchemaIdentifier schemaIdentifier = new StandardSchemaIdentifier.Builder().name(GENERIC_SYSLOG_SCHEMA_NAME).build(); final RecordSchema schema = new SimpleRecordSchema(fields,schemaIdentifier); return schema; }
@Override public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { Stream<FieldValue> concatenated = Stream.empty(); for (final RecordPathSegment valuePath : valuePaths) { final Stream<FieldValue> stream = valuePath.evaluate(context); concatenated = Stream.concat(concatenated, stream); } final StringBuilder sb = new StringBuilder(); concatenated.forEach(fv -> sb.append(DataTypeUtils.toString(fv.getValue(), (String) null))); final RecordField field = new RecordField("concat", RecordFieldType.STRING.getDataType()); final FieldValue responseValue = new StandardFieldValue(sb.toString(), field, null); return Stream.of(responseValue); }
protected RecordSchema convertSchema(Map<String, Object> result) { List<RecordField> fields = new ArrayList<>(); for (Map.Entry<String, Object> entry : result.entrySet()) { RecordField field; if (entry.getValue() instanceof Integer) { field = new RecordField(entry.getKey(), RecordFieldType.INT.getDataType()); } else if (entry.getValue() instanceof Long) { field = new RecordField(entry.getKey(), RecordFieldType.LONG.getDataType()); } else if (entry.getValue() instanceof Boolean) { field = new RecordField(entry.getKey(), RecordFieldType.BOOLEAN.getDataType()); } else if (entry.getValue() instanceof Double) { field = new RecordField(entry.getKey(), RecordFieldType.DOUBLE.getDataType()); } else if (entry.getValue() instanceof Date) { field = new RecordField(entry.getKey(), RecordFieldType.DATE.getDataType()); } else if (entry.getValue() instanceof List) { field = new RecordField(entry.getKey(), RecordFieldType.ARRAY.getDataType()); } else if (entry.getValue() instanceof Map) { RecordSchema nestedSchema = convertSchema((Map)entry.getValue()); RecordDataType rdt = new RecordDataType(nestedSchema); field = new RecordField(entry.getKey(), rdt); } else { field = new RecordField(entry.getKey(), RecordFieldType.STRING.getDataType()); } fields.add(field); } return new SimpleRecordSchema(fields); }
public static Object[] toArray(final Object value, final String fieldName, final DataType elementDataType, final Charset charset) { if (value == null) { return null; } if (value instanceof Object[]) { return (Object[]) value; } if (value instanceof String && RecordFieldType.BYTE.getDataType().equals(elementDataType)) { byte[] src = ((String) value).getBytes(charset); Byte[] dest = new Byte[src.length]; for (int i = 0; i < src.length; i++) { dest[i] = src[i]; } return dest; } if (value instanceof byte[]) { byte[] src = (byte[]) value; Byte[] dest = new Byte[src.length]; for (int i = 0; i < src.length; i++) { dest[i] = src[i]; } return dest; } if (value instanceof List) { final List<?> list = (List<?>)value; return list.toArray(); } throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Object Array for field " + fieldName); }
static RecordSchema createRecordSchema() { final List<RecordField> fields = new ArrayList<>(); fields.add(new RecordField(SyslogAttributes.PRIORITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.SEVERITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.FACILITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.VERSION.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.TIMESTAMP.key(), RecordFieldType.TIMESTAMP.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.HOSTNAME.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.BODY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.APP_NAME.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.PROCID.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.MESSAGEID.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.STRUCTURED_BASE.key(), RecordFieldType.MAP.getMapDataType(RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType())))); SchemaIdentifier schemaIdentifier = new StandardSchemaIdentifier.Builder().name(RFC_5424_SCHEMA_NAME).build(); final RecordSchema schema = new SimpleRecordSchema(fields,schemaIdentifier); return schema; }
private static void populateSchemaFieldNames(final Grok grok, String grokExpression, final List<RecordField> fields) { final Set<String> namedGroups = GrokUtils.getNameGroups(GrokUtils.GROK_PATTERN.pattern()); while (grokExpression.length() > 0) { final Matcher matcher = GrokUtils.GROK_PATTERN.matcher(grokExpression); if (matcher.find()) { final Map<String, String> extractedGroups = GrokUtils.namedGroups(matcher, namedGroups); final String subName = extractedGroups.get("subname"); if (subName == null) { final String subPatternName = extractedGroups.get("pattern"); if (subPatternName == null) { continue; } final String subExpression = grok.getPatterns().get(subPatternName); populateSchemaFieldNames(grok, subExpression, fields); } else { DataType dataType = RecordFieldType.STRING.getDataType(); final RecordField recordField = new RecordField(subName, dataType); fields.add(recordField); } if (grokExpression.length() > matcher.end() + 1) { grokExpression = grokExpression.substring(matcher.end()); } else { break; } } else { break; } } }
private List<RecordField> getDefaultFields() { final List<RecordField> fields = new ArrayList<>(); fields.add(new RecordField("id", RecordFieldType.INT.getDataType())); fields.add(new RecordField("name", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("address", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("city", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("state", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("zipCode", RecordFieldType.STRING.getDataType())); fields.add(new RecordField("country", RecordFieldType.STRING.getDataType())); return fields; }
@Override public RecordSchema getSchema(Map<String, String> variables, RecordSchema readSchema) throws SchemaNotFoundException, IOException { final List<RecordField> recordFields = columnNames.stream() .map(name -> new RecordField(name, RecordFieldType.STRING.getDataType())) .collect(Collectors.toList()); return new SimpleRecordSchema(recordFields); }
private RecordSchema getAccountSchema() { final List<RecordField> accountFields = new ArrayList<>(); accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType())); accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType())); return new SimpleRecordSchema(accountFields); }
private RecordSchema getTransactionSchema() { final List<RecordField> transactionFields = new ArrayList<>(); transactionFields.add(new RecordField("id", RecordFieldType.INT.getDataType())); transactionFields.add(new RecordField("amount", RecordFieldType.DOUBLE.getDataType())); return new SimpleRecordSchema(transactionFields); }
@Override public RecordSchema getSchema(Map<String, String> variables, final InputStream contentStream, final RecordSchema readSchema) throws SchemaNotFoundException { if (this.context == null) { throw new SchemaNotFoundException("Schema Access Strategy intended only for validation purposes and cannot obtain schema"); } try { final CSVFormat csvFormat = CSVUtils.createCSVFormat(context).withFirstRecordAsHeader(); try (final Reader reader = new InputStreamReader(new BOMInputStream(contentStream)); final CSVParser csvParser = new CSVParser(reader, csvFormat)) { final List<RecordField> fields = new ArrayList<>(); for (final String columnName : csvParser.getHeaderMap().keySet()) { fields.add(new RecordField(columnName, RecordFieldType.STRING.getDataType(), true)); } return new SimpleRecordSchema(fields); } } catch (final Exception e) { throw new SchemaNotFoundException("Failed to read Header line from CSV", e); } }
@Override public Stream<FieldValue> evaluate(RecordPathEvaluationContext context) { final Stream<FieldValue> fieldValues = recordPath.evaluate(context); return fieldValues.filter(fv -> fv.getValue() != null) .map(fv -> { if (!(fv.getValue() instanceof String)) { throw new IllegalArgumentException("Argument supplied to toBytes must be a String"); } final Charset charset = getCharset(this.charsetSegment, context); final byte[] bytesValue; Byte[] src = (Byte[]) DataTypeUtils.toArray(fv.getValue(), fv.getField().getFieldName(), RecordFieldType.BYTE.getDataType(), charset); bytesValue = new byte[src.length]; for (int i = 0; i < src.length; i++) { bytesValue[i] = src[i]; } return new StandardFieldValue(bytesValue, fv.getField(), fv.getParent().orElse(null)); }); }
private RecordSchema getAccountWithTransactionSchema() { final List<RecordField> accountFields = new ArrayList<>(); accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType())); accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType())); final DataType transactionRecordType = RecordFieldType.RECORD.getRecordDataType(getTransactionSchema()); final DataType transactionsType = RecordFieldType.ARRAY.getArrayDataType(transactionRecordType); accountFields.add(new RecordField("transactions", transactionsType)); return new SimpleRecordSchema(accountFields); }
@Test public void testForkExtractNotAnArrayOfRecords() throws IOException, MalformedRecordException, InitializationException { TestRunner runner = TestRunners.newTestRunner(new ForkRecord()); final DataType accountRecordType = RecordFieldType.STRING.getDataType(); final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType); final List<RecordField> fields = getDefaultFields(); fields.add(new RecordField("accounts", accountsType)); final RecordSchema schema = new SimpleRecordSchema(fields); final List<RecordField> fieldsWrite = getDefaultFields(); fieldsWrite.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType())); final RecordSchema schemaWrite = new SimpleRecordSchema(fieldsWrite); final JsonRecordReader readerService = new JsonRecordReader(schema); final MockRecordWriter writerService = new CustomRecordWriter("header", false, schemaWrite); runner.addControllerService("reader", readerService); runner.enableControllerService(readerService); runner.addControllerService("writer", writerService); runner.enableControllerService(writerService); runner.setProperty(ForkRecord.RECORD_READER, "reader"); runner.setProperty(ForkRecord.RECORD_WRITER, "writer"); runner.setProperty(ForkRecord.MODE, ForkRecord.MODE_EXTRACT); runner.setProperty(ForkRecord.INCLUDE_PARENT_FIELDS, "true"); runner.setProperty("my-path", "/accounts"); runner.enqueue(new File("src/test/resources/TestForkRecord/single-element-nested-array-strings.json").toPath()); runner.run(1); runner.assertTransferCount(ForkRecord.REL_ORIGINAL, 1); runner.assertTransferCount(ForkRecord.REL_FORK, 1); final MockFlowFile mff = runner.getFlowFilesForRelationship(ForkRecord.REL_FORK).get(0); mff.assertAttributeEquals("record.count", "0"); }
@Test public void testForkExtractNotAnArray() throws IOException, MalformedRecordException, InitializationException { TestRunner runner = TestRunners.newTestRunner(new ForkRecord()); final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema()); final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType); final List<RecordField> fields = getDefaultFields(); fields.add(new RecordField("accounts", accountsType)); final RecordSchema schema = new SimpleRecordSchema(fields); final List<RecordField> fieldsWrite = getDefaultFields(); fieldsWrite.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType())); final RecordSchema schemaWrite = new SimpleRecordSchema(fieldsWrite); final JsonRecordReader readerService = new JsonRecordReader(schema); final MockRecordWriter writerService = new CustomRecordWriter("header", false, schemaWrite); runner.addControllerService("reader", readerService); runner.enableControllerService(readerService); runner.addControllerService("writer", writerService); runner.enableControllerService(writerService); runner.setProperty(ForkRecord.RECORD_READER, "reader"); runner.setProperty(ForkRecord.RECORD_WRITER, "writer"); runner.setProperty(ForkRecord.MODE, ForkRecord.MODE_EXTRACT); runner.setProperty(ForkRecord.INCLUDE_PARENT_FIELDS, "true"); runner.setProperty("my-path", "/country"); runner.enqueue(new File("src/test/resources/TestForkRecord/single-element-nested-array.json").toPath()); runner.run(1); runner.assertTransferCount(ForkRecord.REL_ORIGINAL, 1); runner.assertTransferCount(ForkRecord.REL_FORK, 1); final MockFlowFile mff = runner.getFlowFilesForRelationship(ForkRecord.REL_FORK).get(0); mff.assertAttributeEquals("record.count", "0"); }