protected void sendData(final ReportingContext context, final Transaction transaction, Map<String, String> attributes, final JsonArray jsonArray) throws IOException { if(context.getProperty(RECORD_WRITER).isSet()) { transaction.send(getData(context, new ByteArrayInputStream(jsonArray.toString().getBytes(StandardCharsets.UTF_8)), attributes), attributes); } else { transaction.send(jsonArray.toString().getBytes(StandardCharsets.UTF_8), attributes); } }
@Override protected void preProcessConfiguration(Configuration config, ProcessContext context) { // Set umask once, to avoid thread safety issues doing it in onTrigger final PropertyValue umaskProp = context.getProperty(UMASK); final short dfsUmask; if (umaskProp.isSet()) { dfsUmask = Short.parseShort(umaskProp.getValue(), 8); } else { dfsUmask = FsPermission.DEFAULT_UMASK; } FsPermission.setUMask(config, new FsPermission(dfsUmask)); }
static List<String> getAuthorizations(ConfigurationContext context) { List<String> tokens = new ArrayList<>(); String authorizationString = context.getProperty(AUTHORIZATIONS).isSet() ? context.getProperty(AUTHORIZATIONS).getValue() : ""; if (!StringUtils.isEmpty(authorizationString)) { tokens = Arrays.asList(authorizationString.split(",[\\s]*")); } return tokens; } }
static List<String> getAuthorizations(ConfigurationContext context) { List<String> tokens = new ArrayList<>(); String authorizationString = context.getProperty(AUTHORIZATIONS).isSet() ? context.getProperty(AUTHORIZATIONS).getValue() : ""; if (!StringUtils.isEmpty(authorizationString)) { tokens = Arrays.asList(authorizationString.split(",[\\s]*")); } return tokens; } }
default Optional<ValidationResult> validateRequiredField(ValidationContext context, PropertyDescriptor prop) { if (!context.getProperty(prop).isSet()) { return Optional.of(new ValidationResult.Builder() .subject(prop.getDisplayName()) .valid(false) .explanation(String.format("required by '%s' auth.", this.getClass().getSimpleName())) .build()); } return Optional.empty(); } }
private JoltTransform getTransform(final ProcessContext context, final FlowFile flowFile) throws Exception { final Optional<String> specString; if (context.getProperty(JOLT_SPEC).isSet()) { specString = Optional.of(context.getProperty(JOLT_SPEC).evaluateAttributeExpressions(flowFile).getValue()); } else { specString = Optional.empty(); } return transformCache.get(specString); }
private JoltTransform getTransform(final ProcessContext context, final FlowFile flowFile) { final Optional<String> specString; if (context.getProperty(JOLT_SPEC).isSet()) { specString = Optional.of(context.getProperty(JOLT_SPEC).evaluateAttributeExpressions(flowFile).getValue()); } else { specString = Optional.empty(); } return transformCache.get(specString); }
public static void validateProxyProperties(ValidationContext context, Collection<ValidationResult> results) { if (context.getProperty(PROXY_HOST).isSet() && !context.getProperty(PROXY_PORT).isSet()) { results.add(new ValidationResult.Builder() .explanation("Proxy Host was set but no Proxy Port was specified") .valid(false) .subject("Proxy server configuration") .build()); } ProxyConfiguration.validateProxySpec(context, results, PROXY_SPECS); } }
@Override protected Collection<ValidationResult> customValidate(ValidationContext validationContext) { List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext)); final PropertyValue columnForPartitioning = validationContext.getProperty(COLUMN_FOR_VALUE_PARTITIONING); // If no EL is present, ensure it's a single column (i.e. no commas in the property value) if (columnForPartitioning.isSet() && !columnForPartitioning.isExpressionLanguagePresent() && columnForPartitioning.getValue().contains(",")) { results.add(new ValidationResult.Builder().valid(false).explanation( COLUMN_FOR_VALUE_PARTITIONING.getDisplayName() + " requires a single column name, but a comma was detected").build()); } return results; }
@OnScheduled public void onScheduled(ProcessContext context) { attributesToRemove = context.getProperty(INCLUDE_CORE_ATTRIBUTES).asBoolean() ? Collections.EMPTY_SET : Arrays.stream(CoreAttributes.values()) .map(CoreAttributes::key) .collect(Collectors.toSet()); attributes = buildAtrs(context.getProperty(ATTRIBUTES_LIST).getValue(), attributesToRemove); nullValueForEmptyString = context.getProperty(NULL_VALUE_FOR_EMPTY_STRING).asBoolean(); destinationContent = DESTINATION_CONTENT.equals(context.getProperty(DESTINATION).getValue()); if(context.getProperty(ATTRIBUTES_REGEX).isSet()) { pattern = Pattern.compile(context.getProperty(ATTRIBUTES_REGEX).evaluateAttributeExpressions().getValue()); } }
private static void validateRequiredProperty(ValidationContext context, Collection<ValidationResult> results, PropertyDescriptor property) { if (!context.getProperty(property).isSet()) { final String displayName = property.getDisplayName(); results.add(new ValidationResult.Builder() .subject(displayName) .explanation(format("'%s' is required to use '%s' listing strategy", displayName, AbstractListProcessor.BY_ENTITIES.getDisplayName())) .valid(false) .build()); } }
@Override protected Collection<ValidationResult> customValidate(ValidationContext validationContext) { final List<ValidationResult> results = new ArrayList<>(1); final boolean isUnique = validationContext.getProperty(UNIQUE_FLOWFILES).asBoolean(); final boolean isText = validationContext.getProperty(DATA_FORMAT).getValue().equals(DATA_FORMAT_TEXT); final boolean isCustom = validationContext.getProperty(CUSTOM_TEXT).isSet(); if(isCustom && (isUnique || !isText)) { results.add(new ValidationResult.Builder().subject("Custom Text").valid(false).explanation("If Custom Text is set, then Data Format must be " + "text and Unique FlowFiles must be false.").build()); } return results; }
private void updateDataDogTransport(ReportingContext context) throws IOException { String dataDogTransport = context.getProperty(DATADOG_TRANSPORT).getValue(); if (dataDogTransport.equalsIgnoreCase(DATADOG_AGENT.getValue())) { ddMetricRegistryBuilder.build("agent"); } else if (dataDogTransport.equalsIgnoreCase(DATADOG_HTTP.getValue()) && context.getProperty(API_KEY).isSet()) { ddMetricRegistryBuilder.build(context.getProperty(API_KEY).getValue()); } }
@OnScheduled public void setup(ProcessContext context) { boolean confFileProvided = context.getProperty(ORC_CONFIGURATION_RESOURCES).isSet(); if (confFileProvided) { final String configFiles = context.getProperty(ORC_CONFIGURATION_RESOURCES).getValue(); orcConfig = HiveJdbcCommon.getConfigurationFromFiles(configFiles); } }
@OnScheduled public void onSchedule(ProcessContext context) { this.removeTrailingNewLines = context.getProperty(REMOVE_TRAILING_NEWLINES).isSet() ? context.getProperty(REMOVE_TRAILING_NEWLINES).asBoolean() : false; this.maxSplitSize = context.getProperty(FRAGMENT_MAX_SIZE).isSet() ? context.getProperty(FRAGMENT_MAX_SIZE).asDataSize(DataUnit.B).longValue() : Long.MAX_VALUE; this.lineCount = context.getProperty(LINE_SPLIT_COUNT).asInteger(); this.headerLineCount = context.getProperty(HEADER_LINE_COUNT).asInteger(); this.headerMarker = context.getProperty(HEADER_MARKER).getValue(); }
@OnScheduled public void setup(ProcessContext context) { // If the query is not set, then an incoming flow file is needed. Otherwise fail the initialization if (!context.getProperty(HIVEQL_SELECT_QUERY).isSet() && !context.hasIncomingConnection()) { final String errorString = "Either the Select Query must be specified or there must be an incoming connection " + "providing flowfile(s) containing a SQL select query"; getLogger().error(errorString); throw new ProcessException(errorString); } }
@OnScheduled public void setup(ProcessContext context) { // If the query is not set, then an incoming flow file is needed. Otherwise fail the initialization if (!context.getProperty(HIVEQL_SELECT_QUERY).isSet() && !context.hasIncomingConnection()) { final String errorString = "Either the Select Query must be specified or there must be an incoming connection " + "providing flowfile(s) containing a SQL select query"; getLogger().error(errorString); throw new ProcessException(errorString); } }
@Override protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) { final List<ValidationResult> results = new ArrayList<>(); if (COMPLETION_MOVE.getValue().equalsIgnoreCase(validationContext.getProperty(COMPLETION_STRATEGY).getValue())) { if (!validationContext.getProperty(MOVE_DESTINATION_DIR).isSet()) { results.add(new ValidationResult.Builder().subject(MOVE_DESTINATION_DIR.getName()).input(null).valid(false).explanation( MOVE_DESTINATION_DIR.getName() + " must be specified if " + COMPLETION_STRATEGY.getName() + " is set to " + COMPLETION_MOVE.getDisplayName()).build()); } } return results; }
@OnScheduled public void onScheduled(final ProcessContext context) { super.onScheduled(context); // Either input connection or scheduled query is required if ( ! context.getProperty(INFLUX_DB_QUERY).isSet() && ! context.hasIncomingConnection() ) { String error = "The InfluxDB Query processor requires input connection or scheduled InfluxDB query"; getLogger().error(error); throw new ProcessException(error); } }
@OnScheduled public void setup(ProcessContext context) { // If the query is not set, then an incoming flow file is needed. Otherwise fail the initialization if (!context.getProperty(SQL_SELECT_QUERY).isSet() && !context.hasIncomingConnection()) { final String errorString = "Either the Select Query must be specified or there must be an incoming connection " + "providing flowfile(s) containing a SQL select query"; getLogger().error(errorString); throw new ProcessException(errorString); } dbcpService = context.getProperty(DBCP_SERVICE).asControllerService(DBCPService.class); }