/** * Adds a restriction this handler should be restricted to. * @param field metadata property/field * @param regex regular expression * @param caseSensitive whether regular expression should be case sensitive */ public synchronized void addRestriction( String field, String regex, boolean caseSensitive) { restrictions.add(new PropertyMatcher(field, regex, caseSensitive)); }
public Properties extractFields(CharSequence text) { Properties dest = new Properties(); extractFields(dest, text); return dest; }
public JobStatusUpdater(final MutableJobStatus status) { this.status = status; status.getProperties().addMapChangeListener( new IMapChangeListener<String, List<String>>() { @Override public void mapChanged(MapChangeEvent<String, List<String>> event) { statusUpdated(status); } }); }
throws IOException { Properties config = new Properties(); config.setString("jobId", jobStatus.getJobId()); config.setDouble("progress", jobStatus.getProgress()); if (jobStatus.getNote() != null) { config.setString("note", jobStatus.getNote()); config.setInt("resumeAttempts", jobStatus.getResumeAttempts()); config.setDate("resumedStartTime", duration.getResumedStartTime()); config.setDate("resumedLastActivity", duration.getResumedLastActivity()); config.setDate("startTime", duration.getStartTime()); config.setDate("endTime", duration.getEndTime()); config.setBoolean("stopped", true); for (String key : props.keySet()) { config.put("prop." + key, props.get(key)); FileLock lock = channel.lock()) { StringWriter sw = new StringWriter(); config.store(sw, "Status for job: " + jobStatus.getJobId()); ras.writeUTF(sw.toString());
Properties config = new Properties(); config.load(sr); } catch (IOException e) { LOG.error("Cannot read file: " + file.getAbsolutePath()); jobStatus.setProgress(config.getDouble("progress", 0d)); jobStatus.setNote(config.getString("note", null)); jobStatus.setResumeAttempts(config.getInt("resumeAttempts", 0)); config.getDate("resumedStartTime", null)); duration.setResumedLastActivity( config.getDate("resumedLastActivity", null)); duration.setStartTime(config.getDate("startTime", null)); duration.setEndTime(config.getDate("endTime", null)); jobStatus.setDuration(duration); jobStatus.setStopRequested(config.getBoolean("stopped", false)); for (String key : config.keySet()) { if (key.startsWith("prop.")) { props.put(StringUtils.removeStart( "prop.", key), props.get(key));
referenceValue = metadata.getString(sourceReferenceField); metadata.setString(targetReferenceField, referenceValue); && !Objects.equals( sourceReferenceField, targetReferenceField)) { metadata.remove(sourceReferenceField); List<String >content = metadata.getStrings(sourceContentField); metadata.setString(targetContentField, content.toArray(ArrayUtils.EMPTY_STRING_ARRAY)); if (!keepSourceContentField && !Objects.equals( sourceContentField, targetContentField)) { metadata.remove(sourceContentField); metadata.setString(targetContentField, IOUtils.toString(is, StandardCharsets.UTF_8)); IOUtils.closeQuietly(is);
"Could not load reference for " + refFile, e); this.metadata = new Properties(); synchronized (metadata) { if (metaFile.exists()) { try { is = new FileInputStream(metaFile); metadata.load(is); } catch (IOException e) { throw new CommitterException(
appendValues(b, field, metadata.getStrings(field)); for (Entry<String, List<String>> entry : metadata.entrySet()) { if (p.matcher(entry.getKey()).matches()) { appendValues(b, entry.getKey(), entry.getValue());
@Override public final void saveToXML(Writer out) throws IOException { try { EnhancedXMLStreamWriter writer = new EnhancedXMLStreamWriter(out); writer.writeStartElement(xmltag); writer.writeAttribute("class", getClass().getCanonicalName()); saveHandlerToXML(writer); for (PropertyMatcher restriction : restrictions) { writer.writeStartElement("restrictTo"); if (restriction.getKey() != null) { writer.writeAttribute("field", restriction.getKey()); } writer.writeAttribute("caseSensitive", Boolean.toString(restriction.isCaseSensitive())); if (restriction.getRegex() != null) { writer.writeCharacters(restriction.getRegex()); } writer.writeEndElement(); } writer.writeEndElement(); writer.flush(); writer.close(); } catch (XMLStreamException e) { throw new IOException("Cannot save as XML.", e); } }
@Override public final String createMetadataChecksum(Properties metadata) { String checksum = doCreateMetaChecksum(metadata); if (isKeep()) { String field = getTargetField(); if (StringUtils.isBlank(field)) { field = CollectorMetadata.COLLECTOR_CHECKSUM_METADATA; } metadata.addString(field, checksum); if (LOG.isDebugEnabled()) { LOG.debug("Meta checksum stored in " + field); } } return checksum; }
/** * Removes all restrictions on a given field. * @param field the field to remove restrictions on * @return how many elements were removed * @since 2.4.0 */ public synchronized int removeRestriction(String field) { Iterator<PropertyMatcher> it = restrictions.iterator(); int count = 0; while (it.hasNext()) { PropertyMatcher r = it.next(); if (r.isCaseSensitive() && r.getKey().equals(field) || !r.isCaseSensitive() && r.getKey().equalsIgnoreCase(field)) { it.remove(); count++; } } return count; }
@Override public void add( String reference, InputStream content, Properties metadata) { try { File targetFile = createFile(FILE_SUFFIX_ADD); // Content FileUtils.copyInputStreamToFile(content, new File(targetFile.getAbsolutePath() + EXTENSION_CONTENT)); // Metadata FileOutputStream out = new FileOutputStream(new File( targetFile.getAbsolutePath() + EXTENSION_METADATA)); metadata.store(out, ""); IOUtils.closeQuietly(out); // Reference FileUtils.writeStringToFile(new File( targetFile.getAbsolutePath() + EXTENSION_REFERENCE), reference, StandardCharsets.UTF_8); } catch (IOException e) { throw new CommitterException( "Cannot queue document addition. Ref: " + reference, e); } } @Override
@Override public boolean acceptMetadata(String reference, Properties metadata) { if (StringUtils.isBlank(regex)) { return getOnMatch() == OnMatch.INCLUDE; } Collection<String> values = metadata.getStrings(field); for (Object value : values) { String strVal = Objects.toString(value, StringUtils.EMPTY); if (getCachedPattern().matcher(strVal).matches()) { return getOnMatch() == OnMatch.INCLUDE; } } return getOnMatch() == OnMatch.EXCLUDE; }
xml.writeElementString("reference", reference); xml.writeStartElement("metadata"); for (Entry<String, List<String>> entry : metadata.entrySet()) { for (String value : entry.getValue()) { xml.writeStartElement("meta");
/** * Class to invoke by subclasses to find out if this handler should be * rejected or not based on the metadata restriction provided. * @param reference document reference * @param metadata document metadata. * @param parsed if the document was parsed (i.e. imported) already * @return <code>true</code> if this handler is applicable to the document */ protected final boolean isApplicable( String reference, ImporterMetadata metadata, boolean parsed) { if (restrictions.isEmpty()) { return true; } for (PropertyMatcher restriction : restrictions) { if (restriction.matches(metadata)) { return true; } } if (LOG.isDebugEnabled()) { LOG.debug(getClass() + " handler does not apply to: " + reference); } return false; }
@Override public void execute(JobStatusUpdater statusUpdater, JobSuite suite) { double commandCount = systemCommands.length; double commandsRan = statusUpdater.getProperties().getDouble("ran", 0d); for (int i = (int) commandsRan; i < commandCount; i++) { SystemCommand systemCommand = systemCommands[i]; if (LOG.isDebugEnabled()) { LOG.debug("Executing command: " + systemCommand); } statusUpdater.setNote("Executing: " + systemCommand); try { systemCommand.execute(); statusUpdater.setProgress(commandsRan / commandCount); } catch (SystemCommandException e) { throw new JobException("Cannot execute command: " + systemCommand, e); } } statusUpdater.setNote("Done."); }
public static Properties extractFields( CharSequence text, RegexFieldExtractor... patterns) { Properties dest = new Properties(); extractFields(dest, text, patterns); return dest; } }
public void extractFields(Properties dest, CharSequence text) { Matcher m = matcher(text); while (m.find()) { String fieldName = extractFieldName(m); String fieldValue = extractFieldValue(m); if (StringUtils.isBlank(fieldName)) { LOG.warn("No field name for value: " + fieldValue); } else if (fieldValue == null) { LOG.warn("Null value for field: " + field); } else { dest.addString(fieldName, fieldValue); } } } public Properties extractFields(CharSequence text) {
private static List<PropertyMatcher> build( String field, String... regexes) { List<PropertyMatcher> list = new ArrayList<>(); for (String regex : regexes) { list.add(new PropertyMatcher(field, regex, false)); } return list; } }
Properties metadata = new Properties(); ImporterConfig config = loadCommandLineConfig(cmd, configFile, varFile);