Matcher(String path) { this.path = path; tokens = StringUtils.tokenize(path, "."); }
public static List<String> tokenize(String string) { return tokenize(string, DEFAULT_DELIMITER); }
public static List<String> tokenize(String string, String delimiters) { return tokenize(string, delimiters, true, true); }
/** * @param columnString The comma separated list of hive column names * @return A map of column names to "_colX" where "X" is the place in the struct */ private static Map<String, String> columnMap(String columnString) { // add default aliases for serialization (mapping name -> _colX) List<String> columnNames = StringUtils.tokenize(columnString, ","); if (columnNames.isEmpty()) { return Collections.emptyMap(); } Map<String, String> columns = new LinkedHashMap<String, String>(); for (int i = 0; i < columnNames.size(); i++) { columns.put(columnNames.get(i), HiveConstants.UNNAMED_COLUMN_PREFIX + i); } return columns; }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Override public void prepareToRead(RecordReader reader, PigSplit split) throws IOException { this.reader = reader; aliasesTupleNames = StringUtils.tokenize(getUDFProperties().getProperty( InternalConfigurationOptions.INTERNAL_ES_TARGET_FIELDS)); }
public static Map<String, String> aliases(String definition, boolean caseInsensitive) { List<String> aliases = StringUtils.tokenize(definition); Map<String, String> aliasMap = new LinkedHashMap<String, String>(); if (aliases != null) { for (String string : aliases) { // split alias string = string.trim(); int index = string.indexOf(":"); if (index > 0) { String key = string.substring(0, index); aliasMap.put(key, string.substring(index + 1)); aliasMap.put(caseInsensitive ? key.toLowerCase(Locale.ROOT) : key, string.substring(index + 1)); } } } return aliasMap; }
public List<String> getStormSpoutFields() { return StringUtils.tokenize(getProperty(ES_STORM_SPOUT_FIELDS, ES_STORM_SPOUT_FIELDS_DEFAULT)); }
@Override public void setSettings(Settings settings) { List<String> includeAsStrings = StringUtils.tokenize(settings.getMappingIncludes()); includes = (includeAsStrings.isEmpty() ? Collections.<NumberedInclude> emptyList() : new ArrayList<NumberedInclude>(includeAsStrings.size())); for (String include : includeAsStrings) { includes.add(new NumberedInclude(include)); } excludes = StringUtils.tokenize(settings.getMappingExcludes()); }
@Override public void setSettings(Settings settings) { this.settings = settings; String paramString = settings.getUpdateScriptParams(); List<String> fields = StringUtils.tokenize(paramString); for (String string : fields) { List<String> param = StringUtils.tokenize(string, ":"); Assert.isTrue(param.size() == 2, "Invalid param definition " + string); params.put(param.get(0), createFieldExtractor(param.get(1))); } }
private static List<String> qualifyNodes(String nodes, int defaultPort, boolean resolveHostNames) { List<String> list = StringUtils.tokenize(nodes); for (int i = 0; i < list.size(); i++) { String nodeIp = (resolveHostNames ? resolveHostToIpIfNecessary(list.get(i)) : list.get(i)); list.set(i, qualifyNode(nodeIp, defaultPort)); } return list; }
/** * Given a string of "application/x-www-form-urlencoded" format values, tokenize it and decode the values. * * @param string A "application/x-www-form-urlencoded" format string. * @param delimiters delimiters to use for tokenizing the given string. * @return List of decoded values (UTF-8) from the original string. */ public static List<String> tokenizeAndUriDecode(String string, String delimiters) { List<String> tokenize = StringUtils.tokenize(string, delimiters, true, true); List<String> decoded = new ArrayList<String>(tokenize.size()); for (String token : tokenize) { decoded.add(decode(token)); } return decoded; } }
static StandardStructObjectInspector structObjectInspector(Properties tableProperties) { // extract column info - don't use Hive constants as they were renamed in 0.9 breaking compatibility // the column names are saved as the given inspector to #serialize doesn't preserves them (maybe because it's an external table) // use the class since StructType requires it ... List<String> columnNames = StringUtils.tokenize(tableProperties.getProperty(HiveConstants.COLUMNS), ","); List<TypeInfo> colTypes = TypeInfoUtils.getTypeInfosFromTypeString(tableProperties.getProperty(HiveConstants.COLUMNS_TYPES)); // create a standard writable Object Inspector - used later on by serialization/deserialization List<ObjectInspector> inspectors = new ArrayList<ObjectInspector>(); for (TypeInfo typeInfo : colTypes) { inspectors.add(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo)); } return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors); }
/** * Seeks the field with the given name in the stream and positions (and returns) the parser to the next available token (value or not). * Return null if no token is found. * * @param path * @param parser * @return token associated with the given path or null if not found */ public static Token seek(Parser parser, String path) { // return current token if no path is given if (!StringUtils.hasText(path)) { return null; } List<String> tokens = StringUtils.tokenize(path, "."); return seek(parser, tokens.toArray(new String[tokens.size()])); }
@Override public void setSettings(Settings settings) { autoQuote = settings.getMappingConstantAutoQuote(); String fldName = property(settings); if (fldName.startsWith("<") && fldName.endsWith(">")) { value = initValue(fldName.substring(1, fldName.length() - 1)); } if (value == null) { fieldNames = StringUtils.tokenize(fldName, "."); processField(settings, fieldNames); } }
public static void validateMapping(String fields, Mapping mapping, FieldPresenceValidation validation, Log log) { if (StringUtils.hasText(fields)) { validateMapping(StringUtils.tokenize(fields), mapping, validation, log); } }
public static List<String> discoveredOrDeclaredNodes(Settings settings) { // returned the discovered nodes or, if not defined, the set nodes String discoveredNodes = settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_DISCOVERED_NODES); return (StringUtils.hasText(discoveredNodes) ? StringUtils.tokenize(discoveredNodes) : declaredNodes(settings)); }
public static List<NumberedInclude> getFieldArrayFilterInclude(Settings settings) { String includeString = settings.getReadFieldAsArrayInclude(); List<String> includes = StringUtils.tokenize(includeString); try { return FieldFilter.toNumberedFilter(includes); } catch (EsHadoopIllegalArgumentException iae) { throw new EsHadoopIllegalArgumentException("Failed to parse [" + ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE + "] option with value of [" + includeString + "]", iae); } }
public ScrollReaderConfigBuilder(Settings settings, ValueReader reader) { this.reader = reader; // Source defaults from Settings this.returnRawJson = settings.getOutputAsJson(); this.ignoreUnmappedFields = settings.getReadMappingMissingFieldsIgnore(); this.readMetadata = settings.getReadMetadata(); this.metadataName = settings.getReadMetadataField(); this.includeFields = StringUtils.tokenize(settings.getReadFieldInclude()); this.excludeFields = StringUtils.tokenize(settings.getReadFieldExclude()); this.includeArrayFields = StringUtils.tokenize(settings.getReadFieldAsArrayInclude()); DeserializationHandlerLoader loader = new DeserializationHandlerLoader(); loader.setSettings(settings); this.errorHandlerLoader = loader; // No default value this.resolvedMapping = null; }
@Override public void setSettings(Settings settings) { emptyAsNull = settings.getReadFieldEmptyAsNull(); richDate = settings.getMappingDateRich(); arrayInclude = SettingsUtils.getFieldArrayFilterInclude(settings); arrayExclude = StringUtils.tokenize(settings.getReadFieldAsArrayExclude()); } }