private String autoDetectDelim(String line) { if (expectedColumnNumber > 0) { for (String delim : DETECT_DELIMS) { if (StringSplitter.split(line, delim).length == expectedColumnNumber) { logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line); return delim; } } } logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath); return null; }
public static int addInputDirs(String input, Job job) throws IOException { int folderNum = addInputDirs(StringSplitter.split(input, ","), job); logger.info("Number of added folders:" + folderNum); return folderNum; }
private String[] split(String line, String delim) { // FIXME CVS line should be parsed considering escapes String[] str = StringSplitter.split(line, delim); // un-escape CSV if (DFSFileTable.DELIM_COMMA.equals(delim)) { for (int i = 0; i < str.length; i++) { str[i] = unescapeCsv(str[i]); } } return str; }
public void setName(String name) { if (name != null) { String[] splits = StringSplitter.split(name, "."); if (splits.length == 2) { this.setDatabase(splits[0]); this.name = splits[1]; } else if (splits.length == 1) { this.name = splits[0]; } } else { this.name = null; } }
final String[] inputFolders = StringSplitter.split(inputPath, ","); FileSystem fs = HadoopUtil.getWorkingFileSystem(); boolean isLegacyMode = false;
private String autoDetectDelim(String line) { if (expectedColumnNumber > 0) { for (String delim : DETECT_DELIMS) { if (StringSplitter.split(line, delim).length == expectedColumnNumber) { logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line); return delim; } } } logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath); return null; }
private String[] split(String line, String delim) { // FIXME CVS line should be parsed considering escapes String str[] = StringSplitter.split(line, delim); // un-escape CSV if (ReadableTable.DELIM_COMMA.equals(delim)) { for (int i = 0; i < str.length; i++) { str[i] = unescapeCsv(str[i]); } } return str; }
public void setName(String name) { if (name != null) { String[] splits = StringSplitter.split(name, "."); if (splits.length == 2) { this.setDatabase(splits[0]); this.name = splits[1]; } else if (splits.length == 1) { this.name = splits[0]; } } else { this.name = name; } }
public void init(Map<String, TableDesc> tables) { if (StringUtils.isNotEmpty(partitionDateColumn)) { partitionDateColumn = partitionDateColumn.toUpperCase(); String[] columns = StringSplitter.split(partitionDateColumn, "."); if (null != columns && columns.length == 3) { String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase(); TableDesc table = tables.get(tableName); ColumnDesc col = table.findColumnByName(columns[2]); if (col != null) { partitionDateColumnRef = new TblColRef(col); } else { throw new IllegalStateException("The column '" + partitionDateColumn + "' provided in 'partition_date_column' doesn't exist."); } } else { throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}."); } } }
public void addInputDirs(String input, Job job) throws IOException { for (String inp : StringSplitter.split(input, ",")) { inp = inp.trim(); if (inp.endsWith("/*")) { inp = inp.substring(0, inp.length() - 2); FileSystem fs = FileSystem.get(job.getConfiguration()); Path path = new Path(inp); FileStatus[] fileStatuses = fs.listStatus(path); boolean hasDir = false; for (FileStatus stat : fileStatuses) { if (stat.isDirectory() && !stat.getPath().getName().startsWith("_")) { hasDir = true; addInputDirs(stat.getPath().toString(), job); } } if (fileStatuses.length > 0 && !hasDir) { addInputDirs(path.toString(), job); } } else { logger.debug("Add input " + inp); FileInputFormat.addInputPath(job, new Path(inp)); } } }
private String autoDetectDelim(String line) { if (expectedColumnNumber > 0) { for (String delim : DETECT_DELIMS) { if (StringSplitter.split(line, delim).length == expectedColumnNumber) { logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line); return delim; } } } logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath); return null; }
private String autoDetectDelim(String line) { if (expectedColumnNumber > 0) { for (String delim : DETECT_DELIMS) { if (StringSplitter.split(line, delim).length == expectedColumnNumber) { logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line); return delim; } } } logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath); return null; }
public void setName(String name) { if (name != null) { String[] splits = StringSplitter.split(name, "."); if (splits.length == 2) { this.setDatabase(splits[0]); this.name = splits[1]; } else if (splits.length == 1) { this.name = splits[0]; } } else { this.name = name; } }
private String[] split(String line, String delim) { // FIXME CVS line should be parsed considering escapes String[] str = StringSplitter.split(line, delim); // un-escape CSV if (DFSFileTable.DELIM_COMMA.equals(delim)) { for (int i = 0; i < str.length; i++) { str[i] = unescapeCsv(str[i]); } } return str; }
public static int addInputDirs(String input, Job job) throws IOException { int folderNum = addInputDirs(StringSplitter.split(input, ","), job); logger.info("Number of added folders:" + folderNum); return folderNum; }
private String[] split(String line, String delim) { // FIXME CVS line should be parsed considering escapes String str[] = StringSplitter.split(line, delim); // un-escape CSV if (FileTable.DELIM_COMMA.equals(delim)) { for (int i = 0; i < str.length; i++) { str[i] = unescapeCsv(str[i]); } } return str; }
public void setName(String name) { if (name != null) { String[] splits = StringSplitter.split(name, "."); if (splits.length == 2) { this.setDatabase(splits[0]); this.name = splits[1]; } else if (splits.length == 1) { this.name = splits[0]; } } else { this.name = null; } }
public void addInputDirs(String input, Job job) throws IOException { for (String inp : StringSplitter.split(input, ",")) { inp = inp.trim(); if (inp.endsWith("/*")) { inp = inp.substring(0, inp.length() - 2); FileSystem fs = FileSystem.get(job.getConfiguration()); Path path = new Path(inp); FileStatus[] fileStatuses = fs.listStatus(path); boolean hasDir = false; for (FileStatus stat : fileStatuses) { if (stat.isDirectory() && !stat.getPath().getName().startsWith("_")) { hasDir = true; addInputDirs(stat.getPath().toString(), job); } } if (fileStatuses.length > 0 && !hasDir) { addInputDirs(path.toString(), job); } } else { logger.debug("Add input " + inp); FileInputFormat.addInputPath(job, new Path(inp)); } } }
public void init(Map<String, TableDesc> tables) { if (StringUtils.isEmpty(partitionDateColumn)) return; partitionDateColumn = partitionDateColumn.toUpperCase(); String[] columns = StringSplitter.split(partitionDateColumn, "."); if (null != columns && columns.length == 3) { String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase(); TableDesc table = tables.get(tableName); ColumnDesc col = table.findColumnByName(columns[2]); if (col != null) { partitionDateColumnRef = new TblColRef(col); } else { throw new IllegalStateException("The column '" + partitionDateColumn + "' provided in 'partition_date_column' doesn't exist."); } } else { throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}."); } partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz); }
final String[] inputFolders = StringSplitter.split(inputPath, ","); FileSystem fs = HadoopUtil.getWorkingFileSystem(); boolean isLegacyMode = false;