private static Schema getSchema(CombineFileSplit split, TaskAttemptContext cx, Integer idx) throws IOException { Schema schema = AvroJob.getInputKeySchema(cx.getConfiguration()); if (schema != null) { return schema; } Path path = split.getPath(idx); FileSystem fs = path.getFileSystem(cx.getConfiguration()); return AvroUtils.getSchemaFromDataFile(path, fs); }
public static Schema getNewestSchemaFromSource(Path sourceDir, FileSystem fs) throws IOException { FileStatus[] files = fs.listStatus(sourceDir); Arrays.sort(files, new LastModifiedDescComparator()); for (FileStatus status : files) { if (status.isDirectory()) { Schema schema = getNewestSchemaFromSource(status.getPath(), fs); if (schema != null) return schema; } else if (FilenameUtils.isExtension(status.getPath().getName(), AVRO)) { return AvroUtils.getSchemaFromDataFile(status.getPath(), fs); } } return null; }