/** * Read the metadata from a hadoop SequenceFile * * @param fs The filesystem to read from * @param path The file to read from * @return The metadata from this file */ public static Map<String, String> getMetadataFromSequenceFile(FileSystem fs, Path path) { try { Configuration conf = new Configuration(); conf.setInt("io.file.buffer.size", 4096); SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, new Configuration()); SequenceFile.Metadata meta = reader.getMetadata(); reader.close(); TreeMap<Text, Text> map = meta.getMetadata(); Map<String, String> values = new HashMap<String, String>(); for(Map.Entry<Text, Text> entry: map.entrySet()) values.put(entry.getKey().toString(), entry.getValue().toString()); return values; } catch(IOException e) { throw new RuntimeException(e); } }
@Override public RecordReader<BytesWritable, BytesWritable> getRecordReader(InputSplit split, JobConf conf, Reporter reporter) throws IOException { String inputPathString = ((FileSplit) split).getPath().toUri().getPath(); log.info("Input file path:" + inputPathString); Path inputPath = new Path(inputPathString); SequenceFile.Reader reader = new SequenceFile.Reader(inputPath.getFileSystem(conf), inputPath, conf); SequenceFile.Metadata meta = reader.getMetadata(); try { Text keySchema = meta.get(new Text("key.schema")); Text valueSchema = meta.get(new Text("value.schema")); if(0 == keySchema.getLength() || 0 == valueSchema.getLength()) { throw new Exception(); } // update Joboconf with schemas conf.set("mapper.input.key.schema", keySchema.toString()); conf.set("mapper.input.value.schema", valueSchema.toString()); } catch(Exception e) { throw new IOException("Failed to Load Schema from file:" + inputPathString + "\n"); } return super.getRecordReader(split, conf, reporter); }
.metadata(reader.getMetadata());
sequenceFileMetaData = reader.getMetadata();
final Text uuidText = reader.getMetadata().get(new Text(MetadataConfiguration.UUID_KEY)); if (uuidText != null) uuid = uuidText.toString();
/** * Return the metadata map. Read mode only. * * @return metadata */ public Map<Text, Text> getMetadata() { if (!isReader) { throw new UnsupportedOperationException("Cannot read metadata in write mode"); } Reader reader = null; try { reader = createReader(); final Map<Text, Text> metadata = reader.getMetadata().getMetadata(); return metadata; } catch (final Exception e) { throw new RuntimeException(e); } finally { if (reader != null) try { reader.close(); } catch (final IOException e1) { } } }
/** * Return the metadata map. Read mode only. * * @return metadata */ public Map<Text, Text> getMetadata() { if (!isReader) { throw new UnsupportedOperationException("Cannot read metadata in write mode"); } Reader reader = null; try { reader = createReader(); final Map<Text, Text> metadata = reader.getMetadata().getMetadata(); return metadata; } catch (final Exception e) { throw new RuntimeException(e); } finally { if (reader != null) try { reader.close(); } catch (final IOException e1) { } } }
/** * Open and read just the metadata header from a SequenceFile. * * @param fs The FileSystem the SequenceFile is on. * @param path The path to the file. * @param conf The Hadoop configuration. * @return The metadata header. * @throws IOException If the metadata cannot be read from the file. */ private static Metadata getMetadata(FileSystem fs, Path path, Configuration conf) throws IOException { SequenceFile.Reader metadataReader = null; try { metadataReader = new SequenceFile.Reader(fs, path, conf); return metadataReader.getMetadata(); } finally { if (null != metadataReader) { metadataReader.close(); } } } }
@Override protected boolean hasCompression() { return isWALCompressionEnabled(reader.getMetadata()); }
/** * Returns the metadata object of the file */ public SequenceFile.Metadata getMetadata() { return innerReader.getMetadata(); }
private void loadSchema() throws IOException { SequenceFile.Metadata meta = innerReader.getMetadata(); Text schemaText = meta.get(new Text("schema")); if (schemaText == null) { throw new IOException("Invalid Sequence File with Tuples [" + file + "] : it does not contain the tuple's schema in the metadata"); } try { schema = Schema.parse(schemaText.toString()); } catch (Schema.SchemaParseException e) { throw new IOException("Invalid Schema found in file: " + file + ". Schema: " + schemaText.toString()); } }
private SequenceFile.Metadata readMetadata(FileSystem fs, Path file) throws IOException { LOG.info("reading file: " + file.toString()); SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf); SequenceFile.Metadata meta = reader.getMetadata(); reader.close(); return meta; }
public HDFSTransactionLogReaderSupplier(SequenceFile.Reader reader) { this.reader = reader; Text versionInfo = reader.getMetadata().get(new Text(TxConstants.TransactionLog.VERSION_KEY)); this.version = versionInfo == null ? 1 : Byte.parseByte(versionInfo.toString()); }
public HDFSTransactionLogReaderSupplier(SequenceFile.Reader reader) { this.reader = reader; Text versionInfo = reader.getMetadata().get(new Text(TxConstants.TransactionLog.VERSION_KEY)); this.version = versionInfo == null ? 1 : Byte.parseByte(versionInfo.toString()); }
/** * Returns the metadata object of the file */ public SequenceFile.Metadata getMetadata() { return innerReader.getMetadata(); }
private static SequenceFile.Metadata readMetadata(FileSystem fs, Path file) throws IOException { LOG.info("reading file: " + file.toString()); SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf); SequenceFile.Metadata meta = reader.getMetadata(); reader.close(); return meta; }
public HDFSTransactionLogReaderSupplier(SequenceFile.Reader reader) { this.reader = reader; Text versionInfo = reader.getMetadata().get(new Text(TxConstants.TransactionLog.VERSION_KEY)); this.version = versionInfo == null ? 1 : Byte.parseByte(versionInfo.toString()); }
private void loadSchema() throws IOException { SequenceFile.Metadata meta = innerReader.getMetadata(); Text schemaText = meta.get(new Text("schema")); if (schemaText == null) { throw new IOException("Invalid Sequence File with Tuples [" + file + "] : it does not contain the tuple's schema in the metadata"); } try { schema = Schema.parse(schemaText.toString()); } catch (Schema.SchemaParseException e) { throw new IOException("Invalid Schema found in file: " + file + ". Schema: " + schemaText.toString()); } }
private SequenceFile.Metadata readMetadata(FileSystem fs, Path file) throws IOException { LOG.info("reading file: " + file.toString()); SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf); SequenceFile.Metadata meta = reader.getMetadata(); reader.close(); return meta; }
/** * Open and read just the metadata header from a SequenceFile. * * @param fs The FileSystem the SequenceFile is on. * @param path The path to the file. * @param conf The Hadoop configuration. * @return The metadata header. * @throws IOException If the metadata cannot be read from the file. */ private static Metadata getMetadata(FileSystem fs, Path path, Configuration conf) throws IOException { SequenceFile.Reader metadataReader = null; try { metadataReader = new SequenceFile.Reader(fs, path, conf); return metadataReader.getMetadata(); } finally { if (null != metadataReader) { metadataReader.close(); } } } }