) throws IOException { KeyClassOption keyClassOption = Options.getOption(KeyClassOption.class, opts); ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); if ((keyClassOption == null) == (comparatorOption == null)) { throw new IllegalArgumentException("key class or comparator option " Options.prependOptions(opts, SequenceFile.Writer.file(dataFile), SequenceFile.Writer.keyClass(keyClass)); Options.prependOptions(opts, SequenceFile.Writer.file(indexFile), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(LongWritable.class),
public Reader(Path dir, Configuration conf, SequenceFile.Reader.Option... opts) throws IOException { ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); WritableComparator comparator = comparatorOption == null ? null : comparatorOption.getValue(); INDEX_SKIP = conf.getInt( IO_MAP_INDEX_SKIP_KEY, IO_MAP_INDEX_SKIP_DEFAULT); open(dir, comparator, conf, opts); }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
public Reader(Configuration conf, Option... opts) throws IOException { FileOption fileOpt = Options.getOption(FileOption.class, opts); InputStreamOption streamOpt = Options.getOption(InputStreamOption.class, opts); StartOption startOpt = Options.getOption(StartOption.class, opts); LengthOption lenOpt = Options.getOption(LengthOption.class, opts); BufferSizeOption bufOpt = Options.getOption(BufferSizeOption.class,opts); OnlyHeaderOption headerOnly = Options.getOption(OnlyHeaderOption.class, opts);
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
/** * Create a new Writer with the given options. * @param conf the configuration to use * @param opts the options to create the file with * @return a new Writer * @throws IOException */ public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } }
Option... opts) throws IOException { BlockSizeOption blockSizeOption = Options.getOption(BlockSizeOption.class, opts); BufferSizeOption bufferSizeOption = Options.getOption(BufferSizeOption.class, opts); ReplicationOption replicationOption = Options.getOption(ReplicationOption.class, opts); ProgressableOption progressOption = Options.getOption(ProgressableOption.class, opts); FileOption fileOption = Options.getOption(FileOption.class, opts); AppendIfExistsOption appendIfExistsOption = Options.getOption( AppendIfExistsOption.class, opts); FileSystemOption fsOption = Options.getOption(FileSystemOption.class, opts); StreamOption streamOption = Options.getOption(StreamOption.class, opts); KeyClassOption keyClassOption = Options.getOption(KeyClassOption.class, opts); ValueClassOption valueClassOption = Options.getOption(ValueClassOption.class, opts); MetadataOption metadataOption = Options.getOption(MetadataOption.class, opts); CompressionOption compressionTypeOption = Options.getOption(CompressionOption.class, opts); SyncIntervalOption syncIntervalOption = Options.getOption(SyncIntervalOption.class, opts);
@Test public void testAppend() throws Exception { assertArrayEquals("first append", new String[]{"Dr.", "Who", "hi", "there"}, Options.prependOptions(new String[]{"hi", "there"}, "Dr.", "Who")); assertArrayEquals("second append", new String[]{"aa","bb","cc","dd","ee","ff"}, Options.prependOptions(new String[]{"dd", "ee", "ff"}, "aa", "bb", "cc")); }
) throws IOException { KeyClassOption keyClassOption = Options.getOption(KeyClassOption.class, opts); ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); if ((keyClassOption == null) == (comparatorOption == null)) { throw new IllegalArgumentException("key class or comparator option " Options.prependOptions(opts, SequenceFile.Writer.file(dataFile), SequenceFile.Writer.keyClass(keyClass)); Options.prependOptions(opts, SequenceFile.Writer.file(indexFile), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(LongWritable.class),
@Test public void testFind() throws Exception { Object[] opts = new Object[]{1, "hi", true, "bye", 'x'}; assertEquals(1, Options.getOption(Integer.class, opts).intValue()); assertEquals("hi", Options.getOption(String.class, opts)); assertEquals(true, Options.getOption(Boolean.class, opts).booleanValue()); } }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
) throws IOException { KeyClassOption keyClassOption = Options.getOption(KeyClassOption.class, opts); ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); if ((keyClassOption == null) == (comparatorOption == null)) { throw new IllegalArgumentException("key class or comparator option " Options.prependOptions(opts, SequenceFile.Writer.file(dataFile), SequenceFile.Writer.keyClass(keyClass)); Options.prependOptions(opts, SequenceFile.Writer.file(indexFile), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(LongWritable.class),
public Reader(Path dir, Configuration conf, SequenceFile.Reader.Option... opts) throws IOException { ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); WritableComparator comparator = comparatorOption == null ? null : comparatorOption.getValue(); INDEX_SKIP = conf.getInt( IO_MAP_INDEX_SKIP_KEY, IO_MAP_INDEX_SKIP_DEFAULT); open(dir, comparator, conf, opts); }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
) throws IOException { KeyClassOption keyClassOption = Options.getOption(KeyClassOption.class, opts); ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); if ((keyClassOption == null) == (comparatorOption == null)) { throw new IllegalArgumentException("key class or comparator option " Options.prependOptions(opts, SequenceFile.Writer.file(dataFile), SequenceFile.Writer.keyClass(keyClass)); Options.prependOptions(opts, SequenceFile.Writer.file(indexFile), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(LongWritable.class),
public Reader(Path dir, Configuration conf, SequenceFile.Reader.Option... opts) throws IOException { ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); WritableComparator comparator = comparatorOption == null ? null : comparatorOption.getValue(); INDEX_SKIP = conf.getInt("io.map.index.skip", 0); open(dir, comparator, conf, opts); }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
) throws IOException { KeyClassOption keyClassOption = Options.getOption(KeyClassOption.class, opts); ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); if ((keyClassOption == null) == (comparatorOption == null)) { throw new IllegalArgumentException("key class or comparator option " Options.prependOptions(opts, SequenceFile.Writer.file(dataFile), SequenceFile.Writer.keyClass(keyClass)); Options.prependOptions(opts, SequenceFile.Writer.file(indexFile), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(LongWritable.class),
public Reader(Path dir, Configuration conf, SequenceFile.Reader.Option... opts) throws IOException { ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); WritableComparator comparator = comparatorOption == null ? null : comparatorOption.getValue(); INDEX_SKIP = conf.getInt("io.map.index.skip", 0); open(dir, comparator, conf, opts); }
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }