MockRecordWriter(Path basedir, AcidOutputFormat.Options options) throws IOException { FileSystem fs = FileSystem.get(options.getConfiguration()); Path p = AcidUtils.createFilename(basedir, options); os = fs.create(p); }
@Override public void flush() throws IOException { if (out == null) { FileSystem fs = path.getFileSystem(options.getConfiguration()); out = fs.create(path); } for (Writable r : records) r.write(out); records.clear(); out.flush(); }
@Test public void testCreatesRecordReader() throws IOException { verify(mockOutputFormat).getRecordUpdater(eq(PATH), captureOptions.capture()); Options options = captureOptions.getValue(); assertThat(options.getBucketId(), is(BUCKET_ID)); assertThat(options.getConfiguration(), is((Configuration) configuration)); assertThat(options.getInspector(), is(mockObjectInspector)); assertThat(options.getRecordIdColumn(), is(RECORD_ID_COLUMN)); assertThat(options.getMinimumWriteId(), is(WRITE_ID)); assertThat(options.getMaximumWriteId(), is(WRITE_ID)); }
FileSystem fs = options.getFilesystem(); if (fs == null) { fs = path.getFileSystem(options.getConfiguration()); writerOptions = OrcFile.writerOptions(options.getConfiguration());
final Path filename = AcidUtils.createFilename(path, options); final OrcFile.WriterOptions opts = OrcFile.writerOptions(options.getConfiguration()); if (!options.isWritingBase()) { opts.bufferSize(OrcRecordUpdater.DELTA_BUFFER_SIZE)
public PhoenixRecordUpdater(Path path, AcidOutputFormat.Options options) throws IOException { this.config = options.getConfiguration(); tableName = config.get(PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME);
public PhoenixRecordWriter(Path path, AcidOutputFormat.Options options) throws IOException { Configuration config = options.getConfiguration(); Properties props = new Properties(); try { initialize(config, props); } catch (SQLException e) { throw new IOException(e); } this.objInspector = options.getInspector(); try { phoenixSerializer = new PhoenixSerializer(config, options.getTableProperties()); } catch (SerDeException e) { throw new IOException(e); } }
} else { this.acidOperationalProperties = AcidUtils.getAcidOperationalProperties(options.getConfiguration()); FileSystem fs = options.getFilesystem(); if (fs == null) { fs = path.getFileSystem(options.getConfiguration()); options.getConfiguration()); optionsCloneForDelta.getConfiguration()); int ratio = (int) OrcConf.BASE_DELTA_RATIO.getLong(options.getConfiguration()); writerOptions.bufferSize(baseBufferSizeValue / ratio); writerOptions.stripeSize(baseStripeSizeValue / ratio);
} else { this.acidOperationalProperties = AcidUtils.getAcidOperationalProperties(options.getConfiguration()); if(options.getConfiguration() != null) { Configuration hc = options.getConfiguration(); if(hc.getBoolean(HiveConf.ConfVars.HIVE_IN_TEST.name(), false) || hc.getBoolean(HiveConf.ConfVars.HIVE_IN_TEZ_TEST.name(), false)) { FileSystem fs = options.getFilesystem(); if (fs == null) { fs = partitionRoot.getFileSystem(options.getConfiguration()); options.getConfiguration()); optionsCloneForDelta.getConfiguration()); optionsCloneForDelta.getConfiguration()); this.deleteWriterOptions.inspector(createEventObjectInspector(findRecId(options.getInspector(), options.getRecordIdColumn()))); int ratio = (int) OrcConf.BASE_DELTA_RATIO.getLong(options.getConfiguration()); writerOptions.bufferSize(baseBufferSizeValue / ratio); writerOptions.stripeSize(baseStripeSizeValue / ratio); writerOptions.blockPadding(false); if (optionsCloneForDelta.getConfiguration().getBoolean( HiveConf.ConfVars.HIVE_ORC_DELTA_STREAMING_OPTIMIZATIONS_ENABLED.varname, false)) { writerOptions.encodingStrategy(org.apache.orc.OrcFile.EncodingStrategy.SPEED);
final Path filename = AcidUtils.createFilename(path, options); final OrcFile.WriterOptions opts = OrcFile.writerOptions(options.getConfiguration()); if (!options.isWritingBase()) { opts.bufferSize(OrcRecordUpdater.DELTA_BUFFER_SIZE)
continue; Reader reader = OrcFile.createReader(matchingBucket, OrcFile.readerOptions(options.getConfiguration()));
final Path filename = AcidUtils.createFilename(path, options); final OrcFile.WriterOptions opts = OrcFile.writerOptions(options.getTableProperties(), options.getConfiguration()); if (!options.isWritingBase()) { opts.bufferSize(OrcRecordUpdater.DELTA_BUFFER_SIZE) .stripeSize(OrcRecordUpdater.DELTA_STRIPE_SIZE) .blockPadding(false); if(!MetastoreConf.getBoolVar(options.getConfiguration(), MetastoreConf.ConfVars.COMPACTOR_MINOR_STATS_COMPRESSION)) { opts.compress(CompressionKind.NONE).rowIndexStride(0);