@Override public BufferedMutator getBufferedMutator(TableName tableName) throws IOException { return this.conn.getBufferedMutator(tableName); }
@Override public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws IOException { return this.conn.getBufferedMutator(params); }
@Override void testSetup() throws IOException { this.mutator = connection.getBufferedMutator(tableName); }
private MyRecordWriter getMyRecordWriter(JobConf jobConf) throws IOException { String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME); jobConf.set(TableOutputFormat.OUTPUT_TABLE, hbaseTableName); final boolean walEnabled = HiveConf.getBoolVar( jobConf, HiveConf.ConfVars.HIVE_HBASE_WAL_ENABLED); final Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create(jobConf)); final BufferedMutator table = conn.getBufferedMutator(TableName.valueOf(hbaseTableName)); return new MyRecordWriter(table, conn, walEnabled); }
/** * Instantiate a TableRecordWriter with a BufferedMutator for batch writing. */ public TableRecordWriter(JobConf job) throws IOException { // expecting exactly one path TableName tableName = TableName.valueOf(job.get(OUTPUT_TABLE)); try { this.conn = ConnectionFactory.createConnection(job); this.m_mutator = conn.getBufferedMutator(tableName); } finally { if (this.m_mutator == null) { conn.close(); conn = null; } } }
/** * @param tableName * the name of the table, as a string * @return the named mutator * @throws IOException * if there is a problem opening a table */ BufferedMutator getBufferedMutator(ImmutableBytesWritable tableName) throws IOException { if(this.connection == null){ this.connection = ConnectionFactory.createConnection(conf); } if (!mutatorMap.containsKey(tableName)) { LOG.debug("Opening HTable \"" + Bytes.toString(tableName.get())+ "\" for writing"); BufferedMutator mutator = connection.getBufferedMutator(TableName.valueOf(tableName.get())); mutatorMap.put(tableName, mutator); } return mutatorMap.get(tableName); }
public void getHTable(String table) throws IOException { final TableName tName = TableName.valueOf(table); this.currentTable = connection.getTable(tName); if (clientSideBuffering) { final BufferedMutatorParams p = new BufferedMutatorParams(tName); p.writeBufferSize(writeBufferSize); this.bufferedMutator = connection.getBufferedMutator(p); } }
Writer() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); table = conn.getBufferedMutator(htableName); }
@Override void onStartup() throws IOException { BufferedMutatorParams p = new BufferedMutatorParams(TableName.valueOf(opts.tableName)); p.writeBufferSize(opts.bufferSize); this.mutator = connection.getBufferedMutator(p); this.table = connection.getTable(TableName.valueOf(opts.tableName)); }
public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows, byte[]... families) throws IOException, InterruptedException { BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName); loadData(util, mutator, rows, families); }
@Override public void setup(Context context) throws IOException { conf = context.getConfiguration(); recordsToWrite = conf.getLong(NUM_TO_WRITE_KEY, NUM_TO_WRITE_DEFAULT); String tableName = conf.get(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); numBackReferencesPerRow = conf.getInt(NUM_BACKREFS_KEY, NUM_BACKREFS_DEFAULT); this.connection = ConnectionFactory.createConnection(conf); mutator = connection.getBufferedMutator( new BufferedMutatorParams(TableName.valueOf(tableName)) .writeBufferSize(4 * 1024 * 1024)); String taskId = conf.get("mapreduce.task.attempt.id"); Matcher matcher = Pattern.compile(".+_m_(\\d+_\\d+)").matcher(taskId); if (!matcher.matches()) { throw new RuntimeException("Strange task ID: " + taskId); } shortTaskId = matcher.group(1); rowsWritten = context.getCounter(Counters.ROWS_WRITTEN); refsWritten = context.getCounter(Counters.REFERENCES_WRITTEN); }
private void setUpForPolicyTest(String tableNameAsString, MobCompactPartitionPolicy type) throws IOException { tableName = TableName.valueOf(tableNameAsString); hcd1 = new HColumnDescriptor(family1); hcd1.setMobEnabled(true); hcd1.setMobThreshold(10); hcd1.setMobCompactPartitionPolicy(type); desc = new HTableDescriptor(tableName); desc.addFamily(hcd1); admin.createTable(desc); table = conn.getTable(tableName); bufMut = conn.getBufferedMutator(tableName); }
@Test public void testAlternateBufferedMutatorImpl() throws IOException { BufferedMutatorParams params = new BufferedMutatorParams(TableName.valueOf(name.getMethodName())); Configuration conf = HBaseConfiguration.create(); conf.set(AsyncRegistryFactory.REGISTRY_IMPL_CONF_KEY, DoNothingAsyncRegistry.class.getName()); try (Connection connection = ConnectionFactory.createConnection(conf)) { BufferedMutator bm = connection.getBufferedMutator(params); // Assert we get default BM if nothing specified. assertTrue(bm instanceof BufferedMutatorImpl); // Now try and set my own BM implementation. params.implementationClassName(MyBufferedMutator.class.getName()); bm = connection.getBufferedMutator(params); assertTrue(bm instanceof MyBufferedMutator); } // Now try creating a Connection after setting an alterate BufferedMutator into // the configuration and confirm we get what was expected. conf.set(BufferedMutator.CLASSNAME_KEY, MyBufferedMutator.class.getName()); try (Connection connection = ConnectionFactory.createConnection(conf)) { BufferedMutator bm = connection.getBufferedMutator(params); assertTrue(bm instanceof MyBufferedMutator); } } }
public void setUp(String tableNameAsString) throws IOException { tableName = TableName.valueOf(tableNameAsString); hcd1 = new HColumnDescriptor(family1); hcd1.setMobEnabled(true); hcd1.setMobThreshold(5); hcd2 = new HColumnDescriptor(family2); hcd2.setMobEnabled(true); hcd2.setMobThreshold(5); desc = new HTableDescriptor(tableName); desc.addFamily(hcd1); desc.addFamily(hcd2); admin.createTable(desc, getSplitKeys()); table = conn.getTable(tableName); bufMut = conn.getBufferedMutator(tableName); }
@Override protected void instantiateHTable() throws IOException { for (int i = 0; i < DEFAULT_TABLES_COUNT; i++) { BufferedMutatorParams params = new BufferedMutatorParams(getTableName(i)); params.writeBufferSize(4 * 1024 * 1024); BufferedMutator table = connection.getBufferedMutator(params); this.tables[i] = table; } }
private LinkedBlockingQueue<Long> insertData() throws IOException, InterruptedException { LinkedBlockingQueue<Long> rowKeys = new LinkedBlockingQueue<>(25000); BufferedMutator ht = util.getConnection().getBufferedMutator(this.tableName); byte[] value = new byte[300]; TraceUtil.addSampler(Sampler.ALWAYS); for (int x = 0; x < 5000; x++) { try (TraceScope traceScope = TraceUtil.createTrace("insertData")) { for (int i = 0; i < 5; i++) { long rk = random.nextLong(); rowKeys.add(rk); Put p = new Put(Bytes.toBytes(rk)); for (int y = 0; y < 10; y++) { random.nextBytes(value); p.addColumn(familyName, Bytes.toBytes(random.nextLong()), value); } ht.mutate(p); } if ((x % 1000) == 0) { admin.flush(tableName); } } } admin.flush(tableName); return rowKeys; }
private void init() throws Exception { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(family); hcd.setMobEnabled(true); hcd.setMobThreshold(3L); hcd.setMaxVersions(4); desc.addFamily(hcd); admin = TEST_UTIL.getAdmin(); admin.createTable(desc); table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()) .getBufferedMutator(tableName); }
private void migrateNamespaceTable() throws IOException { try (Table nsTable = masterServices.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME); ResultScanner scanner = nsTable.getScanner( new Scan().addFamily(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES).readAllVersions()); BufferedMutator mutator = masterServices.getConnection().getBufferedMutator(TableName.META_TABLE_NAME)) { for (Result result;;) { result = scanner.next(); if (result == null) { break; } Put put = new Put(result.getRow()); result .getColumnCells(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES, TableDescriptorBuilder.NAMESPACE_COL_DESC_BYTES) .forEach(c -> put.addColumn(HConstants.NAMESPACE_FAMILY, HConstants.NAMESPACE_COL_DESC_QUALIFIER, c.getTimestamp(), CellUtil.cloneValue(c))); mutator.mutate(put); } } // schedule a disable procedure instead of block waiting here, as when disabling a table we will // wait until master is initialized, but we are part of the initialization... masterServices.getMasterProcedureExecutor().submitProcedure( new DisableTableProcedure(masterServices.getMasterProcedureExecutor().getEnvironment(), TableName.NAMESPACE_TABLE_NAME, false)); }
protected void instantiateHTable() throws IOException { mutator = connection.getBufferedMutator( new BufferedMutatorParams(getTableName(connection.getConfiguration())) .writeBufferSize(4 * 1024 * 1024)); }
public static void loadData(final Connection connection, final TableName tableName, int rows, final byte[][] splitKeys, final String... sfamilies) throws IOException { byte[][] families = new byte[sfamilies.length][]; for (int i = 0; i < families.length; ++i) { families[i] = Bytes.toBytes(sfamilies[i]); } BufferedMutator mutator = connection.getBufferedMutator(tableName); // Ensure one row per region assertTrue(rows >= splitKeys.length); for (byte[] k: splitKeys) { byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k); byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value))); mutator.mutate(createPut(families, key, value)); rows--; } // Add other extra rows. more rows, more files while (rows-- > 0) { byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); mutator.mutate(createPut(families, key, value)); } mutator.flush(); }