public void write(ImmutableBytesWritable key, Put value) throws IOException { m_mutator.mutate(new Put(value)); } }
@Override public Void call() throws Exception { // // step 2: each worker sends edits to the shared BufferedMutator instance. They all use // the same backing buffer, call-back "listener", and RPC executor pool. // Put p = new Put(Bytes.toBytes("someRow")); p.addColumn(FAMILY, Bytes.toBytes("someQualifier"), Bytes.toBytes("some value")); mutator.mutate(p); // do work... maybe you want to call mutator.flush() after many edits to ensure any of // this worker's edits are sent before exiting the Callable return null; } }));
/** * Writes an action (Put or Delete) to the specified table. * * @param tableName * the table being updated. * @param action * the update, either a put or a delete. * @throws IllegalArgumentException * if the action is not a put or a delete. */ @Override public void write(ImmutableBytesWritable tableName, Mutation action) throws IOException { BufferedMutator mutator = getBufferedMutator(tableName); // The actions are not immutable, so we defensively copy them if (action instanceof Put) { Put put = new Put((Put) action); put.setDurability(useWriteAheadLogging ? Durability.SYNC_WAL : Durability.SKIP_WAL); mutator.mutate(put); } else if (action instanceof Delete) { Delete delete = new Delete((Delete) action); mutator.mutate(delete); } else throw new IllegalArgumentException( "action must be either Delete or Put"); } }
public void write(ImmutableBytesWritable key, Object value) throws IOException { Put put; if (value instanceof Put){ put = (Put)value; } else if (value instanceof PutWritable) { put = new Put(((PutWritable)value).getPut()); } else { throw new IllegalArgumentException("Illegal Argument " + (value == null ? "null" : value.getClass().getName())); } if(m_walEnabled) { put.setDurability(Durability.SYNC_WAL); } else { put.setDurability(Durability.SKIP_WAL); } m_table.mutate(put); }
@Override public void write(GTRecord rec) throws IOException { assert info.getColumnBlockCount() == 2; rowkey.clear(); for (int i = 0; i < ID_LEN; i++) { rowkey.put((byte) 0); } rec.exportColumnBlock(0, rowkey); rowkey.flip(); value.clear(); rec.exportColumnBlock(1, value); value.flip(); Put put = new Put(rowkey); put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value); table.mutate(put); }
private void loadData(Admin admin, BufferedMutator table, TableName tableName, Put[] puts) throws IOException { table.mutate(Arrays.asList(puts)); table.flush(); admin.flush(tableName); }
private void putKVAndFlush(BufferedMutator table, byte[] row, byte[] value, long ts) throws Exception { Put put = new Put(row, ts); put.addColumn(Bytes.toBytes(family), qf, value); table.mutate(put); table.flush(); admin.flush(tableName); }
public static void generateHBaseDatasetNullStr(Connection conn, Admin admin, TableName tableName, int numberRegions) throws Exception { if (admin.tableExists(tableName)) { admin.disableTable(tableName); admin.deleteTable(tableName); } HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor("f")); if (numberRegions > 1) { admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1)); } else { admin.createTable(desc); } BufferedMutator table = conn.getBufferedMutator(tableName); Put p = new Put("a1".getBytes()); p.addColumn("f".getBytes(), "c1".getBytes(), "".getBytes()); p.addColumn("f".getBytes(), "c2".getBytes(), "".getBytes()); p.addColumn("f".getBytes(), "c3".getBytes(), "5".getBytes()); p.addColumn("f".getBytes(), "c4".getBytes(), "".getBytes()); table.mutate(p); table.close(); }
public static void loadData(final Connection connection, final TableName tableName, int rows, final byte[][] splitKeys, final String... sfamilies) throws IOException { byte[][] families = new byte[sfamilies.length][]; for (int i = 0; i < families.length; ++i) { families[i] = Bytes.toBytes(sfamilies[i]); } BufferedMutator mutator = connection.getBufferedMutator(tableName); // Ensure one row per region assertTrue(rows >= splitKeys.length); for (byte[] k: splitKeys) { byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k); byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value))); mutator.mutate(createPut(families, key, value)); rows--; } // Add other extra rows. more rows, more files while (rows-- > 0) { byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); mutator.mutate(createPut(families, key, value)); } mutator.flush(); }
private LinkedBlockingQueue<Long> insertData() throws IOException, InterruptedException { LinkedBlockingQueue<Long> rowKeys = new LinkedBlockingQueue<>(25000); BufferedMutator ht = util.getConnection().getBufferedMutator(this.tableName); byte[] value = new byte[300]; TraceUtil.addSampler(Sampler.ALWAYS); for (int x = 0; x < 5000; x++) { try (TraceScope traceScope = TraceUtil.createTrace("insertData")) { for (int i = 0; i < 5; i++) { long rk = random.nextLong(); rowKeys.add(rk); Put p = new Put(Bytes.toBytes(rk)); for (int y = 0; y < 10; y++) { random.nextBytes(value); p.addColumn(familyName, Bytes.toBytes(random.nextLong()), value); } ht.mutate(p); } if ((x % 1000) == 0) { admin.flush(tableName); } } } admin.flush(tableName); return rowKeys; }
/** * loads some data to the table. */ private void loadData(Admin admin, BufferedMutator table, TableName tableName, int fileNum, int rowNumPerFile) throws IOException, InterruptedException { if (fileNum <= 0) { throw new IllegalArgumentException(); } for (int i = 0; i < fileNum * rowNumPerFile; i++) { for (byte k0 : KEYS) { byte[] k = new byte[] { k0 }; byte[] key = Bytes.add(k, Bytes.toBytes(i)); byte[] mobVal = makeDummyData(10 * (i + 1)); Put put = new Put(key); put.setDurability(Durability.SKIP_WAL); put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), mobVal); put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal); put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal); table.mutate(put); } if ((i + 1) % rowNumPerFile == 0) { table.flush(); admin.flush(tableName); } } }
private void migrateNamespaceTable() throws IOException { try (Table nsTable = masterServices.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME); ResultScanner scanner = nsTable.getScanner( new Scan().addFamily(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES).readAllVersions()); BufferedMutator mutator = masterServices.getConnection().getBufferedMutator(TableName.META_TABLE_NAME)) { for (Result result;;) { result = scanner.next(); if (result == null) { break; } Put put = new Put(result.getRow()); result .getColumnCells(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES, TableDescriptorBuilder.NAMESPACE_COL_DESC_BYTES) .forEach(c -> put.addColumn(HConstants.NAMESPACE_FAMILY, HConstants.NAMESPACE_COL_DESC_QUALIFIER, c.getTimestamp(), CellUtil.cloneValue(c))); mutator.mutate(put); } } // schedule a disable procedure instead of block waiting here, as when disabling a table we will // wait until master is initialized, but we are part of the initialization... masterServices.getMasterProcedureExecutor().submitProcedure( new DisableTableProcedure(masterServices.getMasterProcedureExecutor().getEnvironment(), TableName.NAMESPACE_TABLE_NAME, false)); }
public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows, byte[]... families) throws IOException, InterruptedException { // Ensure one row per region assertTrue(rows >= KEYS.length); for (byte k0: KEYS) { byte[] k = new byte[] { k0 }; byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k); byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value))); final byte[][] families1 = families; final byte[] key1 = key; final byte[] value1 = value; mutator.mutate(createPut(families1, key1, value1)); rows--; } // Add other extra rows. more rows, more files while (rows-- > 0) { byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); final byte[][] families1 = families; final byte[] key1 = key; final byte[] value1 = value; mutator.mutate(createPut(families1, key1, value1)); } mutator.flush(); waitForTableToBeOnline(util, mutator.getName()); }
@Override protected void map(NullWritable key, NullWritable value, Context context) throws IOException, InterruptedException { String suffix = "/" + shortTaskId; int BLOCK_SIZE = (int) (recordsToWrite / 100); for (long i = 0; i < recordsToWrite;) { for (long idx = 0; idx < BLOCK_SIZE && i < recordsToWrite; idx++, i++) { int expIdx = rand.nextInt(BLOCK_SIZE) % VISIBILITY_EXPS_COUNT; String exp = VISIBILITY_EXPS[expIdx]; byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp)); Put p = new Put(row); p.addColumn(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); p.setCellVisibility(new CellVisibility(exp)); getCounter(expIdx).increment(1); mutator.mutate(p); if (i % 100 == 0) { context.setStatus("Written " + i + "/" + recordsToWrite + " records"); context.progress(); } } // End of block, flush all of them before we start writing anything // pointing to these! mutator.flush(); } }
public static void generateHBaseDatasetIntOB(Connection conn, Admin admin, TableName tableName, int numberRegions) throws Exception { if (admin.tableExists(tableName)) { admin.disableTable(tableName); admin.deleteTable(tableName); } HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(FAMILY_F)); if (numberRegions > 1) { admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1)); } else { admin.createTable(desc); } BufferedMutator table = conn.getBufferedMutator(tableName); for (int i = -49; i <= 100; i++) { byte[] bytes = new byte[5]; PositionedByteRange br = new SimplePositionedMutableByteRange(bytes, 0, 5); OrderedBytes.encodeInt32(br, i, Order.ASCENDING); Put p = new Put(bytes); p.addColumn(FAMILY_F, COLUMN_C, String.format("value %d", i).getBytes()); table.mutate(p); } table.close(); admin.flush(tableName); }
public static void generateHBaseDatasetDoubleOBDesc(Connection conn, Admin admin, TableName tableName, int numberRegions) throws Exception { if (admin.tableExists(tableName)) { admin.disableTable(tableName); admin.deleteTable(tableName); } HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(FAMILY_F)); if (numberRegions > 1) { admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1)); } else { admin.createTable(desc); } BufferedMutator table = conn.getBufferedMutator(tableName); for (double i = 0.5; i <= 100.00; i += 0.75) { byte[] bytes = new byte[9]; PositionedByteRange br = new SimplePositionedMutableByteRange(bytes, 0, 9); OrderedBytes.encodeFloat64(br, i, Order.DESCENDING); Put p = new Put(bytes); p.addColumn(FAMILY_F, COLUMN_C, String.format("value %03f", i).getBytes()); table.mutate(p); } table.close(); admin.flush(tableName); }
@Override void testRow(final int i) throws IOException { byte[] row = getRandomRow(this.rand, this.totalRows); Put put = new Put(row); byte[] value = generateData(this.rand, ROW_LENGTH); if (useTags) { byte[] tag = generateData(this.rand, TAG_LENGTH); Tag[] tags = new Tag[noOfTags]; for (int n = 0; n < noOfTags; n++) { Tag t = new ArrayBackedTag((byte) n, tag); tags[n] = t; } KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP, value, tags); put.add(kv); } else { put.addColumn(FAMILY_NAME, QUALIFIER_NAME, value); } put.setDurability(writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL); mutator.mutate(put); } }
@Override void testRow(final int i) throws IOException { byte[] row = format(i); Put put = new Put(row); byte[] value = generateData(this.rand, ROW_LENGTH); if (useTags) { byte[] tag = generateData(this.rand, TAG_LENGTH); Tag[] tags = new Tag[noOfTags]; for (int n = 0; n < noOfTags; n++) { Tag t = new ArrayBackedTag((byte) n, tag); tags[n] = t; } KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP, value, tags); put.add(kv); } else { put.addColumn(FAMILY_NAME, QUALIFIER_NAME, value); } put.setDurability(writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL); mutator.mutate(put); } }
protected void persist(Context output, long count, byte[][] prev, byte[][] current, byte[] id) throws IOException { for (int i = 0; i < current.length; i++) { if (i % 100 == 0) { // Tickle progress every so often else maprunner will think us hung output.progress(); } Put put = new Put(current[i]); put.addColumn(FAMILY_NAME, COLUMN_PREV, prev == null ? NO_KEY : prev[i]); if (count >= 0) { put.addColumn(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i)); } if (id != null) { put.addColumn(FAMILY_NAME, COLUMN_CLIENT, id); } // See if we are to write multiple columns. if (this.multipleUnevenColumnFamilies) { // Use any column name. put.addColumn(TINY_FAMILY_NAME, TINY_FAMILY_NAME, this.tinyValue); // Use any column name. put.addColumn(BIG_FAMILY_NAME, BIG_FAMILY_NAME, this.bigValue); } mutator.mutate(put); } mutator.flush(); }
@Test public void testMultiMetrics() throws Exception { Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); conf.set(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, "true"); ConnectionImplementation conn = (ConnectionImplementation) ConnectionFactory.createConnection(conf); try { BufferedMutator mutator = conn.getBufferedMutator(TABLE_NAME); byte[][] keys = {Bytes.toBytes("aaa"), Bytes.toBytes("mmm"), Bytes.toBytes("zzz")}; for (byte[] key : keys) { Put p = new Put(key); p.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(10)); mutator.mutate(p); } mutator.flush(); mutator.close(); MetricsConnection metrics = conn.getConnectionMetrics(); assertEquals(1, metrics.multiTracker.reqHist.getCount()); assertEquals(3, metrics.numActionsPerServerHist.getSnapshot().getMean(), 1e-15); assertEquals(1, metrics.numActionsPerServerHist.getCount()); } finally { conn.close(); } } }