Refine search
public static void generateHBaseDatasetNullStr(Connection conn, Admin admin, TableName tableName, int numberRegions) throws Exception { if (admin.tableExists(tableName)) { admin.disableTable(tableName); admin.deleteTable(tableName); } HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor("f")); if (numberRegions > 1) { admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1)); } else { admin.createTable(desc); } BufferedMutator table = conn.getBufferedMutator(tableName); Put p = new Put("a1".getBytes()); p.addColumn("f".getBytes(), "c1".getBytes(), "".getBytes()); p.addColumn("f".getBytes(), "c2".getBytes(), "".getBytes()); p.addColumn("f".getBytes(), "c3".getBytes(), "5".getBytes()); p.addColumn("f".getBytes(), "c4".getBytes(), "".getBytes()); table.mutate(p); table.close(); }
for (Row r : actions) { if (r instanceof Put) { ((Put) r).setDurability(enableWal ? Durability.USE_DEFAULT : Durability.SKIP_WAL); table.mutate(mutations); table.flush(); return null; }); table.mutate(i); table.flush(); return null; });
@Override void testRow(final int i) throws IOException { byte[] row = getRandomRow(this.rand, this.totalRows); Put put = new Put(row); byte[] value = generateData(this.rand, ROW_LENGTH); if (useTags) { byte[] tag = generateData(this.rand, TAG_LENGTH); Tag[] tags = new Tag[noOfTags]; for (int n = 0; n < noOfTags; n++) { Tag t = new ArrayBackedTag((byte) n, tag); tags[n] = t; } KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP, value, tags); put.add(kv); } else { put.addColumn(FAMILY_NAME, QUALIFIER_NAME, value); } put.setDurability(writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL); mutator.mutate(put); } }
Table table = connection.getTable(name); BufferedMutator mutator = connection.getBufferedMutator(name); // co PutWriteBufferExample1-1-CheckFlush Get a mutator instance for the table. Put put1 = new Put(Bytes.toBytes("row1")); put1.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1")); mutator.mutate(put1); // co PutWriteBufferExample1-2-DoPut Store some rows with columns into HBase. Put put2 = new Put(Bytes.toBytes("row2")); put2.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val2")); mutator.mutate(put2); Put put3 = new Put(Bytes.toBytes("row3")); put3.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val3")); mutator.mutate(put3); Get get = new Get(Bytes.toBytes("row1")); System.out.println("Result: " + res1); // co PutWriteBufferExample1-3-Get1 Try to load previously stored row, this will print "Result: keyvalues=NONE". mutator.flush(); // co PutWriteBufferExample1-4-Flush Force a flush, this causes an RPC to occur. mutator.close(); table.close(); connection.close();
/** * loads some data to the table. */ private void loadData(Admin admin, BufferedMutator table, TableName tableName, int fileNum, int rowNumPerFile) throws IOException, InterruptedException { if (fileNum <= 0) { throw new IllegalArgumentException(); } for (int i = 0; i < fileNum * rowNumPerFile; i++) { for (byte k0 : KEYS) { byte[] k = new byte[] { k0 }; byte[] key = Bytes.add(k, Bytes.toBytes(i)); byte[] mobVal = makeDummyData(10 * (i + 1)); Put put = new Put(key); put.setDurability(Durability.SKIP_WAL); put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), mobVal); put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal); put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal); table.mutate(put); } if ((i + 1) % rowNumPerFile == 0) { table.flush(); admin.flush(tableName); } } }
Admin admin = connection.getAdmin(); BufferedMutator mutator = connection.getBufferedMutator(tableName); for (int a = 'A'; a <= 'Z'; a++) for (int b = 'A'; b <= 'Z'; b++) String row = Character.toString((char) a) + Character.toString((char) b) + Character.toString((char) c); // co ClusterOperationExample-02-Put Insert many rows starting from "AAA" to "ZZZ". These will be spread across the regions. Put put = new Put(Bytes.toBytes(row)); put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("col1"), Bytes.toBytes("val1")); System.out.println("Adding row: " + row); mutator.mutate(put); mutator.close(); List<HRegionInfo> list = admin.getTableRegions(tableName); int numRegions = list.size(); HRegionInfo info = list.get(numRegions - 1); admin.splitRegion(info.getRegionName()); // co ClusterOperationExample-04-Split Split the last region this table has, starting at row key "TUV". Adds a new region starting with key "WEI". do { list = admin.getTableRegions(tableName); RegionLocator locator = connection.getRegionLocator(tableName); HRegionLocation location = locator.getRegionLocation(Bytes.toBytes("ZZZ")); // co ClusterOperationExample-06-Cache Retrieve region infos cached and refreshed to show the difference.
byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") }; if (util.getAdmin().tableExists(TABLE)) { util.getAdmin().disableTable(TABLE); util.getAdmin().deleteTable(TABLE); Result r = null; byte[] row = "test".getBytes(); try (Table htable = c.getTable(TABLE)) { Put put = new Put(row); put.addColumn("foo".getBytes(), row, row); BufferedMutator m = c.getBufferedMutator(TABLE); m.mutate(put); m.flush(); try (Table htable = c.getTable(TABLE);) { r = htable.get(get); assertTrue(Arrays.equals(r.getRow(), row));
@Override protected void map(NullWritable key, NullWritable value, Context context) throws IOException, InterruptedException { String suffix = "/" + shortTaskId; int BLOCK_SIZE = (int) (recordsToWrite / 100); for (long i = 0; i < recordsToWrite;) { for (long idx = 0; idx < BLOCK_SIZE && i < recordsToWrite; idx++, i++) { int expIdx = rand.nextInt(BLOCK_SIZE) % VISIBILITY_EXPS_COUNT; String exp = VISIBILITY_EXPS[expIdx]; byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp)); Put p = new Put(row); p.addColumn(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); p.setCellVisibility(new CellVisibility(exp)); getCounter(expIdx).increment(1); mutator.mutate(p); if (i % 100 == 0) { context.setStatus("Written " + i + "/" + recordsToWrite + " records"); context.progress(); } } // End of block, flush all of them before we start writing anything // pointing to these! mutator.flush(); } }
private LinkedBlockingQueue<Long> insertData() throws IOException, InterruptedException { LinkedBlockingQueue<Long> rowKeys = new LinkedBlockingQueue<>(25000); BufferedMutator ht = util.getConnection().getBufferedMutator(this.tableName); byte[] value = new byte[300]; TraceUtil.addSampler(Sampler.ALWAYS); for (int x = 0; x < 5000; x++) { try (TraceScope traceScope = TraceUtil.createTrace("insertData")) { for (int i = 0; i < 5; i++) { long rk = random.nextLong(); rowKeys.add(rk); Put p = new Put(Bytes.toBytes(rk)); for (int y = 0; y < 10; y++) { random.nextBytes(value); p.addColumn(familyName, Bytes.toBytes(random.nextLong()), value); } ht.mutate(p); } if ((x % 1000) == 0) { admin.flush(tableName); } } } admin.flush(tableName); return rowKeys; }
cfBytes[cfIndex++] = Bytes.toBytes(cf); final Table table = createTable(tableName, cfBytes, maxVersions, Bytes.toBytes(String.format(keyFormat, splitStartKey)), Bytes.toBytes(String.format(keyFormat, splitEndKey)), numRegions); BufferedMutator mutator = getConnection().getBufferedMutator(tableName); actualStartKey + rand.nextInt(actualEndKey - actualStartKey))); Put put = new Put(row); Delete del = new Delete(row); for (int iCol = 0; iCol < numColsPerRow; ++iCol) { if (!put.isEmpty()) { mutator.mutate(put); mutator.mutate(del); mutator.flush(); if (hbaseCluster != null) { getMiniHBaseCluster().flushcache(table.getName()); mutator.close();
private void migrateNamespaceTable() throws IOException { try (Table nsTable = masterServices.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME); ResultScanner scanner = nsTable.getScanner( new Scan().addFamily(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES).readAllVersions()); BufferedMutator mutator = masterServices.getConnection().getBufferedMutator(TableName.META_TABLE_NAME)) { for (Result result;;) { result = scanner.next(); if (result == null) { break; } Put put = new Put(result.getRow()); result .getColumnCells(TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES, TableDescriptorBuilder.NAMESPACE_COL_DESC_BYTES) .forEach(c -> put.addColumn(HConstants.NAMESPACE_FAMILY, HConstants.NAMESPACE_COL_DESC_QUALIFIER, c.getTimestamp(), CellUtil.cloneValue(c))); mutator.mutate(put); } } // schedule a disable procedure instead of block waiting here, as when disabling a table we will // wait until master is initialized, but we are part of the initialization... masterServices.getMasterProcedureExecutor().submitProcedure( new DisableTableProcedure(masterServices.getMasterProcedureExecutor().getEnvironment(), TableName.NAMESPACE_TABLE_NAME, false)); }
byte[] row = Bytes.add(new byte[8], Bytes.toBytes(suffix)); Bytes.putLong(row, 0, byteSwapped); Put p = new Put(row); p.addColumn(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); if (blockStart > 0) { for (int j = 0; j < numBackReferencesPerRow; j++) { long referredRow = blockStart - BLOCK_SIZE + rand.nextInt(BLOCK_SIZE); Bytes.putLong(row, 0, swapLong(referredRow)); p.addColumn(TEST_FAMILY, row, HConstants.EMPTY_BYTE_ARRAY); mutator.mutate(p); mutator.flush();
@Test @Ignore(value="We need a better test now that BigtableBufferedMutator has different logic") public void testBufferSizeFlush() throws Exception { int maxSize = 1024; BufferedMutatorParams params = new BufferedMutatorParams(sharedTestEnv.getDefaultTableName()) .writeBufferSize(maxSize); try (BufferedMutator mutator = getConnection().getBufferedMutator(params)) { // HBase 1.0.0 has a bug in it. It returns maxSize instead of the buffer size for // getWriteBufferSize. https://issues.apache.org/jira/browse/HBASE-13113 Assert.assertTrue( 0 == mutator.getWriteBufferSize() || maxSize == mutator.getWriteBufferSize()); Put put = getPut(); mutator.mutate(put); Assert.assertTrue(mutator.getWriteBufferSize() > 0); Put largePut = new Put(dataHelper.randomData("testrow-")); largePut.addColumn(COLUMN_FAMILY, qualifier, Bytes.toBytes(RandomStringUtils.randomAlphanumeric(maxSize * 2))); long heapSize = largePut.heapSize(); Assert.assertTrue("largePut heapsize is : " + heapSize, heapSize > maxSize); mutator.mutate(largePut); // HBase 1.0.0 has a bug in it. It returns maxSize instead of the buffer size for // getWriteBufferSize. https://issues.apache.org/jira/browse/HBASE-13113 Assert.assertTrue( 0 == mutator.getWriteBufferSize() || maxSize == mutator.getWriteBufferSize()); } }
public static void loadData(final Connection connection, final TableName tableName, int rows, final byte[][] splitKeys, final String... sfamilies) throws IOException { byte[][] families = new byte[sfamilies.length][]; for (int i = 0; i < families.length; ++i) { families[i] = Bytes.toBytes(sfamilies[i]); } BufferedMutator mutator = connection.getBufferedMutator(tableName); // Ensure one row per region assertTrue(rows >= splitKeys.length); for (byte[] k: splitKeys) { byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k); byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value))); mutator.mutate(createPut(families, key, value)); rows--; } // Add other extra rows. more rows, more files while (rows-- > 0) { byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); mutator.mutate(createPut(families, key, value)); } mutator.flush(); }
public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows, byte[]... families) throws IOException, InterruptedException { // Ensure one row per region assertTrue(rows >= KEYS.length); for (byte k0: KEYS) { byte[] k = new byte[] { k0 }; byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k); byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value))); final byte[][] families1 = families; final byte[] key1 = key; final byte[] value1 = value; mutator.mutate(createPut(families1, key1, value1)); rows--; } // Add other extra rows. more rows, more files while (rows-- > 0) { byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); final byte[][] families1 = families; final byte[] key1 = key; final byte[] value1 = value; mutator.mutate(createPut(families1, key1, value1)); } mutator.flush(); waitForTableToBeOnline(util, mutator.getName()); }
public void write(ImmutableBytesWritable key, Object value) throws IOException { Put put; if (value instanceof Put){ put = (Put)value; } else if (value instanceof PutWritable) { put = new Put(((PutWritable)value).getPut()); } else { throw new IllegalArgumentException("Illegal Argument " + (value == null ? "null" : value.getClass().getName())); } if(m_walEnabled) { put.setDurability(Durability.SYNC_WAL); } else { put.setDurability(Durability.SKIP_WAL); } m_table.mutate(put); }