public FuzzyRowFilter(List<Pair<byte[], byte[]>> fuzzyKeysData) { List<Pair<byte[], byte[]>> fuzzyKeyDataCopy = new ArrayList<>(fuzzyKeysData.size()); for (Pair<byte[], byte[]> aFuzzyKeysData : fuzzyKeysData) { if (aFuzzyKeysData.getFirst().length != aFuzzyKeysData.getSecond().length) { Pair<String, String> readable = new Pair<>(Bytes.toStringBinary(aFuzzyKeysData.getFirst()), Bytes.toStringBinary(aFuzzyKeysData.getSecond())); throw new IllegalArgumentException("Fuzzy pair lengths do not match: " + readable); } Pair<byte[], byte[]> p = new Pair<>(); // create a copy of pair bytes so that they are not modified by the filter. p.setFirst(Arrays.copyOf(aFuzzyKeysData.getFirst(), aFuzzyKeysData.getFirst().length)); p.setSecond(Arrays.copyOf(aFuzzyKeysData.getSecond(), aFuzzyKeysData.getSecond().length)); // update mask ( 0 -> -1 (0xff), 1 -> 2) p.setSecond(preprocessMask(p.getSecond())); preprocessSearchKey(p); fuzzyKeyDataCopy.add(p); } this.fuzzyKeysData = fuzzyKeyDataCopy; this.tracker = new RowTracker(); }
/** * Set the EventLoopGroup and channel class for {@code AsyncRpcClient}. */ public static void setEventLoopConfig(Configuration conf, EventLoopGroup group, Class<? extends Channel> channelClass) { Preconditions.checkNotNull(group, "group is null"); Preconditions.checkNotNull(channelClass, "channel class is null"); conf.set(EVENT_LOOP_CONFIG, CONFIG_NAME); EVENT_LOOP_CONFIG_MAP.put(CONFIG_NAME, Pair.<EventLoopGroup, Class<? extends Channel>> newPair(group, channelClass)); }
public ExportSnapshotInputSplit(final List<Pair<SnapshotFileInfo, Long>> snapshotFiles) { this.files = new ArrayList(snapshotFiles.size()); for (Pair<SnapshotFileInfo, Long> fileInfo: snapshotFiles) { this.files.add(new Pair<>( new BytesWritable(fileInfo.getFirst().toByteArray()), fileInfo.getSecond())); this.length += fileInfo.getSecond(); } }
private static List<org.apache.hadoop.hbase.util.Pair<byte[], byte[]>> convertToHBasePair(List<org.apache.kylin.common.util.Pair<byte[], byte[]>> pairList) { List<org.apache.hadoop.hbase.util.Pair<byte[], byte[]>> result = Lists.newArrayList(); for (org.apache.kylin.common.util.Pair<byte[], byte[]> pair : pairList) { org.apache.hadoop.hbase.util.Pair<byte[], byte[]> element = new org.apache.hadoop.hbase.util.Pair<byte[], byte[]>(pair.getFirst(), pair.getSecond()); result.add(element); } return result; }
private List<PeekingResultIterator> recreateIterators(ConnectionQueryServices services, boolean isLocalIndex, Queue<PeekingResultIterator> allIterators, List<PeekingResultIterator> iterators, boolean isReverse, long maxQueryEndTime, ScanWrapper previousScan, boolean clearedCache, List<PeekingResultIterator> concatIterators, Iterator<Pair<Scan, Future<PeekingResultIterator>>> scanPairItr, Pair<Scan, Future<PeekingResultIterator>> scanPair, int retryCount) throws SQLException { scanPairItr.remove(); // Resubmit just this portion of work again Scan oldScan = scanPair.getFirst(); byte[] startKey = oldScan.getAttribute(SCAN_ACTUAL_START_ROW); byte[] endKey = oldScan.getStopRow(); List<List<Scan>> newNestedScans = this.getParallelScans(startKey, endKey); // Add any concatIterators that were successful so far // as we need these to be in order addIterator(iterators, concatIterators); concatIterators = Lists.newArrayList(); getIterators(newNestedScans, services, isLocalIndex, allIterators, iterators, isReverse, maxQueryEndTime, newNestedScans.size(), previousScan, retryCount); return concatIterators; }
@Override public Iterable<IndexUpdate> getIndexUpserts(TableState state, IndexMetaData context, byte[] regionStartKey, byte[] regionEndKey) throws IOException { PhoenixIndexMetaData metaData = (PhoenixIndexMetaData)context; List<IndexMaintainer> indexMaintainers = metaData.getIndexMaintainers(); if (indexMaintainers.get(0).isRowDeleted(state.getPendingUpdate())) { return Collections.emptyList(); } ImmutableBytesWritable ptr = new ImmutableBytesWritable(); ptr.set(state.getCurrentRowKey()); List<IndexUpdate> indexUpdates = Lists.newArrayList(); for (IndexMaintainer maintainer : indexMaintainers) { Pair<ValueGetter, IndexUpdate> statePair = state.getIndexUpdateState(maintainer.getAllColumns(), metaData.getReplayWrite() != null, false, context); ValueGetter valueGetter = statePair.getFirst(); IndexUpdate indexUpdate = statePair.getSecond(); indexUpdate.setTable(maintainer.isLocalIndex() ? tableName : maintainer.getIndexTableName()); Put put = maintainer.buildUpdateMutation(KV_BUILDER, valueGetter, ptr, state.getCurrentTimestamp(), regionStartKey, regionEndKey); indexUpdate.setUpdate(put); indexUpdates.add(indexUpdate); } return indexUpdates; }
TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf), Integer.MAX_VALUE); Put p = new Put(Bytes.toBytesBinary(rows[i])); p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes()); ht.put(p); Put p = new Put(Bytes.toBytesBinary(badRow)); p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes()); ht.put(p); TEST_UTIL.flush(); byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044"); byte[] mask = new byte[] { 1,0,0,0}; data.add(new Pair<>(fuzzyKey, mask)); FuzzyRowFilter filter = new FuzzyRowFilter(data); Scan scan = new Scan(); scan.setFilter(filter); TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
Pair<byte[][], byte[][]> keys = UTIL.getConnection() .getRegionLocator(tableName).getStartEndKeys(); assertEquals(2, keys.getFirst().length); byte[][] startKeys = keys.getFirst(); byte[][] endKeys = keys.getSecond(); for (int i = 0; i < startKeys.length; i++) { for (int j = 0; j < timestamp.length; j++) { Put put = new Put(rows[i], timestamp[j]); put.addColumn(HConstants.CATALOG_FAMILY, null, timestamp[j], Bytes.toBytes(timestamp[j])); puts.add(put); Scan scan = new Scan(); scan.addFamily(HConstants.CATALOG_FAMILY); ResultScanner s = table.getScanner(scan); try { scan = new Scan(); scan.setTimeRange(1000L, Long.MAX_VALUE); scan.addFamily(HConstants.CATALOG_FAMILY);
byte [] family = htd.getFamilies().iterator().next().getName(); Path f = new Path(basedir, "hfile"); HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""), Bytes.toBytes("z"), 10); List<Pair<byte[], String>> hfs = new ArrayList<>(1); hfs.add(Pair.newPair(family, f.toString())); region.bulkLoadHFiles(hfs, true, null); byte[] row = tableName.getName(); region.put((new Put(row)).addColumn(family, family, family)); wal.sync(); final int rowsInsertedCount = 11; assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan()))); User user = HBaseTestingUtility.getDifferentUser(newConf, tableName.getNameAsString()); user.runAs(new PrivilegedExceptionAction() {
@Override boolean testRow(final int i) throws IOException { Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow(); Scan scan = new Scan().withStartRow(startAndStopRow.getFirst()) .withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) .setReadType(opts.scanReadType).setScanMetricsEnabled(true); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(familyName, qualifier); Bytes.toString(startAndStopRow.getFirst()), Bytes.toString(startAndStopRow.getSecond()), count));
try (Connection conn = ConnectionFactory.createConnection(UTIL.getConfiguration()); Table t = conn.getTable(TableName.valueOf(exportTable))) { t.put(p1); t.put(p2); SecureTestUtil.verifyAllowed(putAction, getUserByLogin(USER_OWNER)); List<Pair<List<String>, Integer>> labelsAndRowCounts = new LinkedList<>(); labelsAndRowCounts.add(new Pair<>(Arrays.asList(SECRET), 1)); labelsAndRowCounts.add(new Pair<>(Arrays.asList(PRIVATE, CONFIDENTIAL), 1)); labelsAndRowCounts.add(new Pair<>(Arrays.asList(TOPSECRET), 1)); labelsAndRowCounts.add(new Pair<>(Arrays.asList(TOPSECRET, CONFIDENTIAL), 0)); labelsAndRowCounts.add(new Pair<>(Arrays.asList(TOPSECRET, CONFIDENTIAL, PRIVATE, SECRET), 2)); for (final Pair<List<String>, Integer> labelsAndRowCount : labelsAndRowCounts) { final List<String> labels = labelsAndRowCount.getFirst(); final int rowCount = labelsAndRowCount.getSecond(); final FileSystem fs = openDir.getFileSystem(UTIL.getConfiguration()); fs.mkdirs(openDir); fs.setPermission(openDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); SecureTestUtil.verifyAllowed(importAction, getUserByLogin(USER_OWNER)); AccessTestAction scanAction = () -> { Scan scan = new Scan(); scan.setAuthorizations(new Authorizations(labels));
TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3); final TableName tableName = TableName.valueOf(name.getMethodName()); Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10); Admin admin = TEST_UTIL.getAdmin(); byte[] row = Bytes.toBytes(random.nextInt()); performMultiplePutAndFlush((HBaseAdmin) admin, hTable, row, FAMILY, 3, 100); HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor()); htd.setValue("hbase.hstore.compaction.min", String.valueOf(5)); admin.modifyTable(tableName, htd); Pair<Integer, Integer> st; while (null != (st = admin.getAlterStatus(tableName)) && st.getFirst() > 0) { LOG.debug(st.getFirst() + " regions left to update"); Thread.sleep(40); HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); while (null != (st = admin.getAlterStatus(tableName)) && st.getFirst() > 0) { LOG.debug(st.getFirst() + " regions left to update"); Thread.sleep(40); LOG.debug("Waiting for region to come online: " + Bytes.toString(regionName)); while (null != (st = admin.getAlterStatus(tableName)) && st.getFirst() > 0) { LOG.debug(st.getFirst() + " regions left to update"); Thread.sleep(40);
TEST_UTIL.getAdmin().splitRegion(regions.get(1).getFirst().getRegionName()); } catch (IllegalArgumentException ex) { gotException = true; TEST_UTIL.getHBaseAdmin().splitRegionAsync(regions.get(1).getFirst(), new byte[]{(byte)'1'}); } catch (IOException ex) { TEST_UTIL.getHBaseAdmin().splitRegionSync(regions.get(1).getFirst().getRegionName(), new byte[]{(byte)'1'}); } catch (IllegalArgumentException ex) { regions.get(1).getFirst().getEncodedNameAsBytes(), regions.get(2).getFirst().getEncodedNameAsBytes(), true); } catch (IllegalArgumentException m) { nameofRegionsToMerge[0] = regions.get(1).getFirst().getEncodedNameAsBytes(); nameofRegionsToMerge[1] = regions.get(2).getFirst().getEncodedNameAsBytes(); MergeTableRegionsRequest request = RequestConverter .buildMergeTableRegionsRequest(
TableName tableName = TableName.valueOf("testtable"); HColumnDescriptor coldef1 = new HColumnDescriptor("colfam1"); HTableDescriptor desc = new HTableDescriptor(tableName) .addFamily(coldef1) .setValue("Description", "Chapter 5 - ModifyTableExample: Original Table"); admin.createTable(desc, Bytes.toBytes(1L), Bytes.toBytes(10000L), 50); // co ModifyTableExample-1-CreateTable Create the table with the original structure and 50 regions. setSecond(50); }}; for (int i = 0; status.getFirst() != 0 && i < 500; i++) { status = admin.getAlterStatus(desc.getTableName()); // co ModifyTableExample-5-Loop Loop over status until all regions are updated, or 500 seconds have been exceeded. if (status.getSecond() != 0) { int pending = status.getSecond() - status.getFirst(); System.out.println(pending + " of " + status.getSecond() + " regions updated."); Thread.sleep(1 * 1000l); if (status.getFirst() != 0) { throw new IOException("Failed to update regions after 500 seconds.");
static List<Put> createPutForPreparedBulkload(TableName table, byte[] region, final byte[] family, final List<Pair<Path, Path>> pairs) { List<Put> puts = new ArrayList<>(pairs.size()); for (Pair<Path, Path> pair : pairs) { Path path = pair.getSecond(); String file = path.toString(); int lastSlash = file.lastIndexOf("/"); String filename = file.substring(lastSlash + 1); Put put = new Put(rowkey(BULK_LOAD_PREFIX, table.toString(), BLK_LD_DELIM, Bytes.toString(region), BLK_LD_DELIM, filename)); put.addColumn(BackupSystemTable.META_FAMILY, TBL_COL, table.getName()); put.addColumn(BackupSystemTable.META_FAMILY, FAM_COL, family); put.addColumn(BackupSystemTable.META_FAMILY, PATH_COL, file.getBytes()); put.addColumn(BackupSystemTable.META_FAMILY, STATE_COL, BL_PREPARE); puts.add(put); LOG.debug("writing raw bulk path " + file + " for " + table + " " + Bytes.toString(region)); } return puts; }
ConstantSizeRegionSplitPolicy.class.getName()); HTableDescriptor htd = new HTableDescriptor(TABLENAME); htd.setCompactionEnabled(false); htd.addFamily(new HColumnDescriptor(FAMILY1)); conn.close(); Region region = getRegionWithName(TABLENAME).getFirst(); cf1StoreFileCount = region.getStore(FAMILY1).getStorefilesCount(); cf2StoreFileCount = region.getStore(FAMILY2).getStorefilesCount(); NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build()); conn.close(); Region region = getRegionWithName(TABLENAME).getFirst(); cf1StoreFileCount1 = region.getStore(FAMILY1).getStorefilesCount(); cf2StoreFileCount1 = region.getStore(FAMILY2).getStorefilesCount(); LOG.info("disable selective flush: " + Bytes.toString(FAMILY1) + "=>" + cf1StoreFileCount + ", " + Bytes.toString(FAMILY2) + "=>" + cf2StoreFileCount + ", " + Bytes.toString(FAMILY3) + "=>" + cf3StoreFileCount);
Pair<byte[][], byte[][]> keys = regionLocator.getStartEndKeys(); for (Scan scan : scanList) { if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { throw new IOException("Expecting at least one region for table : " + tableName.getNameAsString()); for (int i = 0; i < keys.getFirst().length; i++) { if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) { continue; if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) && (stopRow.length == 0 || Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) { byte[] splitStart = startRow.length == 0 || Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys.getFirst()[i] : startRow; byte[] splitStop = (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys.getSecond()[i] : stopRow; keys.getFirst()[i], false); String regionHostname = hregionLocation.getHostname(); HRegionInfo regionInfo = hregionLocation.getRegionInfo(); String encodedRegionName = regionInfo.getEncodedName(); long regionSize = sizeCalculator.getRegionSize( regionInfo.getRegionName());
.getNamespaceAsString(), snapshotTable.getQualifierAsString()), regionInfo .getEncodedName()), familyDir.getName()), hfileName)); String snapshotRegionName = refPath.getParent().getParent().getName(); String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName))); if (clonedRegionName == null) clonedRegionName = snapshotRegionName; String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes())); if (regionName == null) { regionName = regionInfo.getEncodedName(); daughters = new Pair<>(regionName, regionName); parentsMap.put(clonedRegionName, daughters); } else if (!regionName.equals(daughters.getFirst())) { daughters.setSecond(regionName);
private void doPut(Table table, long memstoreFlushSize) throws IOException, InterruptedException { Region region = getRegionWithName(table.getName()).getFirst(); // cf1 4B per row, cf2 40B per row and cf3 400B per row byte[] qf = Bytes.toBytes("qf"); Random rand = new Random(); byte[] value1 = new byte[100]; byte[] value2 = new byte[200]; byte[] value3 = new byte[400]; for (int i = 0; i < 10000; i++) { Put put = new Put(Bytes.toBytes("row-" + i)); rand.setSeed(i); rand.nextBytes(value1); rand.nextBytes(value2); rand.nextBytes(value3); put.addColumn(FAMILY1, qf, value1); put.addColumn(FAMILY2, qf, value2); put.addColumn(FAMILY3, qf, value3); table.put(put); // slow down to let regionserver flush region. while (region.getMemStoreHeapSize() > memstoreFlushSize) { Thread.sleep(100); } } }