|| preparePutCount > this.parallelPreparePutToStoreThreadLimit) { tooBusyStore = (tooBusyStore == null ? store.getColumnFamilyName() : tooBusyStore + "," + store.getColumnFamilyName()); LOG.trace(store.getColumnFamilyName() + ": preparePutCount=" + preparePutCount + "; currentParallelPutCount=" + store.getCurrentParallelPutCount());
@Override public void refreshHFiles(RpcController controller, RefreshHFilesProtos.RefreshHFilesRequest request, RpcCallback<RefreshHFilesProtos.RefreshHFilesResponse> done) { try { for (Store store : env.getRegion().getStores()) { LOG.debug("Refreshing HFiles for region: " + store.getRegionInfo().getRegionNameAsString() + " and store: " + store.getColumnFamilyName() + "class:" + store.getClass()); store.refreshStoreFiles(); } } catch (IOException ioe) { LOG.error("Exception while trying to refresh store files: ", ioe); CoprocessorRpcUtils.setControllerException(controller, ioe); } done.run(RefreshHFilesProtos.RefreshHFilesResponse.getDefaultInstance()); }
@Test public void testRequestOnRegion() throws IOException, InterruptedException { Tracker tracker = new Tracker(); TRACKER = tracker; region.requestCompaction("test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertEquals(1, tracker.notExecutedStores.size()); assertEquals(Bytes.toString(CF2), tracker.notExecutedStores.get(0).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(0).getSecond(), containsString("compaction request was cancelled")); assertEquals(1, tracker.beforeExecuteStores.size()); assertEquals(Bytes.toString(CF1), tracker.beforeExecuteStores.get(0).getColumnFamilyName()); assertEquals(1, tracker.afterExecuteStores.size()); assertEquals(Bytes.toString(CF1), tracker.afterExecuteStores.get(0).getColumnFamilyName()); }
@Test public void testRequestOnStore() throws IOException, InterruptedException { Tracker tracker = new Tracker(); TRACKER = tracker; region.requestCompaction(CF1, "test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertTrue(tracker.notExecutedStores.isEmpty()); assertEquals(1, tracker.beforeExecuteStores.size()); assertEquals(Bytes.toString(CF1), tracker.beforeExecuteStores.get(0).getColumnFamilyName()); assertEquals(1, tracker.afterExecuteStores.size()); assertEquals(Bytes.toString(CF1), tracker.afterExecuteStores.get(0).getColumnFamilyName()); tracker = new Tracker(); TRACKER = tracker; region.requestCompaction(CF2, "test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertEquals(1, tracker.notExecutedStores.size()); assertEquals(Bytes.toString(CF2), tracker.notExecutedStores.get(0).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(0).getSecond(), containsString("compaction request was cancelled")); assertTrue(tracker.beforeExecuteStores.isEmpty()); assertTrue(tracker.afterExecuteStores.isEmpty()); }
@Test public void testSpaceQuotaViolation() throws IOException, InterruptedException { region.getRegionServerServices().getRegionServerSpaceQuotaManager().enforceViolationPolicy(NAME, new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 10L, 100L)); Tracker tracker = new Tracker(); TRACKER = tracker; region.requestCompaction("test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertEquals(2, tracker.notExecutedStores.size()); tracker.notExecutedStores.sort((p1, p2) -> p1.getFirst().getColumnFamilyName() .compareTo(p2.getFirst().getColumnFamilyName())); assertEquals(Bytes.toString(CF1), tracker.notExecutedStores.get(0).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(0).getSecond(), containsString("space quota violation")); assertEquals(Bytes.toString(CF2), tracker.notExecutedStores.get(1).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(1).getSecond(), containsString("space quota violation")); assertTrue(tracker.beforeExecuteStores.isEmpty()); assertTrue(tracker.afterExecuteStores.isEmpty()); } }
when(mockStore1.getColumnFamilyName()).thenReturn("test_Family_1");
List<? extends Store> stores = region.getStores(); for (Store store : stores) { String cf = store.getColumnFamilyName(); Collection<? extends StoreFile> storeFiles = store.getStorefiles(); out.write("\n\n <h3>Column Family: ");
@Override public InternalScanner createCompactionScanner(RegionCoprocessorEnvironment env, Store store, InternalScanner delegate) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getColumnFamilyDescriptor().getName()); LOG.info("StatisticsScanner created for table: " + tableName + " CF: " + store.getColumnFamilyName()); return new StatisticsScanner(this, statsWriter, env, delegate, cfKey); }
@Override public String toString() { return getColumnFamilyName(); }
public static boolean OmidCompactionEnabled(ObserverContext<RegionCoprocessorEnvironment> env, Store store, String cfFlagValue) { HTableDescriptor desc = env.getEnvironment().getRegion().getTableDesc(); HColumnDescriptor famDesc = desc.getFamily(Bytes.toBytes(store.getColumnFamilyName())); return Boolean.valueOf(famDesc.getValue(cfFlagValue)); }
@Override public void refreshHFiles(RpcController controller, RefreshHFilesProtos.RefreshHFilesRequest request, RpcCallback<RefreshHFilesProtos.RefreshHFilesResponse> done) { try { for (Store store : env.getRegion().getStores()) { LOG.debug("Refreshing HFiles for region: " + store.getRegionInfo().getRegionNameAsString() + " and store: " + store.getColumnFamilyName() + "class:" + store.getClass()); store.refreshStoreFiles(); } } catch (IOException ioe) { LOG.error("Exception while trying to refresh store files: ", ioe); CoprocessorRpcUtils.setControllerException(controller, ioe); } done.run(RefreshHFilesProtos.RefreshHFilesResponse.getDefaultInstance()); }
@Override protected boolean shouldSplit() { if (region.shouldForceSplit()) return true; boolean foundABigStore = false; // Get count of regions that have the same common table as this.region int tableRegionsCount = getCountOfCommonTableRegions(); // Get size to check long sizeToCheck = getSizeToCheck(tableRegionsCount); for (Store store : region.getStores().values()) { // If any of the stores is unable to split (eg they contain reference files) // then don't split if ((!store.canSplit())) { return false; } // Mark if any store is big enough long size = store.getSize(); if (size > sizeToCheck) { LOG.debug("ShouldSplit because " + store.getColumnFamilyName() + " size=" + size + ", sizeToCheck=" + sizeToCheck + ", regionsWithCommonTable=" + tableRegionsCount); foundABigStore = true; break; } } return foundABigStore; }
@Test public void testRequestOnRegion() throws IOException, InterruptedException { Tracker tracker = new Tracker(); TRACKER = tracker; region.requestCompaction("test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertEquals(1, tracker.notExecutedStores.size()); assertEquals(Bytes.toString(CF2), tracker.notExecutedStores.get(0).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(0).getSecond(), containsString("compaction request was cancelled")); assertEquals(1, tracker.beforeExecuteStores.size()); assertEquals(Bytes.toString(CF1), tracker.beforeExecuteStores.get(0).getColumnFamilyName()); assertEquals(1, tracker.afterExecuteStores.size()); assertEquals(Bytes.toString(CF1), tracker.afterExecuteStores.get(0).getColumnFamilyName()); }
private boolean shouldFlush(Store store) { if (store.getMemStoreSize() > this.flushSizeLowerBound) { if (LOG.isDebugEnabled()) { LOG.debug("Flush Column Family " + store.getColumnFamilyName() + " of " + region.getRegionInfo().getEncodedName() + " because memstoreSize=" + store.getMemStoreSize() + " > lower bound=" + this.flushSizeLowerBound); } return true; } return region.shouldFlushStore(store); }
byte[] cf = Bytes.toBytes(store.getColumnFamilyName()); if (stores.get(cf) != store) { LOG.warn("Store " + store.getColumnFamilyName() + " on region " + this + " has been re-instantiated, cancel this compaction request. " + " It may be caused by the roll back of split transaction");
@Test public void testRequestOnStore() throws IOException, InterruptedException { Tracker tracker = new Tracker(); TRACKER = tracker; region.requestCompaction(CF1, "test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertTrue(tracker.notExecutedStores.isEmpty()); assertEquals(1, tracker.beforeExecuteStores.size()); assertEquals(Bytes.toString(CF1), tracker.beforeExecuteStores.get(0).getColumnFamilyName()); assertEquals(1, tracker.afterExecuteStores.size()); assertEquals(Bytes.toString(CF1), tracker.afterExecuteStores.get(0).getColumnFamilyName()); tracker = new Tracker(); TRACKER = tracker; region.requestCompaction(CF2, "test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertEquals(1, tracker.notExecutedStores.size()); assertEquals(Bytes.toString(CF2), tracker.notExecutedStores.get(0).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(0).getSecond(), containsString("compaction request was cancelled")); assertTrue(tracker.beforeExecuteStores.isEmpty()); assertTrue(tracker.afterExecuteStores.isEmpty()); }
@Test public void testSpaceQuotaViolation() throws IOException, InterruptedException { region.getRegionServerServices().getRegionServerSpaceQuotaManager().enforceViolationPolicy(NAME, new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 10L, 100L)); Tracker tracker = new Tracker(); TRACKER = tracker; region.requestCompaction("test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertEquals(2, tracker.notExecutedStores.size()); tracker.notExecutedStores.sort((p1, p2) -> p1.getFirst().getColumnFamilyName() .compareTo(p2.getFirst().getColumnFamilyName())); assertEquals(Bytes.toString(CF1), tracker.notExecutedStores.get(0).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(0).getSecond(), containsString("space quota violation")); assertEquals(Bytes.toString(CF2), tracker.notExecutedStores.get(1).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(1).getSecond(), containsString("space quota violation")); assertTrue(tracker.beforeExecuteStores.isEmpty()); assertTrue(tracker.afterExecuteStores.isEmpty()); } }
when(mockStore1.getColumnFamilyName()).thenReturn("test_Family_1");
@Override protected boolean shouldSplit() { boolean force = region.shouldForceSplit(); boolean foundABigStore = false; // Get count of regions that have the same common table as this.region int tableRegionsCount = getCountOfCommonTableRegions(); // Get size to check long sizeToCheck = getSizeToCheck(tableRegionsCount); for (Store store : region.getStores()) { // If any of the stores is unable to split (eg they contain reference files) // then don't split if (!store.canSplit()) { return false; } // Mark if any store is big enough long size = store.getSize(); if (size > sizeToCheck) { LOG.debug("ShouldSplit because " + store.getColumnFamilyName() + " size=" + size + ", sizeToCheck=" + sizeToCheck + ", regionsWithCommonTable=" + tableRegionsCount); foundABigStore = true; } } return foundABigStore | force; }
if (earliest > 0 && earliest + flushPerChanges < mvcc.getReadPoint()) { if (LOG.isDebugEnabled()) { LOG.debug("Flush column family " + store.getColumnFamilyName() + " of " + getRegionInfo().getEncodedName() + " because unflushed sequenceid=" + earliest + " is > " + this.flushPerChanges + " from current=" + mvcc.getReadPoint()); if (store.timeOfOldestEdit() < now - this.flushCheckInterval) { if (LOG.isDebugEnabled()) { LOG.debug("Flush column family: " + store.getColumnFamilyName() + " of " + getRegionInfo().getEncodedName() + " because time of oldest edit=" + store.timeOfOldestEdit() + " is > " + this.flushCheckInterval + " from now =" + now);