@Override public void flushCache(MonitoredTask status) throws IOException { RegionServerServices rsService = region.getRegionServerServices(); ThroughputController throughputController = rsService == null ? null : rsService.getFlushThroughputController(); tempFiles = HStore.this.flushCache(cacheFlushSeqNum, snapshot, status, throughputController, tracker); }
/** * Try to acquire a lock. Throw RegionTooBusyException * if failed to get the lock in time. Throw InterruptedIOException * if interrupted while waiting for the lock. */ private void lock(final Lock lock, final int multiplier) throws RegionTooBusyException, InterruptedIOException { try { final long waitTime = Math.min(maxBusyWaitDuration, busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier)); if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) { // Don't print millis. Message is used as a key over in // RetriesExhaustedWithDetailsException processing. throw new RegionTooBusyException("Failed to obtain lock; regionName=" + (this.getRegionInfo() == null? "unknown": this.getRegionInfo().getRegionNameAsString()) + ", server=" + (this.getRegionServerServices() == null? "unknown": this.getRegionServerServices().getServerName())); } } catch (InterruptedException ie) { LOG.info("Interrupted while waiting for a lock"); InterruptedIOException iie = new InterruptedIOException(); iie.initCause(ie); throw iie; } }
/** * @return Count of regions on this server that share the table this.region * belongs to */ private int getCountOfCommonTableRegions() { RegionServerServices rss = region.getRegionServerServices(); // Can be null in tests if (rss == null) { return 0; } TableName tablename = region.getTableDescriptor().getTableName(); int tableRegionsCount = 0; try { List<? extends Region> hri = rss.getRegions(tablename); tableRegionsCount = hri == null || hri.isEmpty() ? 0 : hri.size(); } catch (IOException e) { LOG.debug("Failed getOnlineRegions " + tablename, e); } return tableRegionsCount; }
void reportArchivedFilesForQuota(List<? extends StoreFile> archivedFiles, List<Long> fileSizes) { // Sanity check from the caller if (archivedFiles.size() != fileSizes.size()) { throw new RuntimeException("Coding error: should never see lists of varying size"); } RegionServerServices rss = this.region.getRegionServerServices(); if (rss == null) { return; } List<Entry<String,Long>> filesWithSizes = new ArrayList<>(archivedFiles.size()); Iterator<Long> fileSizeIter = fileSizes.iterator(); for (StoreFile storeFile : archivedFiles) { final long fileSize = fileSizeIter.next(); if (storeFile.isHFile() && fileSize != 0) { filesWithSizes.add(Maps.immutableEntry(storeFile.getPath().getName(), fileSize)); } } if (LOG.isTraceEnabled()) { LOG.trace("Files archived: " + archivedFiles + ", reporting the following to the Master: " + filesWithSizes); } boolean success = rss.reportFileArchivalForQuotas(getTableName(), filesWithSizes); if (!success) { LOG.warn("Failed to report archival of files: " + filesWithSizes); } }
@VisibleForTesting void replaceStoreFiles(Collection<HStoreFile> compactedFiles, Collection<HStoreFile> result) throws IOException { this.lock.writeLock().lock(); try { this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result); synchronized (filesCompacting) { filesCompacting.removeAll(compactedFiles); } // These may be null when the RS is shutting down. The space quota Chores will fix the Region // sizes later so it's not super-critical if we miss these. RegionServerServices rsServices = region.getRegionServerServices(); if (rsServices != null && rsServices.getRegionServerSpaceQuotaManager() != null) { updateSpaceQuotaAfterFileReplacement( rsServices.getRegionServerSpaceQuotaManager().getRegionSizeStore(), getRegionInfo(), compactedFiles, result); } } finally { this.lock.writeLock().unlock(); } }
void checkResources() throws RegionTooBusyException { // If catalog region, do not impose resource constraints or block updates. if (this.getRegionInfo().isMetaRegion()) return; MemStoreSize mss = this.memStoreSizing.getMemStoreSize(); if (mss.getHeapSize() + mss.getOffHeapSize() > this.blockingMemStoreSize) { blockedRequestsCount.increment(); requestFlush(); // Don't print current limit because it will vary too much. The message is used as a key // over in RetriesExhaustedWithDetailsException processing. throw new RegionTooBusyException("Over memstore limit=" + org.apache.hadoop.hbase.procedure2.util.StringUtils.humanSize(this.blockingMemStoreSize) + ", regionName=" + (this.getRegionInfo() == null? "unknown": this.getRegionInfo().getEncodedName()) + ", server=" + (this.getRegionServerServices() == null? "unknown": this.getRegionServerServices().getServerName())); } }
@VisibleForTesting protected List<HStoreFile> doCompaction(CompactionRequestImpl cr, Collection<HStoreFile> filesToCompact, User user, long compactionStartTime, List<Path> newFiles) throws IOException { // Do the steps necessary to complete the compaction. List<HStoreFile> sfs = moveCompactedFilesIntoPlace(cr, newFiles, user); writeCompactionWalRecord(filesToCompact, sfs); replaceStoreFiles(filesToCompact, sfs); if (cr.isMajor()) { majorCompactedCellsCount.addAndGet(getCompactionProgress().getTotalCompactingKVs()); majorCompactedCellsSize.addAndGet(getCompactionProgress().totalCompactedSize); } else { compactedCellsCount.addAndGet(getCompactionProgress().getTotalCompactingKVs()); compactedCellsSize.addAndGet(getCompactionProgress().totalCompactedSize); } long outputBytes = getTotalSize(sfs); // At this point the store will use new files for all new scanners. completeCompaction(filesToCompact); // update store size. long now = EnvironmentEdgeManager.currentTime(); if (region.getRegionServerServices() != null && region.getRegionServerServices().getMetrics() != null) { region.getRegionServerServices().getMetrics().updateCompaction( region.getTableDescriptor().getTableName().getNameAsString(), cr.isMajor(), now - compactionStartTime, cr.getFiles().size(), newFiles.size(), cr.getSize(), outputBytes); } logCompactionEndMessage(cr, sfs, now, compactionStartTime); return sfs; }
@VisibleForTesting protected void writeRegionOpenMarker(WAL wal, long openSeqId) throws IOException { Map<byte[], List<Path>> storeFiles = getStoreFiles(); RegionEventDescriptor regionOpenDesc = ProtobufUtil.toRegionEventDescriptor( RegionEventDescriptor.EventType.REGION_OPEN, getRegionInfo(), openSeqId, getRegionServerServices().getServerName(), storeFiles); WALUtil.writeRegionEventMarker(wal, getReplicationScope(), getRegionInfo(), regionOpenDesc, mvcc); }
MemStoreSize mss = store.getFlushableSize(); if (!(abort || mss.getDataSize() == 0 || writestate.readOnly)) { if (getRegionServerServices() != null) { getRegionServerServices().abort("Assertion failed while closing store " + getRegionInfo().getRegionNameAsString() + " " + store + ". flushableSize expected=0, actual={" + mss if (!abort && wal != null && getRegionServerServices() != null && RegionReplicaUtil.isDefaultReplica(getRegionInfo())) { writeRegionCloseMarker(wal);
/** * Test that a regionserver is able to abort properly, even when a coprocessor * throws an exception in preStopRegionServer(). */ @Test public void testAbortFromRPC() throws Exception { TableName tableName = TableName.valueOf("testAbortFromRPC"); // create a test table Table table = testUtil.createTable(tableName, FAMILY_BYTES); // write some edits testUtil.loadTable(table, FAMILY_BYTES); LOG.info("Wrote data"); // force a flush cluster.flushcache(tableName); LOG.info("Flushed table"); // Send a poisoned put to trigger the abort Put put = new Put(new byte[]{0, 0, 0, 0}); put.addColumn(FAMILY_BYTES, Bytes.toBytes("c"), new byte[]{}); put.setAttribute(StopBlockingRegionObserver.DO_ABORT, new byte[]{1}); List<HRegion> regions = cluster.findRegionsForTable(tableName); HRegion firstRegion = cluster.findRegionsForTable(tableName).get(0); table.put(put); // Verify that the regionserver is stopped assertNotNull(firstRegion); assertNotNull(firstRegion.getRegionServerServices()); LOG.info("isAborted = " + firstRegion.getRegionServerServices().isAborted()); assertTrue(firstRegion.getRegionServerServices().isAborted()); LOG.info("isStopped = " + firstRegion.getRegionServerServices().isStopped()); assertTrue(firstRegion.getRegionServerServices().isStopped()); }
/** * Open HRegion. * Calls initialize and sets sequenceId. * @return Returns <code>this</code> */ protected HRegion openHRegion(final CancelableProgressable reporter) throws IOException { // Refuse to open the region if we are missing local compression support checkCompressionCodecs(); LOG.debug("checking encryption for " + this.getRegionInfo().getEncodedName()); // Refuse to open the region if encryption configuration is incorrect or // codec support is missing checkEncryption(); // Refuse to open the region if a required class cannot be loaded LOG.debug("checking classloading for " + this.getRegionInfo().getEncodedName()); checkClassLoading(); this.openSeqNum = initialize(reporter); this.mvcc.advanceTo(openSeqNum); // The openSeqNum must be increased every time when a region is assigned, as we rely on it to // determine whether a region has been successfully reopened. So here we always write open // marker, even if the table is read only. if (wal != null && getRegionServerServices() != null && RegionReplicaUtil.isDefaultReplica(getRegionInfo())) { writeRegionOpenMarker(wal, openSeqNum); } return this; }
final List<HRegion> regions = new ArrayList<>(); Mockito.doReturn(regions).when(rss).getRegions(TABLENAME); Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss); Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(0L); Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(0L);
private void writeRegionCloseMarker(WAL wal) throws IOException { Map<byte[], List<Path>> storeFiles = getStoreFiles(); RegionEventDescriptor regionEventDesc = ProtobufUtil.toRegionEventDescriptor( RegionEventDescriptor.EventType.REGION_CLOSE, getRegionInfo(), mvcc.getReadPoint(), getRegionServerServices().getServerName(), storeFiles); WALUtil.writeRegionEventMarker(wal, getReplicationScope(), getRegionInfo(), regionEventDesc, mvcc); // Store SeqId in WAL FileSystem when a region closes // checking region folder exists is due to many tests which delete the table folder while a // table is still online if (getWalFileSystem().exists(getWALRegionDir())) { WALSplitter.writeRegionSequenceIdFile(getWalFileSystem(), getWALRegionDir(), mvcc.getReadPoint()); } }
final List<HRegion> regions = new ArrayList<>(); Mockito.doReturn(regions).when(rss).getRegions(TABLENAME); Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
if (region.getRegionServerServices() != null) { favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion( region.getRegionInfo().getEncodedName());
@Test public void testSpaceQuotaViolation() throws IOException, InterruptedException { region.getRegionServerServices().getRegionServerSpaceQuotaManager().enforceViolationPolicy(NAME, new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 10L, 100L)); Tracker tracker = new Tracker(); TRACKER = tracker; region.requestCompaction("test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertEquals(2, tracker.notExecutedStores.size()); tracker.notExecutedStores.sort((p1, p2) -> p1.getFirst().getColumnFamilyName() .compareTo(p2.getFirst().getColumnFamilyName())); assertEquals(Bytes.toString(CF1), tracker.notExecutedStores.get(0).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(0).getSecond(), containsString("space quota violation")); assertEquals(Bytes.toString(CF2), tracker.notExecutedStores.get(1).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(1).getSecond(), containsString("space quota violation")); assertTrue(tracker.beforeExecuteStores.isEmpty()); assertTrue(tracker.afterExecuteStores.isEmpty()); } }
RegionServerServices rsService = store.getHRegion().getRegionServerServices(); if (rsService != null && scanInfo.isParallelSeekEnabled()) { this.parallelSeekEnabled = true;
private void checkResources() throws RegionTooBusyException { // If catalog region, do not impose resource constraints or block updates. if (this.getRegionInfo().isMetaRegion()) return; if (this.memstoreSize.get() > this.blockingMemStoreSize) { blockedRequestsCount.increment(); requestFlush(); throw new RegionTooBusyException("Above memstore limit, " + "regionName=" + (this.getRegionInfo() == null ? "unknown" : this.getRegionInfo().getRegionNameAsString()) + ", server=" + (this.getRegionServerServices() == null ? "unknown" : this.getRegionServerServices().getServerName()) + ", memstoreSize=" + memstoreSize.get() + ", blockingMemStoreSize=" + blockingMemStoreSize); } }
private void writeRegionOpenMarker(WAL wal, long openSeqId) throws IOException { Map<byte[], List<Path>> storeFiles = getStoreFiles(); RegionEventDescriptor regionOpenDesc = ProtobufUtil.toRegionEventDescriptor( RegionEventDescriptor.EventType.REGION_OPEN, getRegionInfo(), openSeqId, getRegionServerServices().getServerName(), storeFiles); WALUtil.writeRegionEventMarker(wal, getTableDesc(), getRegionInfo(), regionOpenDesc, mvcc); }
private void writeRegionCloseMarker(WAL wal) throws IOException { Map<byte[], List<Path>> storeFiles = getStoreFiles(); RegionEventDescriptor regionEventDesc = ProtobufUtil.toRegionEventDescriptor( RegionEventDescriptor.EventType.REGION_CLOSE, getRegionInfo(), mvcc.getReadPoint(), getRegionServerServices().getServerName(), storeFiles); WALUtil.writeRegionEventMarker(wal, getTableDesc(), getRegionInfo(), regionEventDesc, mvcc); // Store SeqId in HDFS when a region closes // checking region folder exists is due to many tests which delete the table folder while a // table is still online if (this.fs.getFileSystem().exists(this.fs.getRegionDir())) { WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(), this.fs.getRegionDir(), mvcc.getReadPoint(), 0); } }