@Test public void testGetSplitPoint() throws IOException { ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); // For no stores, should not split assertFalse(policy.shouldSplit()); assertNull(policy.getSplitPoint()); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint(); stores.add(mockStore); assertEquals("store 1 split", Bytes.toString(policy.getSplitPoint())); // Add a bigger store. The split point should come from that one HStore mockStore2 = Mockito.mock(HStore.class); Mockito.doReturn(4000L).when(mockStore2).getSize(); Mockito.doReturn(true).when(mockStore2).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint(); stores.add(mockStore2); assertEquals("store 2 split", Bytes.toString(policy.getSplitPoint())); }
@Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); Configuration conf = getConf(); initialSize = conf.getLong("hbase.increasing.policy.initial.size", -1); if (initialSize > 0) { return; } TableDescriptor desc = region.getTableDescriptor(); if (desc != null) { initialSize = 2 * desc.getMemStoreFlushSize(); } if (initialSize <= 0) { initialSize = 2 * conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE); } }
@Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); Configuration conf = getConf(); TableDescriptor desc = region.getTableDescriptor(); if (desc != null) { this.desiredMaxFileSize = desc.getMaxFileSize(); } if (this.desiredMaxFileSize <= 0) { this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); } double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D); this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter; long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate); // make sure the long value won't overflow with jitter if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE - this.desiredMaxFileSize)) { this.desiredMaxFileSize = Long.MAX_VALUE; } else { this.desiredMaxFileSize += jitterValue; } }
@Test public void testConstantSizePolicyWithJitter() throws IOException { conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); htd.setMaxFileSize(Long.MAX_VALUE); boolean positiveJitter = false; ConstantSizeRegionSplitPolicy policy = null; while (!positiveJitter) { policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); positiveJitter = policy.positiveJitterRate(); } // add a store HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE assertFalse(policy.shouldSplit()); }
@Override protected final byte[] getSplitPoint() { return getSplitPoint(super.getSplitPoint()); } }
@Test public void testForceSplitRegionWithReference() throws IOException { htd.setMaxFileSize(1024L); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); stores.add(mockStore); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, IncreasingToUpperBoundRegionSplitPolicy.class.getName()); policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); }
@Test public void testCreateDefault() throws IOException { conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L); // Using a default HTD, should pick up the file size from // configuration. ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( mockRegion, conf); assertWithinJitter(1234L, policy.getDesiredMaxFileSize()); // If specified in HTD, should use that htd.setMaxFileSize(9999L); policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( mockRegion, conf); assertWithinJitter(9999L, policy.getDesiredMaxFileSize()); }
@Override protected byte[] getSplitPoint() { byte[] splitPoint = super.getSplitPoint(); int offset = SchemaUtil.getVarCharLength(splitPoint, 0, splitPoint.length); // Split only on Phoenix schema name, so this is ok b/c we won't be splitting // in the middle of a Phoenix table. if (offset == splitPoint.length) { return splitPoint; } // offset = SchemaUtil.getVarCharLength(splitPoint, offset+1, splitPoint.length-offset-1); // // Split only on Phoenix schema and table name, so this is ok b/c we won't be splitting // // in the middle of a Phoenix table. // if (offset == splitPoint.length) { // return splitPoint; // } // Otherwise, an attempt is being made to split in the middle of a table. // Just return a split point at the schema boundary instead byte[] newSplitPoint = new byte[offset + 1]; System.arraycopy(splitPoint, 0, newSplitPoint, 0, offset+1); return newSplitPoint; }
@Test public void testConstantSizePolicyWithJitter() throws IOException { conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); htd.setMaxFileSize(Long.MAX_VALUE); boolean positiveJitter = false; ConstantSizeRegionSplitPolicy policy = null; while (!positiveJitter) { policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); positiveJitter = policy.positiveJitterRate(); } // add a store HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE assertFalse(policy.shouldSplit()); }
/** * Run through tests for a ConstantSizeRegionSplitPolicy * @param policy */ private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) { // For no stores, should not split assertFalse(policy.shouldSplit()); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); assertTrue(policy.shouldSplit()); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); assertFalse(policy.shouldSplit()); // Reset splittability after above Mockito.doReturn(true).when(mockStore).canSplit(); // Set to a small size but turn on forceSplit. Should result in a split. Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); Mockito.doReturn(100L).when(mockStore).getSize(); assertTrue(policy.shouldSplit()); // Turn off forceSplit, should not split Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); // Clear families we added above stores.clear(); }
@Test public void testCreateDefault() throws IOException { conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L); // Using a default HTD, should pick up the file size from // configuration. ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( mockRegion, conf); assertWithinJitter(1234L, policy.getDesiredMaxFileSize()); // If specified in HTD, should use that htd.setMaxFileSize(9999L); policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( mockRegion, conf); assertWithinJitter(9999L, policy.getDesiredMaxFileSize()); }
@Override protected final byte[] getSplitPoint() { return getSplitPoint(super.getSplitPoint()); } }
@Test public void testGetSplitPoint() throws IOException { ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); // For no stores, should not split assertFalse(policy.shouldSplit()); assertNull(policy.getSplitPoint()); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint(); stores.add(mockStore); assertEquals("store 1 split", Bytes.toString(policy.getSplitPoint())); // Add a bigger store. The split point should come from that one HStore mockStore2 = Mockito.mock(HStore.class); Mockito.doReturn(4000L).when(mockStore2).getSize(); Mockito.doReturn(true).when(mockStore2).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint(); stores.add(mockStore2); assertEquals("store 2 split", Bytes.toString(policy.getSplitPoint())); }
@Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); Configuration conf = getConf(); HTableDescriptor desc = region.getTableDesc(); if (desc != null) { this.flushSize = desc.getMemStoreFlushSize(); } if (this.flushSize <= 0) { this.flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE); } }
@Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); Configuration conf = getConf(); HTableDescriptor desc = region.getTableDesc(); if (desc != null) { this.desiredMaxFileSize = desc.getMaxFileSize(); } if (this.desiredMaxFileSize <= 0) { this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); } }
@Test public void testForceSplitRegionWithReference() throws IOException { htd.setMaxFileSize(1024L); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); stores.add(mockStore); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, IncreasingToUpperBoundRegionSplitPolicy.class.getName()); policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); }
@Override protected final byte[] getSplitPoint() { return getSplitPoint(super.getSplitPoint()); } }
@Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); Configuration conf = getConf(); initialSize = conf.getLong("hbase.increasing.policy.initial.size", -1); if (initialSize > 0) { return; } HTableDescriptor desc = region.getTableDesc(); if (desc != null) { initialSize = 2 * desc.getMemStoreFlushSize(); } if (initialSize <= 0) { initialSize = 2 * conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE); } }
@Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); Configuration conf = getConf(); HTableDescriptor desc = region.getTableDesc(); if (desc != null) { this.desiredMaxFileSize = desc.getMaxFileSize(); } if (this.desiredMaxFileSize <= 0) { this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); } double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D); this.desiredMaxFileSize += (long)(desiredMaxFileSize * (RANDOM.nextFloat() - 0.5D) * jitter); }
/** * Run through tests for a ConstantSizeRegionSplitPolicy * @param policy */ private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) { // For no stores, should not split assertFalse(policy.shouldSplit()); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); assertTrue(policy.shouldSplit()); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); assertFalse(policy.shouldSplit()); // Reset splittability after above Mockito.doReturn(true).when(mockStore).canSplit(); // Set to a small size but turn on forceSplit. Should result in a split. Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); Mockito.doReturn(100L).when(mockStore).getSize(); assertTrue(policy.shouldSplit()); // Turn off forceSplit, should not split Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); // Clear families we added above stores.clear(); }