@Test public void testConstantSizePolicyWithJitter() throws IOException { conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); htd.setMaxFileSize(Long.MAX_VALUE); boolean positiveJitter = false; ConstantSizeRegionSplitPolicy policy = null; while (!positiveJitter) { policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); positiveJitter = policy.positiveJitterRate(); } // add a store HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE assertFalse(policy.shouldSplit()); }
@Test public void testConstantSizePolicy() throws IOException { htd.setMaxFileSize(1024L); ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); doConstantSizePolicyTests(policy); }
/** * Test default value handling for maxFileSize */ @Test public void testGetMaxFileSize() { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); assertEquals(-1, desc.getMaxFileSize()); desc.setMaxFileSize(1111L); assertEquals(1111L, desc.getMaxFileSize()); }
@Test public void testForceSplitRegionWithReference() throws IOException { htd.setMaxFileSize(1024L); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); stores.add(mockStore); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, IncreasingToUpperBoundRegionSplitPolicy.class.getName()); policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); }
htd.setMaxFileSize(10L * 1024 * 1024 * 1024); htd.setValue(HTableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName()); htd.addFamily(new HColumnDescriptor(FAMILY1));
@Test public void testCreateDefault() throws IOException { conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L); // Using a default HTD, should pick up the file size from // configuration. ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( mockRegion, conf); assertWithinJitter(1234L, policy.getDesiredMaxFileSize()); // If specified in HTD, should use that htd.setMaxFileSize(9999L); policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( mockRegion, conf); assertWithinJitter(9999L, policy.getDesiredMaxFileSize()); }
htd.setMaxFileSize(maxSplitSize);
@Test public void testPb() throws DeserializationException, IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); final int v = 123; htd.setMaxFileSize(v); htd.setDurability(Durability.ASYNC_WAL); htd.setReadOnly(true); htd.setRegionReplication(2); byte [] bytes = htd.toByteArray(); HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(bytes); assertEquals(htd, deserializedHtd); assertEquals(v, deserializedHtd.getMaxFileSize()); assertTrue(deserializedHtd.isReadOnly()); assertEquals(Durability.ASYNC_WAL, deserializedHtd.getDurability()); assertEquals(2, deserializedHtd.getRegionReplication()); }
checkTableIsLegal(htd); htd.setMaxFileSize(1024); // 1K checkTableIsIllegal(htd); htd.setMaxFileSize(0); checkTableIsIllegal(htd); htd.setMaxFileSize(1024 * 1024 * 1024); // 1G checkTableIsLegal(htd);
@Test public void testModifyTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf"); UTIL.getAdmin().disableTable(tableName); // Modify the table descriptor HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); // Test 1: Modify 1 property long newMaxFileSize = htd.getMaxFileSize() * 2; htd.setMaxFileSize(newMaxFileSize); htd.setRegionReplication(3); long procId1 = ProcedureTestingUtility.submitAndWait( procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); assertEquals(newMaxFileSize, currentHtd.getMaxFileSize()); // Test 2: Modify multiple properties boolean newReadOnlyOption = htd.isReadOnly() ? false : true; long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2; htd.setReadOnly(newReadOnlyOption); htd.setMemStoreFlushSize(newMemStoreFlushSize); long procId2 = ProcedureTestingUtility.submitAndWait( procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2)); currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); assertEquals(newReadOnlyOption, currentHtd.isReadOnly()); assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize()); }
htd.setMaxFileSize(512 * 1024 * 1024); modifyTableSync(admin, tableName, htd); assertTrue("Test table should have been modified", htd.setMaxFileSize(512 * 1024 * 1024); modifyTableSync(admin, tableName, htd); assertTrue("Test table should have been modified",
htd1 .addFamily(coldef2) .setMaxFileSize(1024 * 1024 * 1024L) .setValue("Description", "Chapter 5 - ModifyTableExample: Modified Table");
/** * Test default value handling for maxFileSize */ @Test public void testGetMaxFileSize() { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); assertEquals(-1, desc.getMaxFileSize()); desc.setMaxFileSize(1111L); assertEquals(1111L, desc.getMaxFileSize()); }
@Test public void testPb() throws DeserializationException, IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); final int v = 123; htd.setMaxFileSize(v); htd.setDurability(Durability.ASYNC_WAL); htd.setReadOnly(true); htd.setRegionReplication(2); byte [] bytes = htd.toByteArray(); HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(bytes); assertEquals(htd, deserializedHtd); assertEquals(v, deserializedHtd.getMaxFileSize()); assertTrue(deserializedHtd.isReadOnly()); assertEquals(Durability.ASYNC_WAL, deserializedHtd.getDurability()); assertEquals(2, deserializedHtd.getRegionReplication()); }
private static boolean createTable(byte[] tableName, byte[] columnFamilyName, short regionCount, long regionMaxSize, HBaseAdmin admin) throws IOException { if (admin.tableExists(tableName)) { return false; } HTableDescriptor tableDescriptor = new HTableDescriptor(); tableDescriptor.setName(tableName); HColumnDescriptor columnDescriptor = new HColumnDescriptor(columnFamilyName); columnDescriptor.setCompressionType(Compression.Algorithm.SNAPPY); columnDescriptor.setBlocksize(64 * 1024); columnDescriptor.setBloomFilterType(BloomType.ROW); columnDescriptor.setMaxVersions(10); tableDescriptor.addFamily(columnDescriptor); tableDescriptor.setMaxFileSize(regionMaxSize); tableDescriptor.setValue(tableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName()); tableDescriptor.setDeferredLogFlush(true); regionCount = (short) Math.abs(regionCount); int regionRange = Short.MAX_VALUE / regionCount; int counter = 0; byte[][] splitKeys = new byte[regionCount][]; for (byte[] splitKey : splitKeys) { counter = counter + regionRange; String key = StringUtils.leftPad(Integer.toString(counter), 5, '0'); splitKey = Bytes.toBytes(key); System.out.println(" - Split: " + splitKey); } return true; }
@Test public void testConstantSizePolicyWithJitter() throws IOException { conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); htd.setMaxFileSize(Long.MAX_VALUE); boolean positiveJitter = false; ConstantSizeRegionSplitPolicy policy = null; while (!positiveJitter) { policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); positiveJitter = policy.positiveJitterRate(); } // add a store HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE assertFalse(policy.shouldSplit()); }
@Test public void testConstantSizePolicy() throws IOException { htd.setMaxFileSize(1024L); ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); doConstantSizePolicyTests(policy); }
@Test public void testForceSplitRegionWithReference() throws IOException { htd.setMaxFileSize(1024L); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); stores.add(mockStore); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, IncreasingToUpperBoundRegionSplitPolicy.class.getName()); policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); }
@Test public void testCreateDefault() throws IOException { conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L); // Using a default HTD, should pick up the file size from // configuration. ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( mockRegion, conf); assertWithinJitter(1234L, policy.getDesiredMaxFileSize()); // If specified in HTD, should use that htd.setMaxFileSize(9999L); policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( mockRegion, conf); assertWithinJitter(9999L, policy.getDesiredMaxFileSize()); }
@Test public void testModifyTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf"); UTIL.getAdmin().disableTable(tableName); // Modify the table descriptor HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); // Test 1: Modify 1 property long newMaxFileSize = htd.getMaxFileSize() * 2; htd.setMaxFileSize(newMaxFileSize); htd.setRegionReplication(3); long procId1 = ProcedureTestingUtility.submitAndWait( procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); assertEquals(newMaxFileSize, currentHtd.getMaxFileSize()); // Test 2: Modify multiple properties boolean newReadOnlyOption = htd.isReadOnly() ? false : true; long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2; htd.setReadOnly(newReadOnlyOption); htd.setMemStoreFlushSize(newMemStoreFlushSize); long procId2 = ProcedureTestingUtility.submitAndWait( procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2)); currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); assertEquals(newReadOnlyOption, currentHtd.isReadOnly()); assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize()); }