/** * Test default value handling for maxFileSize */ @Test public void testGetMaxFileSize() { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); assertEquals(-1, desc.getMaxFileSize()); desc.setMaxFileSize(1111L); assertEquals(1111L, desc.getMaxFileSize()); }
@Test public void testPb() throws DeserializationException, IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); final int v = 123; htd.setMaxFileSize(v); htd.setDurability(Durability.ASYNC_WAL); htd.setReadOnly(true); htd.setRegionReplication(2); byte [] bytes = htd.toByteArray(); HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(bytes); assertEquals(htd, deserializedHtd); assertEquals(v, deserializedHtd.getMaxFileSize()); assertTrue(deserializedHtd.isReadOnly()); assertEquals(Durability.ASYNC_WAL, deserializedHtd.getDurability()); assertEquals(2, deserializedHtd.getRegionReplication()); }
@Test public void testModifyTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf"); UTIL.getAdmin().disableTable(tableName); // Modify the table descriptor HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); // Test 1: Modify 1 property long newMaxFileSize = htd.getMaxFileSize() * 2; htd.setMaxFileSize(newMaxFileSize); htd.setRegionReplication(3); long procId1 = ProcedureTestingUtility.submitAndWait( procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); assertEquals(newMaxFileSize, currentHtd.getMaxFileSize()); // Test 2: Modify multiple properties boolean newReadOnlyOption = htd.isReadOnly() ? false : true; long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2; htd.setReadOnly(newReadOnlyOption); htd.setMemStoreFlushSize(newMemStoreFlushSize); long procId2 = ProcedureTestingUtility.submitAndWait( procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2)); currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); assertEquals(newReadOnlyOption, currentHtd.isReadOnly()); assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize()); }
/** * Test default value handling for maxFileSize */ @Test public void testGetMaxFileSize() { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); assertEquals(-1, desc.getMaxFileSize()); desc.setMaxFileSize(1111L); assertEquals(1111L, desc.getMaxFileSize()); }
HTableDescriptor descriptor = connection.getQueryServices().getTableDescriptor(parent.getPhysicalName().getBytes()); if (descriptor != null) { // Is null for connectionless long maxFileSize = descriptor.getMaxFileSize(); if (maxFileSize == -1) { // If unset, use default maxFileSize = HConstants.DEFAULT_MAX_FILE_SIZE;
@Test public void testPb() throws DeserializationException, IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); final int v = 123; htd.setMaxFileSize(v); htd.setDurability(Durability.ASYNC_WAL); htd.setReadOnly(true); htd.setRegionReplication(2); byte [] bytes = htd.toByteArray(); HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(bytes); assertEquals(htd, deserializedHtd); assertEquals(v, deserializedHtd.getMaxFileSize()); assertTrue(deserializedHtd.isReadOnly()); assertEquals(Durability.ASYNC_WAL, deserializedHtd.getDurability()); assertEquals(2, deserializedHtd.getRegionReplication()); }
@Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); Configuration conf = getConf(); HTableDescriptor desc = region.getTableDesc(); if (desc != null) { this.desiredMaxFileSize = desc.getMaxFileSize(); } if (this.desiredMaxFileSize <= 0) { this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); } }
@Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); Configuration conf = getConf(); HTableDescriptor desc = region.getTableDesc(); if (desc != null) { this.desiredMaxFileSize = desc.getMaxFileSize(); } if (this.desiredMaxFileSize <= 0) { this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); } double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D); this.desiredMaxFileSize += (long)(desiredMaxFileSize * (RANDOM.nextFloat() - 0.5D) * jitter); }
@Override public int compare(HTableDescriptor o1, HTableDescriptor o2) { int nameResult = o1.getNameAsString().compareTo(o2.getNameAsString()); if (nameResult != 0) { return nameResult; } int maxFileSizeResult = Long.valueOf(o1.getMaxFileSize()).compareTo(o2.getMaxFileSize()); if (maxFileSizeResult != 0) { return maxFileSizeResult; } int memstoreFlushSizeResult = Long.valueOf(o1.getMemStoreFlushSize()).compareTo(o2.getMemStoreFlushSize()); if (memstoreFlushSizeResult != 0) { return memstoreFlushSizeResult; } HColumnDescriptor[] families1 = o1.getColumnFamilies(); HColumnDescriptor[] families2 = o2.getColumnFamilies(); int familiesResult = Integer.valueOf(families1.length).compareTo(families2.length); if (familiesResult != 0) { return familiesResult; } HColumnDescriptorComparator columnComparator = new HColumnDescriptorComparator(); for (int i = 0; i < families1.length; i++) { int result = columnComparator.compare(families1[i], families2[i]); if (result != 0) { return result; } } return 0; }
if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) { printStream.printf(" Changing max_filesize from %d to %d: %n", currentTableDescriptor.getMaxFileSize(), newTableDescriptor.getMaxFileSize()); if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) { printStream.printf(" Changing memstore_flushsize from %d to %d: %n", currentTableDescriptor.getMemStoreFlushSize(),
static public ATableDescriptor htdToATD(HTableDescriptor table) throws IOException { ATableDescriptor atd = new ATableDescriptor(); atd.name = ByteBuffer.wrap(table.getName()); Collection<HColumnDescriptor> families = table.getFamilies(); Schema afdSchema = Schema.createArray(AFamilyDescriptor.SCHEMA$); GenericData.Array<AFamilyDescriptor> afamilies = null; if (families.size() > 0) { afamilies = new GenericData.Array<AFamilyDescriptor>(families.size(), afdSchema); for (HColumnDescriptor hcd : families) { AFamilyDescriptor afamily = hcdToAFD(hcd); afamilies.add(afamily); } } else { afamilies = new GenericData.Array<AFamilyDescriptor>(0, afdSchema); } atd.families = afamilies; atd.maxFileSize = table.getMaxFileSize(); atd.memStoreFlushSize = table.getMemStoreFlushSize(); atd.rootRegion = table.isRootRegion(); atd.metaRegion = table.isMetaRegion(); atd.metaTable = table.isMetaTable(); atd.readOnly = table.isReadOnly(); atd.deferredLogFlush = table.isDeferredLogFlush(); return atd; }
long maxFileSize = htd.getMaxFileSize(); if (maxFileSize < 0) { maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);
table.setAttribute(ATTR_TABLE_MAX_FILESIZE, htableDescriptor.getMaxFileSize()); table.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htableDescriptor.getRegionReplication()); table.setAttribute(ATTR_TABLE_ISREADONLY, htableDescriptor.isReadOnly());
table.setAttribute(ATTR_TABLE_MAX_FILESIZE, htableDescriptor.getMaxFileSize()); table.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htableDescriptor.getRegionReplication()); table.setAttribute(ATTR_TABLE_ISREADONLY, htableDescriptor.isReadOnly());
private AtlasEntity getTableEntity(String nameSpace, String tableName, String owner, AtlasEntity nameSpaceEntity, HTableDescriptor htd, AtlasEntity atlasEntity) { AtlasEntity ret = null; if (atlasEntity == null) { ret = new AtlasEntity(HBaseDataTypes.HBASE_TABLE.getName()); } else { ret = atlasEntity; } String tableQualifiedName = getTableQualifiedName(clusterName, nameSpace, tableName); ret.setAttribute(QUALIFIED_NAME, tableQualifiedName); ret.setAttribute(CLUSTERNAME, clusterName); ret.setAttribute(NAMESPACE, AtlasTypeUtil.getAtlasObjectId(nameSpaceEntity)); ret.setAttribute(NAME, tableName); ret.setAttribute(DESCRIPTION_ATTR, tableName); ret.setAttribute(OWNER, owner); ret.setAttribute(URI, tableName); ret.setAttribute(ATTR_TABLE_MAX_FILESIZE, htd.getMaxFileSize()); ret.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htd.getRegionReplication()); ret.setAttribute(ATTR_TABLE_ISREADONLY, htd.isReadOnly()); ret.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, htd.isCompactionEnabled()); ret.setAttribute(ATTR_TABLE_DURABLILITY, (htd.getDurability() != null ? htd.getDurability().name() : null)); return ret; }
private AtlasEntity getTableEntity(String nameSpace, String tableName, String owner, AtlasEntity nameSpaceEntity, HTableDescriptor htd, AtlasEntity atlasEntity) { AtlasEntity ret = null; if (atlasEntity == null) { ret = new AtlasEntity(HBaseDataTypes.HBASE_TABLE.getName()); } else { ret = atlasEntity; } String tableQualifiedName = getTableQualifiedName(clusterName, nameSpace, tableName); ret.setAttribute(QUALIFIED_NAME, tableQualifiedName); ret.setAttribute(CLUSTERNAME, clusterName); ret.setAttribute(NAMESPACE, AtlasTypeUtil.getAtlasObjectId(nameSpaceEntity)); ret.setAttribute(NAME, tableName); ret.setAttribute(DESCRIPTION_ATTR, tableName); ret.setAttribute(OWNER, owner); ret.setAttribute(URI, tableName); ret.setAttribute(ATTR_TABLE_MAX_FILESIZE, htd.getMaxFileSize()); ret.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htd.getRegionReplication()); ret.setAttribute(ATTR_TABLE_ISREADONLY, htd.isReadOnly()); ret.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, htd.isCompactionEnabled()); ret.setAttribute(ATTR_TABLE_DURABLILITY, (htd.getDurability() != null ? htd.getDurability().name() : null)); return ret; }
@Test public void testModifyTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf"); UTIL.getAdmin().disableTable(tableName); // Modify the table descriptor HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); // Test 1: Modify 1 property long newMaxFileSize = htd.getMaxFileSize() * 2; htd.setMaxFileSize(newMaxFileSize); htd.setRegionReplication(3); long procId1 = ProcedureTestingUtility.submitAndWait( procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); assertEquals(newMaxFileSize, currentHtd.getMaxFileSize()); // Test 2: Modify multiple properties boolean newReadOnlyOption = htd.isReadOnly() ? false : true; long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2; htd.setReadOnly(newReadOnlyOption); htd.setMemStoreFlushSize(newMemStoreFlushSize); long procId2 = ProcedureTestingUtility.submitAndWait( procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2)); currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); assertEquals(newReadOnlyOption, currentHtd.isReadOnly()); assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize()); }