public double getHoodieIndexQPSFraction() { final double value = getProperty(HOODIE_INDEX_QPS_FRACTION, DEFAULT_HOODIE_INDEX_QPS_FRACTION); Preconditions.checkState(value > 0 && value <= 1, "%s must be between 0 and 1", HOODIE_INDEX_QPS_FRACTION); return value; }
/** * Configure the Hoodie HBase index. */ public HoodieIndexConfig configureHoodieIndex() { final String version; if (getVersion().isPresent()) { version = getVersion().get(); } else { version = DEFAULT_VERSION; } final String topicName = getTableName(); final HoodieIndexConfig.Builder builder = HoodieIndexConfig.newBuilder() .withIndexType(getHoodieIndexType()); if (HoodieIndex.IndexType.HBASE.equals(getHoodieIndexType())) { final String quorum = getHoodieIndexZookeeperQuorum(); final Integer port = getHoodieIndexZookeeperPort(); final String zkZnodeParent = getZkZnodeParent(); createHbaseIndexTableIfNotExists(topicName, quorum, port.toString(), zkZnodeParent, version); } return builder.build(); }
private String getHoodiePropertyFromConfig(@NonNull final Configuration conf, @NotEmpty final String property, @NonNull final Optional<String> version) { final HoodieIndexConfiguration indexConf = new HoodieIndexConfiguration(conf, TABLE_KEY); final HoodieIndexConfig hoodieIndexConfig = indexConf.configureHoodieIndex(); final Properties props = hoodieIndexConfig.getProps(); return props.getProperty(property); }
public void createHbaseIndexTableIfNotExists(@NotEmpty final String dataFeed, @NotEmpty final String zkQuorum, @NotEmpty final String zkPort, @NotEmpty final String zkZnodeParent, @NotEmpty final String version) { final String tableName = getHoodieHbaseIndexTableName(); final String family = "_s"; final org.apache.hadoop.conf.Configuration hbaseConfig = new org.apache.hadoop.conf.Configuration(); hbaseConfig.set("hbase.zookeeper.quorum", zkQuorum); hbaseConfig.set("hbase.zookeeper.property.clientPort", zkPort); hbaseConfig.set("zookeeper.znode.parent", zkZnodeParent); try { try (final Connection connection = ConnectionFactory.createConnection(hbaseConfig)) { if (!connection.getAdmin().tableExists(TableName.valueOf(tableName))) { final HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName)); final HColumnDescriptor familyDesc = new HColumnDescriptor(Bytes.toBytes(family)); familyDesc.setBloomFilterType(BloomType.ROW); familyDesc.setCompressionType(Compression.Algorithm.SNAPPY); tableDesc.addFamily(familyDesc); connection.getAdmin().createTable(tableDesc); log.info("Created HBase table {} with family {}", tableName, family); } else { log.debug("HBase table {} exists", tableName); } } } catch (IOException e) { //todo: better handle try catch log.error("Error creating HBase table {} ", tableName, e); throw new JobRuntimeException(e); } } }
builder.withIndexConfig(new HoodieIndexConfiguration(getConf(), getTableKey()).configureHoodieIndex());
public int getHoodieIndexMaxQpsPerRegionServer() { final int value = getProperty(HOODIE_INDEX_MAX_QPS_PER_REGION_SERVER, DEFAULT_HOODIE_INDEX_MAX_QPS_PER_REGION_SERVER); Preconditions.checkState(value > 0, "%s must be greater than zero", HOODIE_INDEX_MAX_QPS_PER_REGION_SERVER); return value; }
public int getHoodieIndexZookeeperPort() { final int value = getProperty(HOODIE_INDEX_ZOKEEPER_PORT, 0); Preconditions.checkState(value > 0, "%s must be greater than zero", HOODIE_INDEX_ZOKEEPER_PORT); return value; }
public int getHoodieIndexGetBatchSize() { final int value = getProperty(HOODIE_INDEX_GET_BATCH_SIZE, DEFAULT_HOODIE_INDEX_GET_BATCH_SIZE); Preconditions.checkState(value > 0, "%s must be greater than zero", HOODIE_INDEX_GET_BATCH_SIZE); return value; }
public String getZkZnodeParent() { final String value = getProperty(HOODIE_INDEX_HBASE_ZK_ZNODEPARENT, StringTypes.EMPTY); Preconditions.checkState(!value.isEmpty(), "%s must always be set", HOODIE_INDEX_HBASE_ZK_ZNODEPARENT); return value; }
public String getHoodieIndexZookeeperQuorum() { final String value = getProperty(HOODIE_INDEX_ZOOKEEPER_QUORUM, StringTypes.EMPTY); Preconditions.checkState(!value.isEmpty(), "%s must not be empty", HOODIE_INDEX_ZOOKEEPER_QUORUM); return value; }
public String getHoodieHbaseIndexTableName() { final String value = getProperty(HOODIE_HBASE_INDEX_TABLE_NAME, StringTypes.EMPTY); Preconditions.checkState(!value.isEmpty(), "%s must not be empty", HOODIE_HBASE_INDEX_TABLE_NAME); return value; }
public HoodieIndex.IndexType getHoodieIndexType() { final String indexName = getProperty(HOODIE_INDEX_TYPE, DEFAULT_HOODIE_INDEX_TYPE); if (HOODIE_BLOOM_INDEX.equals(indexName.toLowerCase())) { return HoodieIndex.IndexType.BLOOM; } else if (HOODIE_HBASE_INDEX.equals(indexName.toLowerCase())) { return HoodieIndex.IndexType.HBASE; } else { throw new IllegalStateException("Unsupported index type " + indexName); } }