CoprocessorManager coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil); Location location = coprocessorManager.ensureCoprocessorExists(overwrite); LOG.info("coprocessor exists at {}.", location); } catch (IOException e) {
/** * Get the location of the coprocessor and ensure it exists. * In distributed mode, the coprocessor jar is loaded onto hdfs by the CoprocessorBuildTool, * but in other modes it is still useful to create the jar on demand. * * @return the location of the coprocessor * @throws IOException if there was an issue accessing the location */ public synchronized Location ensureCoprocessorExists() throws IOException { return ensureCoprocessorExists(false); }
protected AbstractHBaseDataSetAdmin(TableId tableId, Configuration hConf, CConfiguration cConf, HBaseTableUtil tableUtil, LocationFactory locationFactory) { this.tableId = tableId; this.hConf = hConf; this.cConf = cConf; this.tableUtil = tableUtil; this.tablePrefix = cConf.get(Constants.Dataset.TABLE_PREFIX); this.ddlExecutorFactory = new HBaseDDLExecutorFactory(cConf, hConf); this.coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil); }
protected void addCoprocessor(HTableDescriptorBuilder tableDescriptor, Class<? extends Coprocessor> coprocessor, Integer priority) throws IOException { CoprocessorDescriptor descriptor = coprocessorManager.getCoprocessorDescriptor(coprocessor, priority); Path path = descriptor.getPath() == null ? null : new Path(descriptor.getPath()); tableDescriptor.addCoprocessor(descriptor.getClassName(), path, descriptor.getPriority(), descriptor.getProperties()); }
final Location targetPath = jarDir.append(getCoprocessorName()); if (!overwrite && targetPath.exists()) { return targetPath;
/** * Get the location of the coprocessor and ensure it exists. * In distributed mode, the coprocessor jar is loaded onto hdfs by the CoprocessorBuildTool, * but in other modes it is still useful to create the jar on demand. * * @return the location of the coprocessor * @throws IOException if there was an issue accessing the location */ public synchronized Location ensureCoprocessorExists() throws IOException { return ensureCoprocessorExists(false); }
protected AbstractHBaseDataSetAdmin(TableId tableId, Configuration hConf, CConfiguration cConf, HBaseTableUtil tableUtil, LocationFactory locationFactory) { this.tableId = tableId; this.hConf = hConf; this.cConf = cConf; this.tableUtil = tableUtil; this.tablePrefix = cConf.get(Constants.Dataset.TABLE_PREFIX); this.ddlExecutorFactory = new HBaseDDLExecutorFactory(cConf, hConf); this.coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil); }
protected void addCoprocessor(HTableDescriptorBuilder tableDescriptor, Class<? extends Coprocessor> coprocessor, Integer priority) throws IOException { CoprocessorDescriptor descriptor = coprocessorManager.getCoprocessorDescriptor(coprocessor, priority); Path path = descriptor.getPath() == null ? null : new Path(descriptor.getPath()); tableDescriptor.addCoprocessor(descriptor.getClassName(), path, descriptor.getPriority(), descriptor.getProperties()); }
final Location targetPath = jarDir.append(getCoprocessorName()); if (!overwrite && targetPath.exists()) { return targetPath;
/** * Get the descriptor for a single coprocessor that uses the pre-built coprocessor jar. */ public CoprocessorDescriptor getCoprocessorDescriptor(Class<? extends Coprocessor> coprocessor, @Nullable Integer priority) throws IOException { if (priority == null) { priority = Coprocessor.PRIORITY_USER; } Location jarFile = ensureCoprocessorExists(); String jarPath = manageCoprocessors ? jarFile.toURI().getPath() : null; return new CoprocessorDescriptor(coprocessor.getName(), jarPath, priority, null); }
@Inject HBaseTableFactory(CConfiguration cConf, Configuration hConf, HBaseTableUtil tableUtil, LocationFactory locationFactory) { this.cConf = cConf; this.hConf = hConf; this.tableUtil = tableUtil; this.tableDescriptors = new ConcurrentHashMap<>(); this.coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil); RejectedExecutionHandler callerRunsPolicy = new RejectedExecutionHandler() { @Override public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { REJECTION_LOG.info( "No more threads in the HBase scan thread pool. Consider increase {}. Scan from caller thread {}", Constants.MessagingSystem.HBASE_MAX_SCAN_THREADS, Thread.currentThread().getName() ); // Runs it from the caller thread if (!executor.isShutdown()) { r.run(); } } }; // Creates a executor that will shrink to 0 threads if left idle // Uses daemon thread, hence no need to worry about shutdown // When all threads are busy, use the caller thread to execute this.scanExecutor = new ThreadPoolExecutor(0, cConf.getInt(Constants.MessagingSystem.HBASE_MAX_SCAN_THREADS), 60L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), Threads.createDaemonThreadFactory("messaging-hbase-scanner-%d"), callerRunsPolicy); this.ddlExecutorFactory = new HBaseDDLExecutorFactory(cConf, hConf); }
@Override public void create() throws IOException { // Create the queue table TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf); for (String key : properties.stringPropertyNames()) { tdBuilder.addProperty(key, properties.getProperty(key)); } ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); // Add coprocessors CoprocessorJar coprocessorJar = createCoprocessorJar(); for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) { tdBuilder.addCoprocessor( coprocessorManager.getCoprocessorDescriptor(coprocessor, coprocessorJar.getPriority(coprocessor))); } // Create queue table with splits. The distributor bucket size is the same as splits. int splits = cConf.getInt(QueueConstants.ConfigKeys.QUEUE_TABLE_PRESPLITS); AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits)); byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor); tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits)); createQueueTable(tdBuilder, splitKeys); }
/** * Get the descriptor for a single coprocessor that uses the pre-built coprocessor jar. */ public CoprocessorDescriptor getCoprocessorDescriptor(Class<? extends Coprocessor> coprocessor, @Nullable Integer priority) throws IOException { if (priority == null) { priority = Coprocessor.PRIORITY_USER; } Location jarFile = ensureCoprocessorExists(); String jarPath = manageCoprocessors ? jarFile.toURI().getPath() : null; return new CoprocessorDescriptor(coprocessor.getName(), jarPath, priority, null); }
UpgradeTool() throws Exception { this.cConf = CConfiguration.create(); if (this.cConf.getBoolean(Constants.Security.Authorization.ENABLED)) { LOG.info("Disabling authorization for {}.", getClass().getSimpleName()); this.cConf.setBoolean(Constants.Security.Authorization.ENABLED, false); } // Note: login has to happen before any objects that need Kerberos credentials are instantiated. SecurityUtil.loginForMasterService(cConf); this.hConf = HBaseConfiguration.create(); Injector injector = createInjector(); this.txService = injector.getInstance(TransactionService.class); this.zkClientService = injector.getInstance(ZKClientService.class); this.dsFramework = injector.getInstance(DatasetFramework.class); this.dsUpgrade = injector.getInstance(DatasetUpgrader.class); this.tmsTableFactory = injector.getInstance(HBaseTableFactory.class); LocationFactory locationFactory = injector.getInstance(LocationFactory.class); HBaseTableUtil tableUtil = injector.getInstance(HBaseTableUtil.class); this.coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { UpgradeTool.this.stop(); } catch (Throwable e) { LOG.error("Failed to upgrade", e); } } }); }
coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER); Path path = coprocessorDescriptor.getPath() == null ? null : new Path(coprocessorDescriptor.getPath()); newDescriptor.addCoprocessor(coprocessorDescriptor.getClassName(), path, coprocessorDescriptor.getPriority(),
@Override protected CoprocessorJar createCoprocessorJar() throws IOException { List<? extends Class<? extends Coprocessor>> coprocessors = getCoprocessors(); if (coprocessors.isEmpty()) { return CoprocessorJar.EMPTY; } Location jarFile = coprocessorManager.ensureCoprocessorExists(); return new CoprocessorJar(coprocessors, jarFile); }
this.tableUtil = tableUtil; this.tableDescriptors = new ConcurrentHashMap<>(); this.coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil);
coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER); Path path = coprocessorDescriptor.getPath() == null ? null : new Path(coprocessorDescriptor.getPath()); newDescriptor.addCoprocessor(coprocessorDescriptor.getClassName(), path, coprocessorDescriptor.getPriority(),
public static CoprocessorJar createCoprocessorJarInternal(CConfiguration conf, CoprocessorManager coprocessorManager, HBaseTableUtil tableUtil, boolean transactional, boolean supportsReadlessIncrement) throws IOException { Class<? extends Coprocessor> dataJanitorClass = tableUtil.getTransactionDataJanitorClassForVersion(); Class<? extends Coprocessor> incrementClass = tableUtil.getIncrementHandlerClassForVersion(); // The ordering of coprocessors is important here. DataJanitor Coprocessor should get higher priority than // IncrementHandler coprocessor. This is because, we have a check in prePutOp, preDeleteOp in DataJanitor // to make sure the operation is within the tx max lifetime. ImmutableList.Builder<Class<? extends Coprocessor>> coprocessors = ImmutableList.builder(); if (transactional) { // tx janitor if (conf.getBoolean(Constants.Transaction.DataJanitor.CFG_TX_JANITOR_ENABLE, Constants.Transaction.DataJanitor.DEFAULT_TX_JANITOR_ENABLE)) { coprocessors.add(dataJanitorClass); } } // read-less increments if (supportsReadlessIncrement) { coprocessors.add(incrementClass); } ImmutableList<Class<? extends Coprocessor>> coprocessorList = coprocessors.build(); if (coprocessorList.isEmpty()) { return CoprocessorJar.EMPTY; } Location jarFile = coprocessorManager.ensureCoprocessorExists(); return new CoprocessorJar(coprocessorList, jarFile); }
.addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER));