@Override public void delete(String tableName, List<byte[]> rowIds, String visibilityLabel) throws IOException { List<Delete> deletes = new ArrayList<>(); for (int index = 0; index < rowIds.size(); index++) { Delete delete = new Delete(rowIds.get(index)); if (!StringUtils.isBlank(visibilityLabel)) { delete.setCellVisibility(new CellVisibility(visibilityLabel)); } deletes.add(delete); } batchDelete(tableName, deletes); }
@Override public void put(final String tableName, final byte[] rowId, final Collection<PutColumn> columns) throws IOException { try (final Table table = connection.getTable(TableName.valueOf(tableName))) { table.put(buildPuts(rowId, new ArrayList(columns))); } }
@Override public void delete(final String tableName, final byte[] rowId) throws IOException { delete(tableName, rowId, null); }
@Override protected void init(ControllerServiceInitializationContext config) throws InitializationException { kerberosConfigFile = config.getKerberosConfigurationFile(); kerberosProperties = getKerberosProperties(kerberosConfigFile); List<PropertyDescriptor> props = new ArrayList<>(); props.add(HADOOP_CONF_FILES); props.add(KERBEROS_CREDENTIALS_SERVICE); props.add(kerberosProperties.getKerberosPrincipal()); props.add(kerberosProperties.getKerberosKeytab()); props.add(ZOOKEEPER_QUORUM); props.add(ZOOKEEPER_CLIENT_PORT); props.add(ZOOKEEPER_ZNODE_PARENT); props.add(HBASE_CLIENT_RETRIES); props.add(PHOENIX_CLIENT_JAR_LOCATION); props.addAll(getAdditionalProperties()); this.properties = Collections.unmodifiableList(props); }
@Override public void scan(final String tableName, final byte[] startRow, final byte[] endRow, final Collection<Column> columns, List<String> authorizations, final ResultHandler handler) throws IOException { try (final Table table = connection.getTable(TableName.valueOf(tableName)); final ResultScanner scanner = getResults(table, startRow, endRow, columns, authorizations)) { for (final Result result : scanner) { final byte[] rowKey = result.getRow(); final Cell[] cells = result.rawCells(); if (cells == null) { continue; } // convert HBase cells to NiFi cells final ResultCell[] resultCells = new ResultCell[cells.length]; for (int i=0; i < cells.length; i++) { final Cell cell = cells[i]; final ResultCell resultCell = getResultCell(cell); resultCells[i] = resultCell; } // delegate to the handler handler.handle(rowKey, resultCells); } } }
getLogger().debug("Reloading validation resources"); resources = new ValidationResources(configFiles, getConfigurationFromFiles(configFiles)); validationResourceHolder.set(resources); problems.addAll(KerberosProperties.validatePrincipalAndKeytab(getClass().getSimpleName(), hbaseConfig, resolvedPrincipal, resolvedKeytab, getLogger()));
@OnDisabled public void shutdown() { if (connection != null) { try { connection.close(); } catch (final IOException ioe) { getLogger().warn("Failed to close connection to HBase due to {}", new Object[]{ioe}); } } }
this.connection = createConnection(context);
@Override public void scan(final String tableName, final String startRow, final String endRow, String filterExpression, final Long timerangeMin, final Long timerangeMax, final Integer limitRows, final Boolean isReversed, final Collection<Column> columns, List<String> visibilityLabels, final ResultHandler handler) throws IOException { try (final Table table = connection.getTable(TableName.valueOf(tableName)); final ResultScanner scanner = getResults(table, startRow, endRow, filterExpression, timerangeMin, timerangeMax, limitRows, isReversed, columns, visibilityLabels)) { int cnt = 0; final int lim = limitRows != null ? limitRows : 0; for (final Result result : scanner) { if (lim > 0 && ++cnt > lim){ break; } final byte[] rowKey = result.getRow(); final Cell[] cells = result.rawCells(); if (cells == null) { continue; } // convert HBase cells to NiFi cells final ResultCell[] resultCells = new ResultCell[cells.length]; for (int i = 0; i < cells.length; i++) { final Cell cell = cells[i]; final ResultCell resultCell = getResultCell(cell); resultCells[i] = resultCell; } // delegate to the handler handler.handle(rowKey, resultCells); } } }
protected Connection createConnection(final ConfigurationContext context) throws IOException, InterruptedException { final String configFiles = context.getProperty(HADOOP_CONF_FILES).evaluateAttributeExpressions().getValue(); final Configuration hbaseConfig = getConfigurationFromFiles(configFiles); getLogger().info("HBase Security Enabled, logging in as principal {} with keytab {}", new Object[] {principal, keyTab}); ugi = SecurityUtil.loginKerberos(hbaseConfig, principal, keyTab); getLogger().info("Successfully logged in as principal {} with keytab {}", new Object[] {principal, keyTab}); getLogger().info("Simple Authentication"); return ConnectionFactory.createConnection(hbaseConfig);
@Override protected void init(ControllerServiceInitializationContext config) throws InitializationException { kerberosConfigFile = config.getKerberosConfigurationFile(); kerberosProperties = getKerberosProperties(kerberosConfigFile); List<PropertyDescriptor> props = new ArrayList<>(); props.add(HADOOP_CONF_FILES); props.add(KERBEROS_CREDENTIALS_SERVICE); props.add(kerberosProperties.getKerberosPrincipal()); props.add(kerberosProperties.getKerberosKeytab()); props.add(ZOOKEEPER_QUORUM); props.add(ZOOKEEPER_CLIENT_PORT); props.add(ZOOKEEPER_ZNODE_PARENT); props.add(HBASE_CLIENT_RETRIES); props.add(PHOENIX_CLIENT_JAR_LOCATION); props.addAll(getAdditionalProperties()); this.properties = Collections.unmodifiableList(props); }
getLogger().error("Error writing cell visibility statement.", de); throw new RuntimeException(de);
/** * As of Apache NiFi 1.5.0, due to changes made to * {@link SecurityUtil#loginKerberos(Configuration, String, String)}, which is used by this * class to authenticate a principal with Kerberos, HBase controller services no longer * attempt relogins explicitly. For more information, please read the documentation for * {@link SecurityUtil#loginKerberos(Configuration, String, String)}. * <p/> * In previous versions of NiFi, a {@link org.apache.nifi.hadoop.KerberosTicketRenewer} was started * when the HBase controller service was enabled. The use of a separate thread to explicitly relogin could cause * race conditions with the implicit relogin attempts made by hadoop/HBase code on a thread that references the same * {@link UserGroupInformation} instance. One of these threads could leave the * {@link javax.security.auth.Subject} in {@link UserGroupInformation} to be cleared or in an unexpected state * while the other thread is attempting to use the {@link javax.security.auth.Subject}, resulting in failed * authentication attempts that would leave the HBase controller service in an unrecoverable state. * * @see SecurityUtil#loginKerberos(Configuration, String, String) */ @OnEnabled public void onEnabled(final ConfigurationContext context) throws InitializationException, IOException, InterruptedException { this.connection = createConnection(context); // connection check if (this.connection != null) { final Admin admin = this.connection.getAdmin(); if (admin != null) { admin.listTableNames(); masterAddress = admin.getClusterStatus().getMaster().getHostAndPort(); } } }
@Override public void scan(String tableName, Collection<Column> columns, String filterExpression, long minTime, List<String> visibilityLabels, ResultHandler handler) throws IOException { Filter filter = null; if (!StringUtils.isBlank(filterExpression)) { ParseFilter parseFilter = new ParseFilter(); filter = parseFilter.parseFilterString(filterExpression); } try (final Table table = connection.getTable(TableName.valueOf(tableName)); final ResultScanner scanner = getResults(table, columns, filter, minTime, visibilityLabels)) { for (final Result result : scanner) { final byte[] rowKey = result.getRow(); final Cell[] cells = result.rawCells(); if (cells == null) { continue; } // convert HBase cells to NiFi cells final ResultCell[] resultCells = new ResultCell[cells.length]; for (int i=0; i < cells.length; i++) { final Cell cell = cells[i]; final ResultCell resultCell = getResultCell(cell); resultCells[i] = resultCell; } // delegate to the handler handler.handle(rowKey, resultCells); } } }
getLogger().debug("Reloading validation resources"); resources = new ValidationResources(configFiles, getConfigurationFromFiles(configFiles)); validationResourceHolder.set(resources); problems.addAll(KerberosProperties.validatePrincipalAndKeytab(getClass().getSimpleName(), hbaseConfig, resolvedPrincipal, resolvedKeytab, getLogger()));
@OnDisabled public void shutdown() { if (connection != null) { try { connection.close(); } catch (final IOException ioe) { getLogger().warn("Failed to close connection to HBase due to {}", new Object[]{ioe}); } } }
@Override public void put(final String tableName, final Collection<PutFlowFile> puts) throws IOException { try (final Table table = connection.getTable(TableName.valueOf(tableName))) { // Create one Put per row.... final Map<String, List<PutColumn>> sorted = new HashMap<>(); final List<Put> newPuts = new ArrayList<>(); for (final PutFlowFile putFlowFile : puts) { final String rowKeyString = new String(putFlowFile.getRow(), StandardCharsets.UTF_8); List<PutColumn> columns = sorted.get(rowKeyString); if (columns == null) { columns = new ArrayList<>(); sorted.put(rowKeyString, columns); } columns.addAll(putFlowFile.getColumns()); } for (final Map.Entry<String, List<PutColumn>> entry : sorted.entrySet()) { newPuts.addAll(buildPuts(entry.getKey().getBytes(StandardCharsets.UTF_8), entry.getValue())); } table.put(newPuts); } }
@Override public void deleteCells(String tableName, List<DeleteRequest> deletes) throws IOException { List<Delete> deleteRequests = new ArrayList<>(); for (int index = 0; index < deletes.size(); index++) { DeleteRequest req = deletes.get(index); Delete delete = new Delete(req.getRowId()) .addColumn(req.getColumnFamily(), req.getColumnQualifier()); if (!StringUtils.isEmpty(req.getVisibilityLabel())) { delete.setCellVisibility(new CellVisibility(req.getVisibilityLabel())); } deleteRequests.add(delete); } batchDelete(tableName, deleteRequests); }
@Override public void delete(String tableName, List<byte[]> rowIds) throws IOException { delete(tableName, rowIds); }
@Override public void scan(final String tableName, final byte[] startRow, final byte[] endRow, final Collection<Column> columns, List<String> authorizations, final ResultHandler handler) throws IOException { try (final Table table = connection.getTable(TableName.valueOf(tableName)); final ResultScanner scanner = getResults(table, startRow, endRow, columns, authorizations)) { for (final Result result : scanner) { final byte[] rowKey = result.getRow(); final Cell[] cells = result.rawCells(); if (cells == null) { continue; } // convert HBase cells to NiFi cells final ResultCell[] resultCells = new ResultCell[cells.length]; for (int i=0; i < cells.length; i++) { final Cell cell = cells[i]; final ResultCell resultCell = getResultCell(cell); resultCells[i] = resultCell; } // delegate to the handler handler.handle(rowKey, resultCells); } } }