/** * Constructor. * * @param registry registry for spectator * @param config Program configuration */ @Autowired public ThreadServiceManager(final Registry registry, final Config config) { final ExecutorService executorService = newFixedThreadPool( config.getServiceMaxNumberOfThreads(), "metacat-service-pool-%d", 1000 ); this.executor = MoreExecutors.listeningDecorator(executorService); RegistryUtil.registerThreadPool(registry, "metacat-service-pool", (ThreadPoolExecutor) executorService); }
private ConnectorTypeConverter getDefaultConverter() { try { return (ConnectorTypeConverter) Class.forName(this.config.getDefaultTypeConverter()).newInstance(); } catch (Exception e) { throw Throwables.propagate(e); } } }
private String getIcebergPartitionURI(final String databaseName, final String tableName, final String partitionName, @Nullable final Long dataTimestampMillis) { return String.format("%s://%s.%s/%s/snapshot_time=%s", context.getConfig().getIcebergPartitionUriScheme(), databaseName, tableName, partitionName, (dataTimestampMillis == null) ? partitionName.hashCode() : Instant.ofEpochMilli(dataTimestampMillis).getEpochSecond()); } }
/** * {@inheritDoc} */ @Override public void checkPermission(final String userName, final QualifiedName name, final MetacatOperation op) { if (config.isAuthorizationEnabled()) { switch (op) { case CREATE: checkPermit(config.getMetacatCreateAcl(), userName, name, op); break; case RENAME: case DELETE: checkPermit(config.getMetacatDeleteAcl(), userName, name, op); break; default: } } }
/** * get iceberg partition map. * * @param icebergTable iceberg table * @param filter iceberg filter expression * @return scan summary map */ public Map<String, ScanSummary.PartitionMetrics> getPartitionMetricsMap(final Table icebergTable, @Nullable final Expression filter) { return (filter != null) ? ScanSummary.of(icebergTable.newScan().filter(filter)) .limit(connectorContext.getConfig().getMaxPartitionsThreshold()) .throwIfLimited() .build() : ScanSummary.of(icebergTable.newScan()) //the top x records .limit(connectorContext.getConfig().getIcebergTableSummaryFetchSize()) .build(); } }
@Override public void deleteMetadata(final String userId, final List<HasMetadata> holders) { try { final List<List<HasMetadata>> subLists = Lists.partition(holders, config.getUserMetadataMaxInClauseItems()); for (List<HasMetadata> hasMetadatas : subLists) { final List<QualifiedName> names = hasMetadatas.stream() .filter(m -> m instanceof HasDefinitionMetadata) .map(m -> ((HasDefinitionMetadata) m).getDefinitionName()) .collect(Collectors.toList()); if (!names.isEmpty()) { _deleteDefinitionMetadata(names); } if (config.canSoftDeleteDataMetadata()) { final List<String> uris = hasMetadatas.stream() .filter(m -> m instanceof HasDataMetadata && ((HasDataMetadata) m).isDataExternal()) .map(m -> ((HasDataMetadata) m).getDataUri()).collect(Collectors.toList()); if (!uris.isEmpty()) { _softDeleteDataMetadata(userId, uris); } } } } catch (Exception e) { log.error("Failed deleting metadatas", e); throw new UserMetadataServiceException("Failed deleting metadatas", e); } }
final String message = String.format("Number of partitions queried for table %s exceeded the threshold %d", tableName, connectorContext.getConfig().getMaxPartitionsThreshold()); log.warn(message); throw new IllegalArgumentException(message);
@Override public TableMetadata refresh() { refreshFromMetadataLocation(this.location, connectorContext.getConfig().getIcebergRefreshFromMetadataLocationRetryNumber()); return current(); }
/** * Constructor. * * @param connectorContext connector context * @param jdbcTemplate JDBC template * @param sequenceGeneration sequence generator * @param fastServiceMetric fast service metric */ public DirectSqlSavePartition(final ConnectorContext connectorContext, final JdbcTemplate jdbcTemplate, final SequenceGeneration sequenceGeneration, final HiveConnectorFastServiceMetric fastServiceMetric) { this.registry = connectorContext.getRegistry(); this.catalogName = connectorContext.getCatalogName(); this.batchSize = connectorContext.getConfig().getHiveMetastoreBatchSize(); this.jdbcTemplate = jdbcTemplate; this.sequenceGeneration = sequenceGeneration; this.fastServiceMetric = fastServiceMetric; }
/** * {@inheritDoc} */ @Override public boolean partition_name_has_valid_characters(final List<String> partVals, final boolean throwException) throws TException { return requestWrapper("partition_name_has_valid_characters", new Object[]{partVals, throwException}, () -> { Pattern pattern = null; final String partitionPattern = config.getHivePartitionWhitelistPattern(); if (!Strings.isNullOrEmpty(partitionPattern)) { pattern = PATTERNS.getUnchecked(partitionPattern); } if (throwException) { MetaStoreUtils.validatePartitionNameCharacters(partVals, pattern); return true; } else { return MetaStoreUtils.partitionNameHasValidCharacters(partVals, pattern); } }); }
tagService.delete(name, false); } else { if (config.canSoftDeleteDataMetadata() && tableDto.isDataExternal()) { userMetadataService.softDeleteDataMetadata(metacatRequestContext.getUserName(), Lists.newArrayList(tableDto.getDataUri()));
/** * {@inheritDoc} */ @Override public void checkPermission(final String userName, final QualifiedName name, final MetacatOperation op) { if (config.isAuthorizationEnabled()) { switch (op) { case CREATE: checkPermit(config.getMetacatCreateAcl(), userName, name, op); break; case RENAME: case DELETE: checkPermit(config.getMetacatDeleteAcl(), userName, name, op); break; default: } } }
/** * get iceberg partition map. * * @param icebergTable iceberg table * @param filter iceberg filter expression * @return scan summary map */ public Map<String, ScanSummary.PartitionMetrics> getPartitionMetricsMap(final Table icebergTable, @Nullable final Expression filter) { return (filter != null) ? ScanSummary.of(icebergTable.newScan().filter(filter)) .limit(connectorContext.getConfig().getMaxPartitionsThreshold()) .throwIfLimited() .build() : ScanSummary.of(icebergTable.newScan()) //the top x records .limit(connectorContext.getConfig().getIcebergTableSummaryFetchSize()) .build(); } }
if (result.size() > config.getMaxPartitionsThreshold()) { registry.counter(registry.createId(HiveMetrics.CounterHiveGetPartitionsExceedThresholdFailure .getMetricName()).withTags(tableQName.parts())).increment(); final String message = String.format("Number of partitions queried for table %s exceeded the threshold %d", tableQName, config.getMaxPartitionsThreshold()); log.warn(message); throw new IllegalArgumentException(message);
@Override public TableMetadata refresh() { refreshFromMetadataLocation(this.location, connectorContext.getConfig().getIcebergRefreshFromMetadataLocationRetryNumber()); return current(); }
/** * Constructor. * * @param connectorContext connector context * @param jdbcTemplate JDBC template * @param sequenceGeneration sequence generator * @param fastServiceMetric fast service metric */ public DirectSqlSavePartition(final ConnectorContext connectorContext, final JdbcTemplate jdbcTemplate, final SequenceGeneration sequenceGeneration, final HiveConnectorFastServiceMetric fastServiceMetric) { this.registry = connectorContext.getRegistry(); this.catalogName = connectorContext.getCatalogName(); this.batchSize = connectorContext.getConfig().getHiveMetastoreBatchSize(); this.jdbcTemplate = jdbcTemplate; this.sequenceGeneration = sequenceGeneration; this.fastServiceMetric = fastServiceMetric; }
/** * {@inheritDoc} */ @Override public boolean partition_name_has_valid_characters(final List<String> partVals, final boolean throwException) throws TException { return requestWrapper("partition_name_has_valid_characters", new Object[]{partVals, throwException}, () -> { Pattern pattern = null; final String partitionPattern = config.getHivePartitionWhitelistPattern(); if (!Strings.isNullOrEmpty(partitionPattern)) { pattern = PATTERNS.getUnchecked(partitionPattern); } if (throwException) { MetaStoreUtils.validatePartitionNameCharacters(partVals, pattern); return true; } else { return MetaStoreUtils.partitionNameHasValidCharacters(partVals, pattern); } }); }
/** * Constructor. * * @param registry registry for spectator * @param config Program configuration */ @Autowired public ThreadServiceManager(final Registry registry, final Config config) { final ExecutorService executorService = newFixedThreadPool( config.getServiceMaxNumberOfThreads(), "metacat-service-pool-%d", 1000 ); this.executor = MoreExecutors.listeningDecorator(executorService); RegistryUtil.registerThreadPool(registry, "metacat-service-pool", (ThreadPoolExecutor) executorService); }
final String message = String.format("Number of partitions queried for table %s exceeded the threshold %d", tableName, connectorContext.getConfig().getMaxPartitionsThreshold()); log.warn(message); throw new IllegalArgumentException(message);
private ConnectorTypeConverter getDefaultConverter() { try { return (ConnectorTypeConverter) Class.forName(this.config.getDefaultTypeConverter()).newInstance(); } catch (Exception e) { throw Throwables.propagate(e); } } }