public boolean forceUpdate() { return forceUpdate.orElseGet(() -> fallback.forceUpdate()); }
public boolean ignoreAuthzErrors() { return ignoreAuthzErrors.orElseGet(() -> fallback.ignoreAuthzErrors()); }
public Builder toBuilder() { return newBuilder() .setIgnoreAuthzErrors(ignoreAuthzErrors.orElse(null)) .setDeleteUnavailableDatasets(deleteUnavailableDatasets.orElse(null)) .setAutoPromote(autoPromote.orElse(null)) .setForceUpdate(forceUpdate.orElse(null)); }
/** * Gets dataset retrieval options as defined on the source. * * @return dataset retrieval options defined on the source */ DatasetRetrievalOptions getDefaultRetrievalOptions() { return DatasetRetrievalOptions.fromMetadataPolicy(metadataPolicy) .withFallback(DatasetRetrievalOptions.DEFAULT); }
SourceTableDefinition getDatasetWithOptions(NamespaceKey datasetPath, TableInstance instance, boolean ignoreAuthErrors, String user) throws Exception{ final FormatPluginConfig fconfig = optionExtractor.createConfigForTable(instance); return getDatasetWithFormat( datasetPath, null, fconfig, DatasetRetrievalOptions.DEFAULT.toBuilder() .setIgnoreAuthzErrors(ignoreAuthErrors) .build(), user ); }
forced[0] = DatasetRetrievalOptions.class.cast(invocation.getArguments()[2]).forceUpdate(); return null; }).when(sp).getDataset(any(), any(), any()); DatasetRetrievalOptions.DEFAULT.toBuilder() .setForceUpdate(true) .build());
final DatasetRetrievalOptions retrievalOptions = DatasetRetrievalOptions.fromMetadataPolicy(metadataPolicy) .withFallback(DatasetRetrievalOptions.DEFAULT); if (!retrievalOptions.deleteUnavailableDatasets()) { logger.debug("Unavailable dataset '{}' will not be deleted", foundKey); } else {
retrievalOptions.autoPromote()) { for (final FormatMatcher matcher : matchers) { try { if (!retrievalOptions.ignoreAuthzErrors()) { logger.debug(e.getMessage()); throw UserException.permissionError(e)
UpdateStatus refreshDataset(NamespaceKey key, DatasetRetrievalOptions retrievalOptions) { retrievalOptions.withFallback(msp.getDefaultRetrievalOptions()); if (!retrievalOptions.deleteUnavailableDatasets()) { logger.debug("Unavailable dataset '{}' will not be deleted", key); return UpdateStatus.UNCHANGED; if (!retrievalOptions.deleteUnavailableDatasets()) { logger.debug("Unavailable dataset '{}' will not be deleted", key); return UpdateStatus.UNCHANGED;
public DatasetRetrievalOptions build() { return new DatasetRetrievalOptions(ignoreAuthzErrors, deleteUnavailableDatasets, autoPromote, forceUpdate); } }
public boolean autoPromote() { return autoPromote.orElseGet(() -> fallback.autoPromote()); }
public boolean deleteUnavailableDatasets() { return deleteUnavailableDatasets.orElseGet(() -> fallback.deleteUnavailableDatasets()); }
@Override public CheckResult checkReadSignature( ByteString key, DatasetConfig oldConfig, DatasetRetrievalOptions retrievalOptions ) throws Exception { final SourceTableDefinition definition = getDataset( new NamespaceKey(oldConfig.getFullPathList()), oldConfig, retrievalOptions.toBuilder() .setIgnoreAuthzErrors(true) .build()); if(definition == null) { return CheckResult.DELETED; } return new CheckResult() { @Override public UpdateStatus getStatus() { return UpdateStatus.CHANGED; } @Override public SourceTableDefinition getDataset() { return definition; }}; }
DatasetRetrievalOptions.DEFAULT.toBuilder() .setDeleteUnavailableDatasets(false) .build()));
boolean newUpdateKey = false; if (retrievalOptions.forceUpdate() || datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getReadSignature() == null) {
DatasetRetrievalOptions.Builder retrievalOptionsBuilder = DatasetRetrievalOptions.newBuilder(); if (delete != null) { retrievalOptionsBuilder.setDeleteUnavailableDatasets(delete.booleanValue());
@Override public Iterable<SourceTableDefinition> getDatasets(String user, DatasetRetrievalOptions retrievalOptions) { final HiveClient client = getClient(user); final ImmutableList.Builder<SourceTableDefinition> accessors = ImmutableList.builder(); try { for(String dbName : client.getDatabases(retrievalOptions.ignoreAuthzErrors())) { try { for(String table : client.getTableNames(dbName, retrievalOptions.ignoreAuthzErrors())) { DatasetBuilder builder = DatasetBuilder.getDatasetBuilder( client, getStorageUser(user), new NamespaceKey(ImmutableList.of(name, dbName, table)), true, // we got the path from HiveClient so it's safe to assume it's canonized retrievalOptions.ignoreAuthzErrors(), getStatsParams(), hiveConf, null); if(builder != null){ accessors.add(builder); } } } catch(TException e){ logger.warn("User {} is unable to retrieve table names for {}.{}.", user, name, dbName, e); } } } catch(TException e){ logger.warn("User {} is unable to retrieve database names for Hive source named {}.", user, name, e); } return accessors.build(); }
DatasetRetrievalOptions.DEFAULT.toBuilder() .setDeleteUnavailableDatasets(false) .build()));
ProtostuffIOUtil.mergeFrom(key.toByteArray(), fileUpdateKey, FileUpdateKey.getSchema()); if (retrievalOptions.forceUpdate() || fileUpdateKey.getCachedEntitiesList() == null || fileUpdateKey.getCachedEntitiesList().isEmpty()) {
/** * Convert a {@link MetadataPolicy} to {@link DatasetRetrievalOptions}. * * @param policy metadata policy * @return dataset retrieval options */ public static DatasetRetrievalOptions fromMetadataPolicy(MetadataPolicy policy) { Preconditions.checkNotNull(policy.getAutoPromoteDatasets()); Preconditions.checkNotNull(policy.getDeleteUnavailableDatasets()); return newBuilder() .setAutoPromote(policy.getAutoPromoteDatasets()) .setDeleteUnavailableDatasets(policy.getDeleteUnavailableDatasets()) // not an option in policy (or UI) .setForceUpdate(false) // not an option in policy (or UI) .setIgnoreAuthzErrors(false) .build(); } }