private void deleteLocalDatasets() { for (final Map.Entry<String, String> entry : datasetFramework.getDatasetNameMapping().entrySet()) { if (keepLocal(entry.getKey())) { continue; } final String localInstanceName = entry.getValue(); final DatasetId instanceId = new DatasetId(workflowRunId.getNamespace(), localInstanceName); LOG.debug("Deleting Workflow local dataset instance: {}", localInstanceName); try { Retries.runWithRetries(() -> datasetFramework.deleteInstance(instanceId), RetryStrategies.fixDelay(Constants.Retry.LOCAL_DATASET_OPERATION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS)); } catch (Exception e) { LOG.warn("Failed to delete the Workflow local dataset instance {}", localInstanceName, e); } } }
private void deleteLocalDatasets() { for (final Map.Entry<String, String> entry : datasetFramework.getDatasetNameMapping().entrySet()) { if (keepLocal(entry.getKey())) { continue; } final String localInstanceName = entry.getValue(); final DatasetId instanceId = new DatasetId(workflowRunId.getNamespace(), localInstanceName); LOG.debug("Deleting Workflow local dataset instance: {}", localInstanceName); try { Retries.runWithRetries(() -> datasetFramework.deleteInstance(instanceId), RetryStrategies.fixDelay(Constants.Retry.LOCAL_DATASET_OPERATION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS)); } catch (Exception e) { LOG.warn("Failed to delete the Workflow local dataset instance {}", localInstanceName, e); } } }
private void createLocalDatasets() throws IOException, DatasetManagementException { final KerberosPrincipalId principalId = ProgramRunners.getApplicationPrincipal(programOptions); for (final Map.Entry<String, String> entry : datasetFramework.getDatasetNameMapping().entrySet()) { final String localInstanceName = entry.getValue(); final DatasetId instanceId = new DatasetId(workflowRunId.getNamespace(), localInstanceName); final DatasetCreationSpec instanceSpec = workflowSpec.getLocalDatasetSpecs().get(entry.getKey()); LOG.debug("Adding Workflow local dataset instance: {}", localInstanceName); try { Retries.callWithRetries(new Retries.Callable<Void, Exception>() { @Override public Void call() throws Exception { DatasetProperties properties = addLocalDatasetProperty(instanceSpec.getProperties(), keepLocal(entry.getKey())); // we have to do this check since addInstance method can only be used when app impersonation is enabled if (principalId != null) { datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties, principalId); } else { datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties); } return null; } }, RetryStrategies.fixDelay(Constants.Retry.LOCAL_DATASET_OPERATION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS)); } catch (IOException | DatasetManagementException e) { throw e; } catch (Exception e) { // this should never happen throw new IllegalStateException(e); } } }
private void createLocalDatasets() throws IOException, DatasetManagementException { final KerberosPrincipalId principalId = ProgramRunners.getApplicationPrincipal(programOptions); for (final Map.Entry<String, String> entry : datasetFramework.getDatasetNameMapping().entrySet()) { final String localInstanceName = entry.getValue(); final DatasetId instanceId = new DatasetId(workflowRunId.getNamespace(), localInstanceName); final DatasetCreationSpec instanceSpec = workflowSpec.getLocalDatasetSpecs().get(entry.getKey()); LOG.debug("Adding Workflow local dataset instance: {}", localInstanceName); try { Retries.callWithRetries(new Retries.Callable<Void, Exception>() { @Override public Void call() throws Exception { DatasetProperties properties = addLocalDatasetProperty(instanceSpec.getProperties(), keepLocal(entry.getKey())); // we have to do this check since addInstance method can only be used when app impersonation is enabled if (principalId != null) { datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties, principalId); } else { datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties); } return null; } }, RetryStrategies.fixDelay(Constants.Retry.LOCAL_DATASET_OPERATION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS)); } catch (IOException | DatasetManagementException e) { throw e; } catch (Exception e) { // this should never happen throw new IllegalStateException(e); } } }