/** * Creates a new ConfigurableCleanableDataset configured through gobblin-config-management. The constructor expects * {@link #VERSION_FINDER_CLASS_KEY} and {@link #RETENTION_POLICY_CLASS_KEY} to be available in the * <code>config</code> passed. */ public ConfigurableCleanableDataset(FileSystem fs, Properties jobProps, Path datasetRoot, Config config, Logger log) throws IOException { super(fs, jobProps, config, log); this.datasetRoot = datasetRoot; this.versionFindersAndPolicies = Lists.newArrayList(); if (config.hasPath(DATASET_VERSION_POLICY_ALIAS)) { initWithSelectionPolicy(config.getConfig(DATASET_VERSION_POLICY_ALIAS), jobProps); } else if (config.hasPath(VERSION_FINDER_CLASS_KEY) && config.hasPath(RETENTION_POLICY_CLASS_KEY)) { initWithRetentionPolicy(config, jobProps, RETENTION_POLICY_CLASS_KEY, VERSION_FINDER_CLASS_KEY); } else if (config.hasPath(VERSION_FINDER_CLASS_KEY)) { initWithSelectionPolicy(config.getConfig(RETENTION_CONFIGURATION_KEY), jobProps); } else if (config.hasPath(DATASET_PARTITIONS_LIST_KEY)) { List<? extends Config> versionAndPolicies = config.getConfigList(DATASET_PARTITIONS_LIST_KEY); for (Config versionAndPolicy : versionAndPolicies) { initWithSelectionPolicy(versionAndPolicy, jobProps); } } else { throw new IllegalArgumentException( String.format("Either set version finder at %s and retention policy at %s or set partitions at %s", VERSION_FINDER_CLASS_KEY, RETENTION_POLICY_CLASS_KEY, DATASET_PARTITIONS_LIST_KEY)); } }
/** * Creates a new ConfigurableCleanableDataset configured through gobblin-config-management. The constructor expects * {@link #VERSION_FINDER_CLASS_KEY} and {@link #RETENTION_POLICY_CLASS_KEY} to be available in the * <code>config</code> passed. */ public ConfigurableCleanableDataset(FileSystem fs, Properties jobProps, Path datasetRoot, Config config, Logger log) throws IOException { super(fs, jobProps, config, log); this.datasetRoot = datasetRoot; this.versionFindersAndPolicies = Lists.newArrayList(); if (config.hasPath(DATASET_VERSION_POLICY_ALIAS)) { initWithSelectionPolicy(config.getConfig(DATASET_VERSION_POLICY_ALIAS), jobProps); } else if (config.hasPath(VERSION_FINDER_CLASS_KEY) && config.hasPath(RETENTION_POLICY_CLASS_KEY)) { initWithRetentionPolicy(config, jobProps, RETENTION_POLICY_CLASS_KEY, VERSION_FINDER_CLASS_KEY); } else if (config.hasPath(VERSION_FINDER_CLASS_KEY)) { initWithSelectionPolicy(config.getConfig(RETENTION_CONFIGURATION_KEY), jobProps); } else if (config.hasPath(DATASET_PARTITIONS_LIST_KEY)) { List<? extends Config> versionAndPolicies = config.getConfigList(DATASET_PARTITIONS_LIST_KEY); for (Config versionAndPolicy : versionAndPolicies) { initWithSelectionPolicy(versionAndPolicy, jobProps); } } else { throw new IllegalArgumentException( String.format("Either set version finder at %s and retention policy at %s or set partitions at %s", VERSION_FINDER_CLASS_KEY, RETENTION_POLICY_CLASS_KEY, DATASET_PARTITIONS_LIST_KEY)); } }