@Override public Path call() throws IOException { Path versionRootPath = PathUtils.combinePaths(SimpleHadoopFilesystemConfigStore.this.physicalStoreRoot.toString(), CONFIG_STORE_NAME, this.version); if (SimpleHadoopFilesystemConfigStore.this.fs.isDirectory(versionRootPath)) { return versionRootPath; } throw new VersionDoesNotExistException(getStoreURI(), this.version, String.format("Cannot find specified version under root %s", versionRootPath)); } }
private static Map<SourceEntity, State> getTableSpecificPropsFromConfigStore( Collection<SourceEntity> tables, State state) { ConfigClient client = ConfigClientCache.getClient(VersionStabilityPolicy.STRONG_LOCAL_STABILITY); String configStoreUri = state.getProp(ConfigurationKeys.CONFIG_MANAGEMENT_STORE_URI); Preconditions.checkNotNull(configStoreUri); Map<SourceEntity, State> result = Maps.newHashMap(); for (SourceEntity table : tables) { try { result.put(table, ConfigUtils.configToState( client.getConfig(PathUtils.combinePaths(configStoreUri, QUERY_BASED_SOURCE, table.getDatasetName()).toUri()))); } catch (VersionDoesNotExistException | ConfigStoreFactoryDoesNotExistsException | ConfigStoreCreationException e) { throw new RuntimeException("Unable to get table config for " + table, e); } } return result; }
/** * Returns the complete path of the audit file. Generate the audit file path with format * * <pre> * |-- <Database> * |-- <Table> * |-- P=<PHASE>.C=<CLUSTER>.E=<EXTRACT_ID>.S=<SNAPSHOT_ID>.D=<DELTA_ID> * |-- *.avro * </pre> * */ public Path getAuditFilePath() { StringBuilder auditFileNameBuilder = new StringBuilder(); auditFileNameBuilder.append("P=").append(auditMetadata.getPhase()).append(FILE_NAME_DELIMITTER).append("C=") .append(auditMetadata.getCluster()).append(FILE_NAME_DELIMITTER).append("E=") .append(auditMetadata.getExtractId()).append(FILE_NAME_DELIMITTER).append("S=") .append(auditMetadata.getSnapshotId()).append(FILE_NAME_DELIMITTER).append("D=") .append(auditMetadata.getDeltaId()); return new Path(auditDirPath, PathUtils.combinePaths(auditMetadata.getTableMetadata().getDatabase(), auditMetadata .getTableMetadata().getTable(), auditFileNameBuilder.toString(), auditMetadata.getPartFileName())); }
@Test(dependsOnMethods = { "testGetCurrentVersion" }) public void testDeploy() throws Exception { Properties props = new Properties(); props.setProperty(ClasspathConfigSource.CONFIG_STORE_CLASSPATH_RESOURCE_NAME_KEY, "_testDeploy"); this._simpleHadoopFilesystemConfigStore.deploy(new FsDeploymentConfig(new ClasspathConfigSource(props), "2.0")); Path versionPath = PathUtils.combinePaths(CONFIG_DIR_NAME, SimpleHadoopFilesystemConfigStore.CONFIG_STORE_NAME, "2.0"); Assert.assertTrue(fs.exists(new Path(versionPath, "dir1"))); Assert.assertTrue(fs.exists(new Path(versionPath, "dir1/f1.conf"))); }
@Test(dataProvider = "retentionTestDataProvider") public void testRetention(final String testName, final String testConfFileName) throws Exception { // Temp path for this test under which test data is generated Path testNameTempPath = new Path(testClassTempPath, testName); RetentionTestDataGenerator dataGenerator = new RetentionTestDataGenerator(testNameTempPath, PathUtils.combinePaths(TEST_PACKAGE_RESOURCE_NAME, testName, SETUP_VALIDATE_CONFIG_CLASSPATH_FILENAME), fs); try { dataGenerator.setup(); RetentionTestHelper.clean(fs, PathUtils.combinePaths(TEST_PACKAGE_RESOURCE_NAME, testName, testConfFileName), testNameTempPath); dataGenerator.validate(); } finally { dataGenerator.cleanup(); } }
RetentionTestHelper.clean(fs, PathUtils.combinePaths(RetentionIntegrationTest.TEST_PACKAGE_RESOURCE_NAME, "testHiveTimeBasedRetention", configFileName), Optional.of(PathUtils.combinePaths(RetentionIntegrationTest.TEST_PACKAGE_RESOURCE_NAME, "testHiveTimeBasedRetention", "jobProps.properties")), testTempPath);
@Override public Path call() throws IOException { Path versionRootPath = PathUtils.combinePaths(SimpleHadoopFilesystemConfigStore.this.physicalStoreRoot.toString(), CONFIG_STORE_NAME, this.version); if (SimpleHadoopFilesystemConfigStore.this.fs.isDirectory(versionRootPath)) { return versionRootPath; } throw new VersionDoesNotExistException(getStoreURI(), this.version, String.format("Cannot find specified version under root %s", versionRootPath)); } }
private static Map<SourceEntity, State> getTableSpecificPropsFromConfigStore( Collection<SourceEntity> tables, State state) { ConfigClient client = ConfigClientCache.getClient(VersionStabilityPolicy.STRONG_LOCAL_STABILITY); String configStoreUri = state.getProp(ConfigurationKeys.CONFIG_MANAGEMENT_STORE_URI); Preconditions.checkNotNull(configStoreUri); Map<SourceEntity, State> result = Maps.newHashMap(); for (SourceEntity table : tables) { try { result.put(table, ConfigUtils.configToState( client.getConfig(PathUtils.combinePaths(configStoreUri, QUERY_BASED_SOURCE, table.getDatasetName()).toUri()))); } catch (VersionDoesNotExistException | ConfigStoreFactoryDoesNotExistsException | ConfigStoreCreationException e) { throw new RuntimeException("Unable to get table config for " + table, e); } } return result; }