@Override public void delete() throws IOException { this.stateStore.delete(getStoreName(), getTableName()); } }
@Override public boolean create(String storeName, String tableName) throws IOException { if (exists(storeName, tableName)) { throw new IOException(String.format("State already exists for storeName %s tableName %s", storeName, tableName)); } return true; }
@Override public void putAll(String storeName, String tableName, Collection<T> states) throws IOException { try (Connection connection = dataSource.getConnection(); PreparedStatement insertStatement = connection.prepareStatement(UPSERT_JOB_STATE_SQL); ByteArrayOutputStream byteArrayOs = new ByteArrayOutputStream(); OutputStream os = compressedValues ? new GZIPOutputStream(byteArrayOs) : byteArrayOs; DataOutputStream dataOutput = new DataOutputStream(os)) { int index = 0; insertStatement.setString(++index, storeName); insertStatement.setString(++index, tableName); for (T state : states) { addStateToDataOutputStream(dataOutput, state); } dataOutput.close(); insertStatement.setBlob(++index, new ByteArrayInputStream(byteArrayOs.toByteArray())); insertStatement.executeUpdate(); connection.commit(); } catch (SQLException e) { throw new IOException("Failure storing state to store " + storeName + " table " + tableName, e); } }
@BeforeClass public void setUp() throws Exception { this.testMetastoreDatabase = TestMetastoreDatabaseFactory.get(); String jdbcUrl = this.testMetastoreDatabase.getJdbcUrl(); ConfigBuilder configBuilder = ConfigBuilder.create(); BasicDataSource mySqlDs = new BasicDataSource(); mySqlDs.setDriverClassName(ConfigurationKeys.DEFAULT_STATE_STORE_DB_JDBC_DRIVER); mySqlDs.setDefaultAutoCommit(false); mySqlDs.setUrl(jdbcUrl); mySqlDs.setUsername(TEST_USER); mySqlDs.setPassword(TEST_PASSWORD); this.dbJobStateStore = new MysqlStateStore<>(mySqlDs, TEST_STATE_STORE, false, JobState.class); configBuilder.addPrimitive("selection.timeBased.lookbackTime", "10m"); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_TYPE_KEY, "mysql"); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, TEST_STATE_STORE); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_URL_KEY, jdbcUrl); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_USER_KEY, TEST_USER); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, TEST_PASSWORD); ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class); DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass("mysql").newInstance(); this.config = configBuilder.build(); this.dbDatasetStateStore = stateStoreFactory.createStateStore(configBuilder.build()); // clear data that may have been left behind by a prior test run this.dbJobStateStore.delete(TEST_JOB_NAME1); this.dbDatasetStateStore.delete(TEST_JOB_NAME1); this.dbJobStateStore.delete(TEST_JOB_NAME2); this.dbDatasetStateStore.delete(TEST_JOB_NAME2); }
this.dbJobStateStore.setTestTimestamp(System.currentTimeMillis()/1000 - (60 * 20)); this.dbJobStateStore.put(TEST_JOB_NAME1, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); this.dbJobStateStore.setTestTimestamp(0); this.dbJobStateStore.put(TEST_JOB_NAME2, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); List<DatasetStoreDataset> datasets = datasetFinder.findDatasets(); Assert.assertTrue(this.dbJobStateStore.exists(TEST_JOB_NAME1, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX)); Assert.assertTrue(this.dbJobStateStore.exists(TEST_JOB_NAME2, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX)); Assert.assertFalse(this.dbJobStateStore.exists(TEST_JOB_NAME1, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX)); Assert.assertTrue(this.dbJobStateStore.exists(TEST_JOB_NAME2, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX));
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { String stateStoreTableName = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { BasicDataSource basicDataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); return new MysqlStateStore(basicDataSource, stateStoreTableName, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlStateStore with factory", e); } } }
@Override public List<T> getAll(String storeName, String tableName) throws IOException { return getAll(storeName, tableName, false); }
@Override public List<MysqlDatasetStateStoreEntryManager> getMetadataForTables(StateStorePredicate predicate) throws IOException { // get the metadata from the parent class and convert List<MysqlStateStoreEntryManager> entryManagers = (List<MysqlStateStoreEntryManager>) super.getMetadataForTables(predicate); return entryManagers.stream().map(entry -> new MysqlDatasetStateStoreEntryManager(entry.getStoreName(), entry.getTableName(), entry.getTimestamp(), this)).collect(Collectors.toList()); } }
@Override public SharedResourceFactoryResponse<BasicDataSource> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, MysqlDataSourceKey> config) throws NotConfiguredException { MysqlDataSourceKey key = config.getKey(); Config configuration = key.getConfig(); BasicDataSource dataSource = MysqlStateStore.newDataSource(configuration); return new ResourceInstance<>(dataSource); }
/** * Get a {@link BasicDataSource} based on the config * @param config configuration * @param broker broker * @return a {@link BasicDataSource} * @throws IOException */ public static <S extends ScopeType<S>> BasicDataSource get(Config config, SharedResourcesBroker<S> broker) throws IOException { try { return broker.getSharedResource(new MysqlDataSourceFactory<S>(), new MysqlDataSourceKey(MysqlStateStore.getDataSourceId(config), config)); } catch (NotConfiguredException nce) { throw new IOException(nce); } }
@Override public T readState() throws IOException { return this.stateStore.get(getStoreName(), getTableName(), ""); }
@Override public void put(String storeName, String tableName, T state) throws IOException { putAll(storeName, tableName, Collections.singleton(state)); }
@BeforeClass public void setUp() throws Exception { testMetastoreDatabase = TestMetastoreDatabaseFactory.get(); String jdbcUrl = testMetastoreDatabase.getJdbcUrl(); ConfigBuilder configBuilder = ConfigBuilder.create(); BasicDataSource mySqlDs = new BasicDataSource(); mySqlDs.setDriverClassName(ConfigurationKeys.DEFAULT_STATE_STORE_DB_JDBC_DRIVER); mySqlDs.setDefaultAutoCommit(false); mySqlDs.setUrl(jdbcUrl); mySqlDs.setUsername(TEST_USER); mySqlDs.setPassword(TEST_PASSWORD); dbJobStateStore = new MysqlStateStore<>(mySqlDs, TEST_STATE_STORE, false, JobState.class); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_URL_KEY, jdbcUrl); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_USER_KEY, TEST_USER); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, TEST_PASSWORD); ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class); DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass("mysql").newInstance(); dbDatasetStateStore = stateStoreFactory.createStateStore(configBuilder.build()); // clear data that may have been left behind by a prior test run dbJobStateStore.delete(TEST_JOB_NAME); dbDatasetStateStore.delete(TEST_JOB_NAME); dbJobStateStore.delete(TEST_JOB_NAME2); dbDatasetStateStore.delete(TEST_JOB_NAME2); }
@Override public List<T> getAll(String storeName) throws IOException { return getAll(storeName, "%", true); }
@Override public List<MysqlDatasetStateStoreEntryManager> getMetadataForTables(StateStorePredicate predicate) throws IOException { // get the metadata from the parent class and convert List<MysqlStateStoreEntryManager> entryManagers = (List<MysqlStateStoreEntryManager>) super.getMetadataForTables(predicate); return entryManagers.stream().map(entry -> new MysqlDatasetStateStoreEntryManager(entry.getStoreName(), entry.getTableName(), entry.getTimestamp(), this)).collect(Collectors.toList()); } }
@Override public SharedResourceFactoryResponse<BasicDataSource> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, MysqlDataSourceKey> config) throws NotConfiguredException { MysqlDataSourceKey key = config.getKey(); Config configuration = key.getConfig(); BasicDataSource dataSource = MysqlStateStore.newDataSource(configuration); return new ResourceInstance<>(dataSource); }
/** * Get a {@link BasicDataSource} based on the config * @param config configuration * @param broker broker * @return a {@link BasicDataSource} * @throws IOException */ public static <S extends ScopeType<S>> BasicDataSource get(Config config, SharedResourcesBroker<S> broker) throws IOException { try { return broker.getSharedResource(new MysqlDataSourceFactory<S>(), new MysqlDataSourceKey(MysqlStateStore.getDataSourceId(config), config)); } catch (NotConfiguredException nce) { throw new IOException(nce); } }
@Override public T readState() throws IOException { return this.stateStore.get(getStoreName(), getTableName(), ""); }
@Override public void put(String storeName, String tableName, T state) throws IOException { putAll(storeName, tableName, Collections.singleton(state)); }
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { String stateStoreTableName = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { BasicDataSource basicDataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); return new MysqlStateStore(basicDataSource, stateStoreTableName, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlStateStore with factory", e); } } }