@Override public void save(FluoConfiguration fluoConfig) { save(fluoConfig.getAppConfiguration()); }
static void setSerializer(FluoConfiguration fluoConfig, String serializerType) { fluoConfig.getAppConfiguration().setProperty("recipes.serializer", serializerType); }
/** * Call this to configure a KryoFactory type before initializing Fluo. */ public static void setKryoFactory(FluoConfiguration config, String factoryType) { config.getAppConfiguration().setProperty(KRYO_FACTORY_PROP, factoryType); }
/** * Call this to configure a KryoFactory type before initializing Fluo. */ public static void setKryoFactory(FluoConfiguration config, Class<? extends KryoFactory> factoryType) { config.getAppConfiguration().setProperty(KRYO_FACTORY_PROP, factoryType.getName()); } }
/** * This method can be called to register table optimizations before initializing Fluo. This will * register {@link Optimizer} with * {@link TableOptimizations#registerOptimization(SimpleConfiguration, String, Class)}. See the * project level documentation for an example. * * @param fluoConfig The config that will be used to initialize Fluo * @param prefix The prefix used for your Row Hasher. If you have a single instance, could call * {@link RowHasher#getPrefix()}. * @param numTablets Initial number of tablet to create. */ public static void configure(FluoConfiguration fluoConfig, String prefix, int numTablets) { fluoConfig.getAppConfiguration().setProperty(PREFIX + prefix + ".numTablets", numTablets); TableOptimizations.registerOptimization(fluoConfig.getAppConfiguration(), prefix, Optimizer.class); }
/** * Read configuration from zookeeper */ private void readZookeeperConfig() { try (CuratorFramework curator = CuratorUtil.newAppCurator(config)) { curator.start(); accumuloInstance = new String(curator.getData().forPath(ZookeeperPath.CONFIG_ACCUMULO_INSTANCE_NAME), StandardCharsets.UTF_8); accumuloInstanceID = new String(curator.getData().forPath(ZookeeperPath.CONFIG_ACCUMULO_INSTANCE_ID), StandardCharsets.UTF_8); fluoApplicationID = new String(curator.getData().forPath(ZookeeperPath.CONFIG_FLUO_APPLICATION_ID), StandardCharsets.UTF_8); table = new String(curator.getData().forPath(ZookeeperPath.CONFIG_ACCUMULO_TABLE), StandardCharsets.UTF_8); observers = ObserverUtil.load(curator); config = FluoAdminImpl.mergeZookeeperConfig(config); // make sure not to include config passed to env, only want config from zookeeper appConfig = config.getAppConfiguration(); } catch (Exception e) { throw new IllegalStateException(e); } }
/** * Read configuration from zookeeper */ private void readZookeeperConfig() { try (CuratorFramework curator = CuratorUtil.newAppCurator(config)) { curator.start(); accumuloInstance = new String(curator.getData().forPath(ZookeeperPath.CONFIG_ACCUMULO_INSTANCE_NAME), StandardCharsets.UTF_8); accumuloInstanceID = new String(curator.getData().forPath(ZookeeperPath.CONFIG_ACCUMULO_INSTANCE_ID), StandardCharsets.UTF_8); fluoApplicationID = new String(curator.getData().forPath(ZookeeperPath.CONFIG_FLUO_APPLICATION_ID), StandardCharsets.UTF_8); table = new String(curator.getData().forPath(ZookeeperPath.CONFIG_ACCUMULO_TABLE), StandardCharsets.UTF_8); observers = ObserverUtil.load(curator); config = FluoAdminImpl.mergeZookeeperConfig(config); // make sure not to include config passed to env, only want config from zookeeper appConfig = config.getAppConfiguration(); } catch (Exception e) { throw new IllegalStateException(e); } }
@Override public void update(CuratorFramework curator, FluoConfiguration config) throws Exception { String obsProviderClass = config.getObserverProvider(); ObserverProvider observerProvider = newObserverProvider(obsProviderClass); Map<Column, NotificationType> obsCols = new HashMap<>(); BiConsumer<Column, NotificationType> obsColConsumer = (col, nt) -> { Objects.requireNonNull(col, "Observed column must be non-null"); Objects.requireNonNull(nt, "Notification type must be non-null"); Preconditions.checkArgument(!obsCols.containsKey(col), "Duplicate observed column %s", col); obsCols.put(col, nt); }; observerProvider.provideColumns(obsColConsumer, new ObserverProviderContextImpl(config.getAppConfiguration())); Gson gson = new Gson(); String json = gson.toJson(new JsonObservers(obsProviderClass, obsCols)); CuratorUtil.putData(curator, CONFIG_FLUO_OBSERVERS2, json.getBytes(UTF_8), CuratorUtil.NodeExistsPolicy.OVERWRITE); }
/** * Call this method before initializing Fluo. * * @param fluoConfig The configuration that will be used to initialize fluo. * @deprecated since 1.1.0 use {@link #configure(String)} and * {@link #registerObserver(ObserverProvider.Registry, org.apache.fluo.recipes.core.export.function.Exporter)} * instead. */ @Deprecated public static void configure(FluoConfiguration fluoConfig, Options opts) { SimpleConfiguration appConfig = fluoConfig.getAppConfiguration(); opts.save(appConfig); fluoConfig.addObserver( new org.apache.fluo.api.config.ObserverSpecification(ExportObserver.class.getName(), Collections.singletonMap("queueId", opts.fluentCfg.queueId))); }
@Override public void update(CuratorFramework curator, FluoConfiguration config) throws Exception { String obsProviderClass = config.getObserverProvider(); ObserverProvider observerProvider = newObserverProvider(obsProviderClass); Map<Column, NotificationType> obsCols = new HashMap<>(); BiConsumer<Column, NotificationType> obsColConsumer = (col, nt) -> { Objects.requireNonNull(col, "Observed column must be non-null"); Objects.requireNonNull(nt, "Notification type must be non-null"); Preconditions.checkArgument(!obsCols.containsKey(col), "Duplicate observed column %s", col); obsCols.put(col, nt); }; observerProvider.provideColumns(obsColConsumer, new ObserverProviderContextImpl(config.getAppConfiguration())); Gson gson = new Gson(); String json = gson.toJson(new JsonObservers(obsProviderClass, obsCols)); CuratorUtil.putData(curator, CONFIG_FLUO_OBSERVERS2, json.getBytes(UTF_8), CuratorUtil.NodeExistsPolicy.OVERWRITE); }
@Override public void save(FluoConfiguration fluoConf) { SimpleConfiguration appConfig = fluoConf.getAppConfiguration(); // TODO Auto-generated method stub appConfig.setProperty(PREFIX + id + ".instance", instance); appConfig.setProperty(PREFIX + id + ".zookeepers", zookeepers); appConfig.setProperty(PREFIX + id + ".user", user); appConfig.setProperty(PREFIX + id + ".password", password); appConfig.setProperty(PREFIX + id + ".table", table); }
oc.toMap()); try { observer.init(new ObserverContext(config.getAppConfiguration(), oc)); } catch (Exception e) { throw new FluoException("Observer '" + ospec.getClassName() + "' could not be initialized",
oc.toMap()); try { observer.init(new ObserverContext(config.getAppConfiguration(), oc)); } catch (Exception e) { throw new FluoException("Observer '" + ospec.getClassName() + "' could not be initialized",
@Override public void save(FluoConfiguration fluoConfig) { SimpleConfiguration appConfig = fluoConfig.getAppConfiguration(); appConfig.setProperty(PREFIX + cqId + ".buckets", numBuckets + ""); appConfig.setProperty(PREFIX + cqId + ".key", keyType); appConfig.setProperty(PREFIX + cqId + ".val", valueType); if (bufferSize != null) { appConfig.setProperty(PREFIX + cqId + ".bufferSize", bufferSize); } if (bucketsPerTablet != null) { appConfig.setProperty(PREFIX + cqId + ".bucketsPerTablet", bucketsPerTablet); } Bytes dataRangeEnd = Bytes.of(cqId + DATA_RANGE_END); Bytes updateRangeEnd = Bytes.of(cqId + UPDATE_RANGE_END); new TransientRegistry(fluoConfig.getAppConfiguration()).addTransientRange("cfm." + cqId, new RowRange(dataRangeEnd, updateRangeEnd)); TableOptimizations.registerOptimization(appConfig, cqId, CombineQueue.Optimizer.class); }
/** * This method configures a collision free map for use. It must be called before initializing * Fluo. */ public static void configure(FluoConfiguration fluoConfig, Options opts) { org.apache.fluo.recipes.core.combine.CombineQueue.FluentOptions cqopts = CombineQueue.configure(opts.mapId).keyType(opts.keyType).valueType(opts.valueType) .buckets(opts.numBuckets); if (opts.bucketsPerTablet != null) { cqopts.bucketsPerTablet(opts.bucketsPerTablet); } if (opts.bufferSize != null) { cqopts.bufferSize(opts.bufferSize); } cqopts.save(fluoConfig); opts.save(fluoConfig.getAppConfiguration()); fluoConfig.addObserver(new org.apache.fluo.api.config.ObserverSpecification( CollisionFreeMapObserver.class.getName(), ImmutableMap.of("mapId", opts.mapId))); }
public static void initialize(CuratorFramework curator, FluoConfiguration config) { logger.info("Setting up observers using app config: {}", config.getAppConfiguration()); ObserverStore ov1 = new ObserverStoreV1(); ObserverStore ov2 = new ObserverStoreV2(); if (ov1.handles(config) && ov2.handles(config)) { throw new IllegalArgumentException( "Old and new observers configuration present. There can only be one."); } try { if (ov1.handles(config)) { ov2.clear(curator); ov1.update(curator, config); } else if (ov2.handles(config)) { ov1.clear(curator); ov2.update(curator, config); } } catch (Exception e) { throw new FluoException("Failed to update shared configuration in Zookeeper", e); } }
public static void initialize(CuratorFramework curator, FluoConfiguration config) { logger.info("Setting up observers using app config: {}", config.getAppConfiguration()); ObserverStore ov1 = new ObserverStoreV1(); ObserverStore ov2 = new ObserverStoreV2(); if (ov1.handles(config) && ov2.handles(config)) { throw new IllegalArgumentException( "Old and new observers configuration present. There can only be one."); } try { if (ov1.handles(config)) { ov2.clear(curator); ov1.update(curator, config); } else if (ov2.handles(config)) { ov1.clear(curator); ov2.update(curator, config); } } catch (Exception e) { throw new FluoException("Failed to update shared configuration in Zookeeper", e); } }
@Before public void setUpFluo() throws Exception { config = new FluoConfiguration(); config.setApplicationName("mini-test" + testCounter.getAndIncrement()); config.setAccumuloInstance(clientInfo.getInstanceName()); config.setAccumuloUser(USER); config.setAccumuloPassword(PASSWORD); config.setAccumuloZookeepers(clientInfo.getZooKeepers()); config.setInstanceZookeepers(clientInfo.getZooKeepers() + "/fluo"); config.setAccumuloTable(getNextTableName()); config.setWorkerThreads(5); setupObservers(config); config.setMiniStartAccumulo(false); setConfig(config); setAppConfig(config.getAppConfiguration()); config.setTransactionRollbackTime(1, TimeUnit.SECONDS); try (FluoAdmin admin = FluoFactory.newAdmin(config)) { InitializationOptions opts = new InitializationOptions().setClearZookeeper(true).setClearTable(true); admin.initialize(opts); } config.getAppConfiguration().clear(); client = FluoFactory.newClient(config); miniFluo = FluoFactory.newMiniFluo(config); }
@Override public void init(org.apache.fluo.recipes.core.export.Exporter.Context context) throws Exception { SimpleConfiguration sc = context.getExporterConfiguration(); String instanceName = sc.getString("instanceName"); String zookeepers = sc.getString("zookeepers"); String user = sc.getString("user"); String password = sc.getString("password"); String table = sc.getString("table"); FluoConfiguration tmpFc = new FluoConfiguration(); org.apache.fluo.recipes.accumulo.export.function.AccumuloExporter.configure("aecfgid") .instance(instanceName, zookeepers).credentials(user, password).table(table).save(tmpFc); accumuloWriter = new org.apache.fluo.recipes.accumulo.export.function.AccumuloExporter<K, V>( "aecfgid", tmpFc.getAppConfiguration(), this::translate); }
@Test public void testBasic() { SimpleConfiguration uc = client.getAppConfiguration(); Assert.assertEquals(50000, uc.getInt("myapp.sizeLimit")); uc.setProperty("myapp.sizeLimit", 3); uc = client.getAppConfiguration(); Assert.assertEquals(50000, uc.getInt("myapp.sizeLimit")); // update shared config SimpleConfiguration appConfig = config.getAppConfiguration(); appConfig.clear(); appConfig.setProperty("myapp.sizeLimit", 40000); appConfig.setProperty("myapp.timeLimit", 30000); try (FluoAdmin admin = FluoFactory.newAdmin(config)) { miniFluo.close(); admin.updateSharedConfig(); } // set app config that differs from what was just put in zk appConfig.setProperty("myapp.sizeLimit", 6); appConfig.setProperty("myapp.timeLimit", 7); try (FluoClient client2 = FluoFactory.newClient(config)) { uc = client2.getAppConfiguration(); Assert.assertEquals(40000, uc.getInt("myapp.sizeLimit")); Assert.assertEquals(30000, uc.getInt("myapp.timeLimit")); } }