/** * Sets up the configuration and prints the arguments. */ public void setUpConfig() { log.info("Setting " + driverName + " config"); // Setup config if (isMock) { configMap.put(MRUtils.AC_MOCK_PROP, Boolean.TRUE.toString()); } configMap.put(MRUtils.AC_INSTANCE_PROP, instanceName); configMap.put(MRUtils.AC_USERNAME_PROP, user); configMap.put(MRUtils.AC_PWD_PROP, userpwd); configMap.put(MRUtils.TABLE_PREFIX_PROPERTY, tablePrefix); configMap.put(MRUtils.AC_AUTH_PROP, auth); configMap.put(MRUtils.AC_ZK_PROP, zooKeepers != null ? zooKeepers : "localhost"); log.info(driverName + " config properties"); config.setTablePrefix(tablePrefix); for (final Entry<String, String> entry : configMap.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue(); final String argument = ToolConfigUtils.makeArgument(isParent ? key : key + MergeTool.CHILD_SUFFIX, value); if (!key.equals(MRUtils.AC_PWD_PROP)) { log.info(argument); } config.set(key, value); } MergeTool.setDuplicateKeys(config); }
/** * Sets up the configuration and prints the arguments. */ public void setUpConfig() { log.info("Setting " + driverName + " config"); // Setup config if (isMock) { configMap.put(MRUtils.AC_MOCK_PROP, Boolean.TRUE.toString()); } configMap.put(MRUtils.AC_INSTANCE_PROP, instanceName); configMap.put(MRUtils.AC_USERNAME_PROP, user); configMap.put(MRUtils.AC_PWD_PROP, userpwd); configMap.put(MRUtils.TABLE_PREFIX_PROPERTY, tablePrefix); configMap.put(MRUtils.AC_AUTH_PROP, auth); configMap.put(MRUtils.AC_ZK_PROP, zooKeepers != null ? zooKeepers : "localhost"); configMap.put(AccumuloExportConstants.ACCUMULO_INSTANCE_TYPE_PROP, instanceType.toString()); log.info(driverName + " config properties"); config.setTablePrefix(tablePrefix); for (final Entry<String, String> entry : configMap.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue(); final String argument = "-D" + key + "=" + value; if (!key.equals(MRUtils.AC_PWD_PROP)) { log.info(argument); } config.set(key, value); } AccumuloExportConstants.setDuplicateKeys(config); }
/** * Sets up the configuration and prints the arguments. */ public void setUpConfig() { log.info("Setting " + driverName + " config"); // Setup config if (isMock) { configMap.put(MRUtils.AC_MOCK_PROP, Boolean.TRUE.toString()); } configMap.put(MRUtils.AC_INSTANCE_PROP, instanceName); configMap.put(MRUtils.AC_USERNAME_PROP, user); configMap.put(MRUtils.AC_PWD_PROP, userpwd); configMap.put(MRUtils.TABLE_PREFIX_PROPERTY, tablePrefix); configMap.put(MRUtils.AC_AUTH_PROP, auth); configMap.put(MRUtils.AC_ZK_PROP, zooKeepers != null ? zooKeepers : "localhost"); configMap.put(AccumuloExportConstants.ACCUMULO_INSTANCE_TYPE_PROP, instanceType.toString()); log.info(driverName + " config properties"); config.setTablePrefix(tablePrefix); for (final Entry<String, String> entry : configMap.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue(); final String argument = "-D" + key + "=" + value; if (!key.equals(MRUtils.AC_PWD_PROP)) { log.info(argument); } config.set(key, value); } AccumuloExportConstants.setDuplicateKeys(config); }
private static AccumuloRyaDAO getRyaIndexer(final Configuration conf) throws IOException { try { if (!conf.getBoolean(ENABLE_CORE, true)) { return null; } final AccumuloRyaDAO ryaIndexer = new AccumuloRyaDAO(); final Connector conn = ConfigUtils.getConnector(conf); ryaIndexer.setConnector(conn); final AccumuloRdfConfiguration ryaConf = new AccumuloRdfConfiguration(); final String tablePrefix = conf.get(OUTPUT_PREFIX_PROPERTY, null); if (tablePrefix != null) { ryaConf.setTablePrefix(tablePrefix); } ryaConf.setDisplayQueryPlan(false); ryaIndexer.setConf(ryaConf); ryaIndexer.init(); return ryaIndexer; } catch (final AccumuloException e) { logger.error("Cannot create RyaIndexer", e); throw new IOException(e); } catch (final AccumuloSecurityException e) { logger.error("Cannot create RyaIndexer", e); throw new IOException(e); } catch (final RyaDAOException e) { logger.error("Cannot create RyaIndexer", e); throw new IOException(e); } }
private static AccumuloRyaDAO getRyaIndexer(final Configuration conf) throws IOException { try { if (!conf.getBoolean(ENABLE_CORE, true)) { return null; } final AccumuloRyaDAO ryaIndexer = new AccumuloRyaDAO(); final Connector conn = ConfigUtils.getConnector(conf); ryaIndexer.setConnector(conn); final AccumuloRdfConfiguration ryaConf = new AccumuloRdfConfiguration(); final String tablePrefix = conf.get(OUTPUT_PREFIX_PROPERTY, null); if (tablePrefix != null) { ryaConf.setTablePrefix(tablePrefix); } ryaConf.setDisplayQueryPlan(false); ryaIndexer.setConf(ryaConf); ryaIndexer.init(); return ryaIndexer; } catch (final AccumuloException e) { logger.error("Cannot create RyaIndexer", e); throw new IOException(e); } catch (final AccumuloSecurityException e) { logger.error("Cannot create RyaIndexer", e); throw new IOException(e); } catch (final RyaDAOException e) { logger.error("Cannot create RyaIndexer", e); throw new IOException(e); } }
/** * * @param ryaInstanceName - The Rya instance to connect to. * @return Constructs a new {@link AccumuloRdfConfiguration} object with values from this object. */ public AccumuloRdfConfiguration buildAccumuloRdfConfiguration(final String ryaInstanceName) { // Note, we don't use the AccumuloRdfConfigurationBuilder here because it explicitly sets // authorizations and visibilities to an empty string if they are not set on the builder. // If they are null in the AccumuloRdfConfiguration object, then Accumulo uses the values stored in accumulo for the user. final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setTablePrefix(ryaInstanceName); conf.setAccumuloZookeepers(zookeepers); conf.setAccumuloInstance(instanceName); conf.setAccumuloUser(username); conf.setAccumuloPassword(new String(userpass)); return conf; } }
/** * * @param ryaInstanceName - The Rya instance to connect to. * @return Constructs a new {@link AccumuloRdfConfiguration} object with values from this object. */ public AccumuloRdfConfiguration buildAccumuloRdfConfiguration(final String ryaInstanceName) { // Note, we don't use the AccumuloRdfConfigurationBuilder here because it explicitly sets // authorizations and visibilities to an empty string if they are not set on the builder. // If they are null in the AccumuloRdfConfiguration object, then Accumulo uses the values stored in accumulo for the user. final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setTablePrefix(ryaInstanceName); conf.setAccumuloZookeepers(zookeepers); conf.setAccumuloInstance(instanceName); conf.setAccumuloUser(username); conf.setAccumuloPassword(new String(userpass)); return conf; } }
/** * Creates the child table if it doesn't already exist. * @param childTableName the name of the child table. * @throws IOException */ public void createTableIfNeeded(final String childTableName) throws IOException { try { final Configuration childConfig = MergeToolMapper.getChildConfig(conf); final AccumuloRdfConfiguration childAccumuloRdfConfiguration = new AccumuloRdfConfiguration(childConfig); childAccumuloRdfConfiguration.setTablePrefix(childTablePrefix); final Connector childConnector = AccumuloRyaUtils.setupConnector(childAccumuloRdfConfiguration); if (!childConnector.tableOperations().exists(childTableName)) { log.info("Creating table: " + childTableName); childConnector.tableOperations().create(childTableName); log.info("Created table: " + childTableName); log.info("Granting authorizations to table: " + childTableName); childConnector.securityOperations().grantTablePermission(childUserName, childTableName, TablePermission.WRITE); log.info("Granted authorizations to table: " + childTableName); } } catch (TableExistsException | AccumuloException | AccumuloSecurityException e) { throw new IOException(e); } }
/** * Imports the child files that hold the table data into the parent instance as a temporary table. * @param childTableName the name of the child table to import into a temporary parent table. * @throws Exception */ public void importChildFilesToTempParentTable(final String childTableName) throws Exception { // Create a temporary table in the parent instance to import the child files to. Then run the merge process on the parent table and temp child table. final String tempChildTable = childTableName + TEMP_SUFFIX; createTempTableIfNeeded(tempChildTable); final AccumuloRdfConfiguration parentAccumuloRdfConfiguration = new AccumuloRdfConfiguration(conf); parentAccumuloRdfConfiguration.setTablePrefix(childTablePrefix); final Connector parentConnector = AccumuloRyaUtils.setupConnector(parentAccumuloRdfConfiguration); final TableOperations parentTableOperations = parentConnector.tableOperations(); final Path localWorkDir = CopyTool.getPath(localMergeFileImportDir, childTableName); final Path hdfsBaseWorkDir = CopyTool.getPath(baseImportDir, childTableName); CopyTool.copyLocalToHdfs(localWorkDir, hdfsBaseWorkDir, conf); final Path files = CopyTool.getPath(hdfsBaseWorkDir.toString(), "files"); final Path failures = CopyTool.getPath(hdfsBaseWorkDir.toString(), "failures"); final FileSystem fs = FileSystem.get(conf); // With HDFS permissions on, we need to make sure the Accumulo user can read/move the files fs.setPermission(hdfsBaseWorkDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); if (fs.exists(failures)) { fs.delete(failures, true); } fs.mkdirs(failures); parentTableOperations.importDirectory(tempChildTable, files.toString(), failures.toString(), false); AccumuloRyaUtils.printTablePretty(tempChildTable, conf); }
/** * Get a Repository from the configuration variables */ static RyaSailRepository getRepository(Configuration conf) throws AccumuloException, AccumuloSecurityException { boolean mock = conf.getBoolean(MRUtils.AC_MOCK_PROP, false); String instance = conf.get(MRUtils.AC_INSTANCE_PROP, "instance"); String username = conf.get(MRUtils.AC_USERNAME_PROP, "root"); String password = conf.get(MRUtils.AC_PWD_PROP, "root"); Instance accumulo; if (mock) { accumulo = new MockInstance(instance); } else { String zookeepers = conf.get(MRUtils.AC_ZK_PROP, "zoo"); accumulo = new ZooKeeperInstance(instance, zookeepers); } Connector connector = accumulo.getConnector(username, new PasswordToken(password)); AccumuloRdfConfiguration aconf = new AccumuloRdfConfiguration(conf); aconf.setTablePrefix(conf.get(MRUtils.TABLE_PREFIX_PROPERTY, RdfCloudTripleStoreConstants.TBL_PRFX_DEF)); AccumuloRyaDAO dao = new AccumuloRyaDAO(); dao.setConnector(connector); dao.setConf(aconf); RdfCloudTripleStore store = new RdfCloudTripleStore(); store.setRyaDAO(dao); return new RyaSailRepository(store); }
private static RyaSailRepository makeRyaRepository(final PcjAdminClientProperties clientProps, final Connector accumulo) throws RepositoryException { checkNotNull(clientProps); checkNotNull(accumulo); // Setup Rya configuration values. final AccumuloRdfConfiguration ryaConf = new AccumuloRdfConfiguration(); ryaConf.setTablePrefix( clientProps.getRyaTablePrefix() ); // Connect to the Rya repo. final AccumuloRyaDAO accumuloRyaDao = new AccumuloRyaDAO(); accumuloRyaDao.setConnector(accumulo); accumuloRyaDao.setConf(ryaConf); final RdfCloudTripleStore ryaStore = new RdfCloudTripleStore(); ryaStore.setRyaDAO(accumuloRyaDao); final RyaSailRepository ryaRepo = new RyaSailRepository(ryaStore); ryaRepo.initialize(); return ryaRepo; }
private static RyaSailRepository makeRyaRepository(final PcjAdminClientProperties clientProps, final Connector accumulo) throws RepositoryException { checkNotNull(clientProps); checkNotNull(accumulo); // Setup Rya configuration values. final AccumuloRdfConfiguration ryaConf = new AccumuloRdfConfiguration(); ryaConf.setTablePrefix( clientProps.getRyaTablePrefix() ); // Connect to the Rya repo. final AccumuloRyaDAO accumuloRyaDao = new AccumuloRyaDAO(); accumuloRyaDao.setConnector(accumulo); accumuloRyaDao.setConf(ryaConf); final RdfCloudTripleStore ryaStore = new RdfCloudTripleStore(); ryaStore.setRyaDAO(accumuloRyaDao); final RyaSailRepository ryaRepo = new RyaSailRepository(ryaStore); ryaRepo.initialize(); return ryaRepo; }
private Sail connectToRya(final String ryaInstanceName) throws RyaClientException { try { final AccumuloConnectionDetails connectionDetails = super.getAccumuloConnectionDetails(); final AccumuloRdfConfiguration ryaConf = new AccumuloRdfConfiguration(); ryaConf.setTablePrefix(ryaInstanceName); ryaConf.set(ConfigUtils.CLOUDBASE_USER, connectionDetails.getUsername()); ryaConf.set(ConfigUtils.CLOUDBASE_PASSWORD, new String(connectionDetails.getUserPass())); ryaConf.set(ConfigUtils.CLOUDBASE_ZOOKEEPERS, connectionDetails.getZookeepers()); ryaConf.set(ConfigUtils.CLOUDBASE_INSTANCE, connectionDetails.getInstanceName()); // Turn PCJs off so that we will only scan the core Rya tables while building the PCJ results. ryaConf.set(ConfigUtils.USE_PCJ, "false"); return RyaSailFactory.getInstance(ryaConf); } catch (SailException | AccumuloException | AccumuloSecurityException | RyaDAOException | InferenceEngineException e) { throw new RyaClientException("Could not connect to the Rya instance named '" + ryaInstanceName + "'.", e); } }
private Sail connectToRya(final String ryaInstanceName) throws RyaClientException { try { final AccumuloConnectionDetails connectionDetails = super.getAccumuloConnectionDetails(); final AccumuloRdfConfiguration ryaConf = new AccumuloRdfConfiguration(); ryaConf.setTablePrefix(ryaInstanceName); ryaConf.set(ConfigUtils.CLOUDBASE_USER, connectionDetails.getUsername()); ryaConf.set(ConfigUtils.CLOUDBASE_PASSWORD, new String(connectionDetails.getUserPass())); ryaConf.set(ConfigUtils.CLOUDBASE_ZOOKEEPERS, connectionDetails.getZookeepers()); ryaConf.set(ConfigUtils.CLOUDBASE_INSTANCE, connectionDetails.getInstanceName()); // Turn PCJs off so that we will only scan the core Rya tables while building the PCJ results. ryaConf.set(ConfigUtils.USE_PCJ, "false"); return RyaSailFactory.getInstance(ryaConf); } catch (SailException | AccumuloException | AccumuloSecurityException | RyaDAOException | InferenceEngineException e) { throw new RyaClientException("Could not connect to the Rya instance named '" + ryaInstanceName + "'.", e); } }
@Override protected Sail makeSail(final Map<String, String> taskConfig) throws ConnectException { requireNonNull(taskConfig); // Parse the configuration object. final AccumuloRyaSinkConfig config = new AccumuloRyaSinkConfig(taskConfig); // Move the configuration into a Rya Configuration object. final AccumuloRdfConfiguration ryaConfig = new AccumuloRdfConfiguration(); ryaConfig.setTablePrefix( config.getRyaInstanceName() ); ryaConfig.setAccumuloZookeepers( config.getZookeepers() ); ryaConfig.setAccumuloInstance( config.getClusterName() ); ryaConfig.setAccumuloUser( config.getUsername() ); ryaConfig.setAccumuloPassword( config.getPassword() ); ryaConfig.setFlush(false); // Create the Sail object. try { return RyaSailFactory.getInstance(ryaConfig); } catch (SailException | AccumuloException | AccumuloSecurityException | RyaDAOException | InferenceEngineException e) { throw new ConnectException("Could not connect to the Rya Instance named " + config.getRyaInstanceName(), e); } } }
protected AccumuloRdfConfiguration makeConfig(final String instanceName, final String zookeepers) { final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setTablePrefix(RYA_INSTANCE_NAME); // Accumulo connection information. conf.setAccumuloUser(AccumuloExportITBase.ACCUMULO_USER); conf.setAccumuloPassword(AccumuloExportITBase.ACCUMULO_PASSWORD); conf.setAccumuloInstance(super.getAccumuloConnector().getInstance().getInstanceName()); conf.setAccumuloZookeepers(super.getAccumuloConnector().getInstance().getZooKeepers()); conf.setAuths(""); // PCJ configuration information. conf.set(ConfigUtils.USE_PCJ, "true"); conf.set(ConfigUtils.USE_PCJ_UPDATER_INDEX, "true"); conf.set(ConfigUtils.FLUO_APP_NAME, super.getFluoConfiguration().getApplicationName()); conf.set(ConfigUtils.PCJ_STORAGE_TYPE, PrecomputedJoinIndexerConfig.PrecomputedJoinStorageType.ACCUMULO.toString()); conf.set(ConfigUtils.PCJ_UPDATER_TYPE, PrecomputedJoinIndexerConfig.PrecomputedJoinUpdaterType.FLUO.toString()); conf.setDisplayQueryPlan(true); return conf; }
private AccumuloRdfConfiguration makeConfig() { final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setTablePrefix(getRyaInstanceName()); // Accumulo connection information. conf.set(ConfigUtils.CLOUDBASE_USER, getUsername()); conf.set(ConfigUtils.CLOUDBASE_PASSWORD, getPassword()); conf.set(ConfigUtils.CLOUDBASE_INSTANCE, super.getMiniAccumuloCluster().getInstanceName()); conf.set(ConfigUtils.CLOUDBASE_ZOOKEEPERS, super.getMiniAccumuloCluster().getZooKeepers()); conf.set(RdfCloudTripleStoreConfiguration.CONF_QUERY_AUTH, "U,V,W"); return conf; }
protected AccumuloRdfConfiguration makeConfig(final String instanceName, final String zookeepers) { final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setTablePrefix(RYA_INSTANCE_NAME); // Accumulo connection information. conf.setAccumuloUser(AccumuloExportITBase.ACCUMULO_USER); conf.setAccumuloPassword(AccumuloExportITBase.ACCUMULO_PASSWORD); conf.setAccumuloInstance(super.getAccumuloConnector().getInstance().getInstanceName()); conf.setAccumuloZookeepers(super.getAccumuloConnector().getInstance().getZooKeepers()); conf.setAuths(""); // PCJ configuration information. conf.set(ConfigUtils.USE_PCJ, "true"); conf.set(ConfigUtils.USE_PCJ_UPDATER_INDEX, "true"); conf.set(ConfigUtils.FLUO_APP_NAME, super.getFluoConfiguration().getApplicationName()); conf.set(ConfigUtils.PCJ_STORAGE_TYPE, PrecomputedJoinIndexerConfig.PrecomputedJoinStorageType.ACCUMULO.toString()); conf.set(ConfigUtils.PCJ_UPDATER_TYPE, PrecomputedJoinIndexerConfig.PrecomputedJoinUpdaterType.FLUO.toString()); conf.setDisplayQueryPlan(true); return conf; }
static void setUpRya() throws AccumuloException, AccumuloSecurityException, RyaDAOException { MockInstance mock = new MockInstance(INSTANCE_NAME); Connector conn = mock.getConnector(USERNAME, new PasswordToken(USERP)); AccumuloRyaDAO dao = new AccumuloRyaDAO(); dao.setConnector(conn); AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setTablePrefix(PREFIX); dao.setConf(conf); dao.init(); String ns = "http://example.com/"; dao.add(new RyaStatement(new RyaURI(ns+"s1"), new RyaURI(ns+"p1"), new RyaURI(ns+"o1"))); dao.add(new RyaStatement(new RyaURI(ns+"s1"), new RyaURI(ns+"p2"), new RyaURI(ns+"o2"))); dao.add(new RyaStatement(new RyaURI(ns+"s2"), new RyaURI(ns+"p1"), new RyaURI(ns+"o3"), new RyaURI(ns+"g1"))); dao.add(new RyaStatement(new RyaURI(ns+"s3"), new RyaURI(ns+"p3"), new RyaURI(ns+"o3"), new RyaURI(ns+"g2"))); dao.destroy(); }
static void setUpRya() throws AccumuloException, AccumuloSecurityException, RyaDAOException { MockInstance mock = new MockInstance(INSTANCE_NAME); Connector conn = mock.getConnector(USERNAME, new PasswordToken(USERP)); AccumuloRyaDAO dao = new AccumuloRyaDAO(); dao.setConnector(conn); AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setTablePrefix(PREFIX); dao.setConf(conf); dao.init(); String ns = "http://example.com/"; dao.add(new RyaStatement(new RyaIRI(ns+"s1"), new RyaIRI(ns+"p1"), new RyaIRI(ns+"o1"))); dao.add(new RyaStatement(new RyaIRI(ns+"s1"), new RyaIRI(ns+"p2"), new RyaIRI(ns+"o2"))); dao.add(new RyaStatement(new RyaIRI(ns+"s2"), new RyaIRI(ns+"p1"), new RyaIRI(ns+"o3"), new RyaIRI(ns+"g1"))); dao.add(new RyaStatement(new RyaIRI(ns+"s3"), new RyaIRI(ns+"p3"), new RyaIRI(ns+"o3"), new RyaIRI(ns+"g2"))); dao.destroy(); }