@Override public MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads) { return wrapped.createMultiTableBatchWriter(maxMemory, maxLatency, maxWriteThreads); }
protected AccumuloRecordWriter(JobConf job) throws AccumuloException, AccumuloSecurityException, IOException { this.isStringEncoded = AccumuloIndexedOutputFormat.getStringEncoding(job).booleanValue(); this.simulate = AccumuloIndexedOutputFormat.getSimulationMode(job).booleanValue(); this.createTables = AccumuloIndexedOutputFormat.canCreateTables(job).booleanValue(); if (this.simulate) { LOG.info("Simulating output only. No writes to tables will occur"); } this.bws = new HashMap(); String tname = AccumuloIndexedOutputFormat.getDefaultTableName(job); this.defaultTableName = tname == null ? null : new Text(tname); String iname = AccumuloIndexedOutputFormat.getIndexTableName(job); if (iname != null) { LOG.info("Index Table = {}", iname); this.indexTableName = new Text(iname); this.indexDef = createIndexDefinition(job, tname, iname); } if (!this.simulate) { this.conn = AccumuloIndexedOutputFormat.getInstance(job) .getConnector(AccumuloIndexedOutputFormat.getPrincipal(job), AccumuloIndexedOutputFormat.getAuthenticationToken(job)); this.mtbw = this.conn.createMultiTableBatchWriter( AccumuloIndexedOutputFormat.getBatchWriterOptions(job)); } }
/** * Gets a multitable batch writer. The same object is reused after the first call unless it is * reset. * * @return multitable batch writer * @throws NumberFormatException * if any of the numeric batch writer configuration properties cannot be parsed * @throws NumberFormatException * if any configuration property cannot be parsed */ public MultiTableBatchWriter getMultiTableBatchWriter() throws AccumuloException, AccumuloSecurityException { if (mtbw == null) { long maxMem = Long.parseLong(p.getProperty(KEY_MAX_MEM)); long maxLatency = Long.parseLong(p.getProperty(KEY_MAX_LATENCY)); int numThreads = Integer.parseInt(p.getProperty(KEY_NUM_THREADS)); mtbw = getConnector().createMultiTableBatchWriter(new BatchWriterConfig().setMaxMemory(maxMem) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(numThreads)); } return mtbw; }
public static MultiTableBatchWriter createMultitableBatchWriter(final Configuration conf) throws AccumuloException, AccumuloSecurityException { final Long DEFAULT_MAX_MEMORY = getWriterMaxMemory(conf); final Long DEFAULT_MAX_LATENCY = getWriterMaxLatency(conf); final Integer DEFAULT_MAX_WRITE_THREADS = getWriterMaxWriteThreads(conf); final Connector connector = ConfigUtils.getConnector(conf); return connector.createMultiTableBatchWriter(DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS); }
public static MultiTableBatchWriter createMultitableBatchWriter(final Configuration conf) throws AccumuloException, AccumuloSecurityException { final Long DEFAULT_MAX_MEMORY = getWriterMaxMemory(conf); final Long DEFAULT_MAX_LATENCY = getWriterMaxLatency(conf); final Integer DEFAULT_MAX_WRITE_THREADS = getWriterMaxWriteThreads(conf); final Connector connector = ConfigUtils.getConnector(conf); return connector.createMultiTableBatchWriter(DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS); }
MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(200000l, 300, 4);
protected AccumuloRecordWriter(JobConf job) throws AccumuloException, AccumuloSecurityException, IOException { Level l = getLogLevel(job); if (l != null) log.setLevel(getLogLevel(job)); this.simulate = getSimulationMode(job); this.createTables = canCreateTables(job); if (simulate) log.info("Simulating output only. No writes to tables will occur"); this.bws = new HashMap<>(); String tname = getDefaultTableName(job); this.defaultTableName = (tname == null) ? null : new Text(tname); if (!simulate) { this.conn = getInstance(job).getConnector(getPrincipal(job), getAuthenticationToken(job)); mtbw = conn.createMultiTableBatchWriter(getBatchWriterOptions(job)); } }
private static TemporalIndexer getTemporalIndexer(final Configuration conf) throws IOException { if (!conf.getBoolean(ENABLE_TEMPORAL, true)) { return null; } final AccumuloTemporalIndexer temporal = new AccumuloTemporalIndexer(); temporal.setConf(conf); Connector connector; try { connector = ConfigUtils.getConnector(conf); } catch (AccumuloException | AccumuloSecurityException e) { throw new IOException("Error when attempting to create a connection for writing the temporal index.", e); } final MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig()); temporal.setConnector(connector); temporal.setMultiTableBatchWriter(mtbw); temporal.init(); return temporal; }
protected AccumuloRecordWriter(TaskAttemptContext context) throws AccumuloException, AccumuloSecurityException, IOException { Level l = getLogLevel(context); if (l != null) log.setLevel(getLogLevel(context)); this.simulate = getSimulationMode(context); this.createTables = canCreateTables(context); if (simulate) log.info("Simulating output only. No writes to tables will occur"); this.bws = new HashMap<>(); String tname = getDefaultTableName(context); this.defaultTableName = (tname == null) ? null : new Text(tname); if (!simulate) { this.conn = getInstance(context).getConnector(getPrincipal(context), getAuthenticationToken(context)); mtbw = conn.createMultiTableBatchWriter(getBatchWriterOptions(context)); } }
private static FreeTextIndexer getFreeTextIndexer(final Configuration conf) throws IOException { if (!conf.getBoolean(ENABLE_FREETEXT, true)) { return null; } final AccumuloFreeTextIndexer freeText = new AccumuloFreeTextIndexer(); freeText.setConf(conf); Connector connector; try { connector = ConfigUtils.getConnector(conf); } catch (AccumuloException | AccumuloSecurityException e) { throw new IOException("Error when attempting to create a connection for writing the freeText index.", e); } final MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig()); freeText.setConnector(connector); freeText.setMultiTableBatchWriter(mtbw); freeText.init(); return freeText; }
private static TemporalIndexer getTemporalIndexer(final Configuration conf) throws IOException { if (!conf.getBoolean(ENABLE_TEMPORAL, true)) { return null; } final AccumuloTemporalIndexer temporal = new AccumuloTemporalIndexer(); temporal.setConf(conf); Connector connector; try { connector = ConfigUtils.getConnector(conf); } catch (AccumuloException | AccumuloSecurityException e) { throw new IOException("Error when attempting to create a connection for writing the temporal index.", e); } final MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig()); temporal.setConnector(connector); temporal.setMultiTableBatchWriter(mtbw); temporal.init(); return temporal; }
private static FreeTextIndexer getFreeTextIndexer(final Configuration conf) throws IOException { if (!conf.getBoolean(ENABLE_FREETEXT, true)) { return null; } final AccumuloFreeTextIndexer freeText = new AccumuloFreeTextIndexer(); freeText.setConf(conf); Connector connector; try { connector = ConfigUtils.getConnector(conf); } catch (AccumuloException | AccumuloSecurityException e) { throw new IOException("Error when attempting to create a connection for writing the freeText index.", e); } final MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig()); freeText.setConnector(connector); freeText.setMultiTableBatchWriter(mtbw); freeText.init(); return freeText; }
public void openIngest() { switch(state) { case New: createTablesSoft(); break; case Open: throw new IllegalStateException("tried to open ingest when already open"); case Closed: break; } BatchWriterConfig BWconfig = new BatchWriterConfig(); BWconfig.setMaxMemory(tconf.batchBytes); mtbw = tconf.connector.createMultiTableBatchWriter(BWconfig); try { if (tconf.useTable) Btable = mtbw.getBatchWriter(TNtable); if (tconf.useTableT) BtableT = TNtableT.equals(TNtable) ? Btable : mtbw.getBatchWriter(TNtableT); if (tconf.useTableDeg) BtableDeg = mtbw.getBatchWriter(TNtableDeg); if (tconf.useTableDegT) BtableDegT = TNtableDegT.equals(TNtableDeg) ? BtableDeg : mtbw.getBatchWriter(TNtableDegT); if (tconf.useTableField) BtableField = mtbw.getBatchWriter(TNtableField); if (tconf.useTableFieldT) BtableFieldT = TNtableFieldT.equals(TNtableField) ? BtableField : mtbw.getBatchWriter(TNtableFieldT); if (tconf.useEdgeTable) BtableEdge = mtbw.getBatchWriter(TNtableEdge); if (tconf.useEdgeTableT) BtableEdgeT = mtbw.getBatchWriter(TNtableEdgeT); if (tconf.useEdgeTableDegT) BtableEdgeDegT = mtbw.getBatchWriter(TNtableEdgeDegT); } catch (TableNotFoundException e) { log.error("crazy. Tables should have been created.", e); } catch (AccumuloSecurityException | AccumuloException e) { log.warn("error creating one of the batch writers for D4MTableWriter base " + TNtable, e); } state = State.Open; }
this.conn = connectionFactory.getConnection(Priority.ADMIN, trackingMap); mtbw = conn.createMultiTableBatchWriter(getMaxMutationBufferSize(conf), getMaxLatency(conf), getMaxWriteThreads(conf)); } catch (Exception e) { log.error(e.getMessage(), e);
writerAll = connector.createMultiTableBatchWriter(bwc); else writerAll = null;
final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
/** * Create an ingester using the given configuration parameters. * * @param config * @throws AccumuloException * @throws AccumuloSecurityException * @throws TableNotFoundException * @throws TableExistsException * @throws InterruptedException * @throws IOException */ public AccumuloBulkIngester(AccumuloGraphConfiguration config) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException, IOException, InterruptedException { this.config = config; connector = config.getConnector(); AccumuloGraphUtils.handleCreateAndClear(config); mtbw = connector.createMultiTableBatchWriter(config.getBatchWriterConfig()); vertexWriter = mtbw.getBatchWriter(config.getVertexTableName()); edgeWriter = mtbw.getBatchWriter(config.getEdgeTableName()); }
/** * Main constructor. * @param config */ public AccumuloGraph(AccumuloGraphConfiguration config) { config.validate(); AccumuloGraphUtils.handleCreateAndClear(config); try { globals = new GlobalInstances(config, config.getConnector() .createMultiTableBatchWriter(config.getBatchWriterConfig()), new ElementCaches(config)); } catch (Exception e) { throw new AccumuloGraphException(e); } }
/** * Create an ingester using the given configuration parameters. * * @param config * @throws AccumuloException * @throws AccumuloSecurityException * @throws TableNotFoundException * @throws TableExistsException * @throws InterruptedException * @throws IOException */ public AccumuloBulkIngester(AccumuloGraphConfiguration config) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException, IOException, InterruptedException { this.config = config; connector = config.getConnector(); AccumuloGraphUtils.handleCreateAndClear(config); mtbw = connector.createMultiTableBatchWriter(config.getBatchWriterConfig()); vertexWriter = mtbw.getBatchWriter(config.getVertexTableName()); edgeWriter = mtbw.getBatchWriter(config.getEdgeTableName()); }
/** * Main constructor. * @param config */ public AccumuloGraph(AccumuloGraphConfiguration config) { config.validate(); AccumuloGraphUtils.handleCreateAndClear(config); try { globals = new GlobalInstances(config, config.getConnector() .createMultiTableBatchWriter(config.getBatchWriterConfig()), new ElementCaches(config)); } catch (Exception e) { throw new AccumuloGraphException(e); } }