/** * Creates a BatchWriter with the expected configuration. * * @param table The table to write to */ private BatchWriter createBatchWriter(String table) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency( Long.parseLong(getProperties() .getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong( getProperties().getProperty("accumulo.batchWriterSize", "100000"))); final String numThreadsValue = getProperties().getProperty("accumulo.batchWriterThreads"); // Try to saturate the client machine. int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); if (null != numThreadsValue) { numThreads = Integer.parseInt(numThreadsValue); } System.err.println("Using " + numThreads + " threads to write data"); bwc.setMaxWriteThreads(numThreads); return connector.createBatchWriter(table, bwc); }
/** * Creates a BatchWriter with the expected configuration. * * @param table The table to write to */ private BatchWriter createBatchWriter(String table) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency( Long.parseLong(getProperties() .getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong( getProperties().getProperty("accumulo.batchWriterSize", "100000"))); final String numThreadsValue = getProperties().getProperty("accumulo.batchWriterThreads"); // Try to saturate the client machine. int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); if (null != numThreadsValue) { numThreads = Integer.parseInt(numThreadsValue); } System.err.println("Using " + numThreads + " threads to write data"); bwc.setMaxWriteThreads(numThreads); return connector.createBatchWriter(table, bwc); }
/** * Creates a BatchWriter with the expected configuration. * * @param table The table to write to */ private BatchWriter createBatchWriter(String table) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency( Long.parseLong(getProperties() .getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong( getProperties().getProperty("accumulo.batchWriterSize", "100000"))); final String numThreadsValue = getProperties().getProperty("accumulo.batchWriterThreads"); // Try to saturate the client machine. int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); if (null != numThreadsValue) { numThreads = Integer.parseInt(numThreadsValue); } System.err.println("Using " + numThreads + " threads to write data"); bwc.setMaxWriteThreads(numThreads); return connector.createBatchWriter(table, bwc); }
BatchWriter createBatchWriter() { try { return context.createBatchWriter(targetTableName, new BatchWriterConfig().setMaxMemory(MAX_MEMORY) .setMaxLatency(LATENCY, TimeUnit.MILLISECONDS).setMaxWriteThreads(THREADS)); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { checkArgument(tableName != null, "tableName is null"); checkArgument(authorizations != null, "authorizations is null"); return new TabletServerBatchDeleter(context, context.getTableId(tableName), authorizations, numQueryThreads, new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException, ConstraintViolationException { shellState.checkTableState(); final Mutation m = new Mutation(new Text(cl.getArgs()[0].getBytes(Shell.CHARSET))); final Text colf = new Text(cl.getArgs()[1].getBytes(Shell.CHARSET)); final Text colq = new Text(cl.getArgs()[2].getBytes(Shell.CHARSET)); if (cl.hasOption(deleteOptAuths.getOpt())) { final ColumnVisibility le = new ColumnVisibility(cl.getOptionValue(deleteOptAuths.getOpt())); if (cl.hasOption(timestampOpt.getOpt())) { m.putDelete(colf, colq, le, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt()))); } else { m.putDelete(colf, colq, le); } } else if (cl.hasOption(timestampOpt.getOpt())) { m.putDelete(colf, colq, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt()))); } else { m.putDelete(colf, colq); } final BatchWriter bw = shellState.getAccumuloClient().createBatchWriter( shellState.getTableName(), new BatchWriterConfig().setMaxMemory(Math.max(m.estimatedMemoryUsed(), 1024)) .setMaxWriteThreads(1).setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS)); bw.addMutation(m); bw.close(); return 0; }
public BatchWriterConfig getBatchWriterConfig() { BatchWriterConfig config = new BatchWriterConfig(); config.setMaxWriteThreads(this.batchThreads); config.setMaxLatency(this.batchLatency, TimeUnit.MILLISECONDS); config.setMaxMemory(this.batchMemory); config.setTimeout(this.batchTimeout, TimeUnit.MILLISECONDS); return config; }
@Override public BatchWriter createBatchWriter(String tableName, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { checkArgument(tableName != null, "tableName is null"); return new BatchWriterImpl(context, context.getTableId(tableName), new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
.setMaxMemory(Math.max(m.estimatedMemoryUsed(), 1024)).setMaxWriteThreads(1) .setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS); if (cl.hasOption(durabilityOption.getOpt())) {
@Override public MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads) { return new MultiTableBatchWriterImpl(context, new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
public BatchWriterConfig getBatchWriterConfig() { ensureOpen(); if (batchWriterConfig == null) { Properties props = info.getProperties(); batchWriterConfig = new BatchWriterConfig(); Long maxMemory = ClientProperty.BATCH_WRITER_MEMORY_MAX.getBytes(props); if (maxMemory != null) { batchWriterConfig.setMaxMemory(maxMemory); } Long maxLatency = ClientProperty.BATCH_WRITER_LATENCY_MAX.getTimeInMillis(props); if (maxLatency != null) { batchWriterConfig.setMaxLatency(maxLatency, TimeUnit.SECONDS); } Long timeout = ClientProperty.BATCH_WRITER_TIMEOUT_MAX.getTimeInMillis(props); if (timeout != null) { batchWriterConfig.setTimeout(timeout, TimeUnit.SECONDS); } String durability = ClientProperty.BATCH_WRITER_DURABILITY.getValue(props); if (!durability.isEmpty()) { batchWriterConfig.setDurability(Durability.valueOf(durability.toUpperCase())); } } return batchWriterConfig; }
bwConfig.setMaxMemory(memoryInBytes); try { bw = context.createBatchWriter(tableName, bwConfig);
try (Scanner ms = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY); BatchWriter bw = new BatchWriterImpl(context, MetadataTable.ID, new BatchWriterConfig().setMaxMemory(1000000) .setMaxLatency(120000L, TimeUnit.MILLISECONDS).setMaxWriteThreads(2))) {
BatchWriter createBatchWriter() { try { return context.getConnector().createBatchWriter(targetTableName, new BatchWriterConfig().setMaxMemory(MAX_MEMORY) .setMaxLatency(LATENCY, TimeUnit.MILLISECONDS).setMaxWriteThreads(THREADS)); } catch (TableNotFoundException e) { // ya, I don't think so throw new RuntimeException(e); } catch (Exception e) { throw new RuntimeException(e); } }
@Override @Deprecated public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { return createBatchDeleter(tableName, authorizations, numQueryThreads, new BatchWriterConfig().setMaxLatency(maxLatency, TimeUnit.MILLISECONDS) .setMaxMemory(maxMemory).setMaxWriteThreads(maxWriteThreads)); }
@Override @Deprecated public MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads) { return createMultiTableBatchWriter(new BatchWriterConfig().setMaxMemory(maxMemory).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS) .setMaxWriteThreads(maxWriteThreads)); }
@Deprecated @Override public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { checkArgument(tableName != null, "tableName is null"); checkArgument(authorizations != null, "authorizations is null"); return new TabletServerBatchDeleter(context, getTableId(tableName), authorizations, numQueryThreads, new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
@Deprecated @Override public BatchWriter createBatchWriter(String tableName, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { checkArgument(tableName != null, "tableName is null"); return new BatchWriterImpl(context, getTableId(tableName), new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
public BatchWriterConfig getBatchWriterConfig() { BatchWriterConfig config = new BatchWriterConfig(); config.setMaxWriteThreads(this.batchThreads); config.setMaxLatency(this.batchLatency, TimeUnit.MILLISECONDS); config.setMaxMemory(this.batchMemory); config.setTimeout(this.batchTimeout, TimeUnit.MILLISECONDS); return config; }
/** * Create a {@link BatchWriterConfig} based on this configuration. * @return */ public BatchWriterConfig getBatchWriterConfig() { return new BatchWriterConfig().setMaxLatency(getMaxWriteLatency(), TimeUnit.MILLISECONDS) .setMaxMemory(getMaxWriteMemory()) .setMaxWriteThreads(getMaxWriteThreads()) .setTimeout(getMaxWriteTimeout(), TimeUnit.MILLISECONDS); }