private MutationSet init() { if (recentFailures == null) { recentFailures = new MutationSet(); initTime = System.currentTimeMillis(); } return recentFailures; }
synchronized void add(TabletServerMutations<Mutation> tsm) { init(); for (Entry<KeyExtent,List<Mutation>> entry : tsm.getMutations().entrySet()) { recentFailures.addAll(entry.getKey().getTableId(), entry.getValue()); } }
public void addAll(MutationSet failures) { Set<Entry<Table.ID,List<Mutation>>> es = failures.getMutations().entrySet(); for (Entry<Table.ID,List<Mutation>> entry : es) { Table.ID table = entry.getKey(); for (Mutation mutation : entry.getValue()) { addMutation(table, mutation); } } }
throws IOException, AccumuloSecurityException, AccumuloServerException { if (tabMuts.size() == 0) { return new MutationSet(); MutationSet allFailures = new MutationSet(); entry.getValue().get(0).toThrift(), DurabilityImpl.toThrift(durability)); } catch (NotServingTabletException e) { allFailures.addAll(entry.getKey().getTableId(), entry.getValue()); getLocator(entry.getKey().getTableId()).invalidateCache(entry.getKey()); } catch (ConstraintViolationException e) { allFailures.addAll(tableId, mutations.subList(numCommitted, mutations.size()));
log.trace("sent " + String.format("%,d", count) + " mutations to " + location + " in " + String.format("%.2f secs (%,.2f mutations/sec) with %,d failures", (st2 - st1) / 1000.0, count / ((st2 - st1) / 1000.0), failures.size())); if (failures.size() > 0) { failedMutations.add(failures); successBytes -= failures.getMemoryUsed();
mutations.addMutation(table, m); totalAdded++; if (mutations.getMemoryUsed() >= maxMem / 2) { startProcessing(); checkForFailures();
Table.ID tableId = null; try { Set<Entry<Table.ID,List<Mutation>>> es = mutationsToProcess.getMutations().entrySet(); for (Entry<Table.ID,List<Mutation>> entry : es) { tableId = entry.getKey();
public TabletServerBatchWriter(ClientContext context, BatchWriterConfig config) { this.context = context; this.maxMem = config.getMaxMemory(); this.maxLatency = config.getMaxLatency(TimeUnit.MILLISECONDS) <= 0 ? Long.MAX_VALUE : config.getMaxLatency(TimeUnit.MILLISECONDS); this.timeout = config.getTimeout(TimeUnit.MILLISECONDS); this.mutations = new MutationSet(); this.lastProcessingStartTime = System.currentTimeMillis(); this.durability = config.getDurability(); this.writer = new MutationWriter(config.getMaxWriteThreads()); if (this.maxLatency != Long.MAX_VALUE) { jtimer.schedule(new TimerTask() { @Override public void run() { try { synchronized (TabletServerBatchWriter.this) { if ((System.currentTimeMillis() - lastProcessingStartTime) > TabletServerBatchWriter.this.maxLatency) startProcessing(); } } catch (Throwable t) { updateUnknownErrors("Max latency task failed " + t.getMessage(), t); } } }, 0, this.maxLatency / 4); } }
@Override public void run() { try { MutationSet rf = null; synchronized (this) { if (recentFailures != null && System.currentTimeMillis() - initTime > 1000) { rf = recentFailures; recentFailures = null; } } if (rf != null) { if (log.isTraceEnabled()) log.trace("tid={} Requeuing {} failed mutations", Thread.currentThread().getId(), rf.size()); addFailedMutations(rf); } } catch (Throwable t) { updateUnknownErrors("tid=" + Thread.currentThread().getId() + " Failed to requeue failed mutations " + t.getMessage(), t); cancel(); } } }
void queueMutations(final MutationSet mutationsToSend) { if (mutationsToSend == null) return; binningThreadPool.execute(Trace.wrap(() -> { if (mutationsToSend != null) { try { log.trace("{} - binning {} mutations", Thread.currentThread().getName(), mutationsToSend.size()); addMutations(mutationsToSend); } catch (Exception e) { updateUnknownErrors("Error processing mutation set", e); } } })); }
synchronized void add(MutationSet failures) { init().addAll(failures); }
synchronized void add(Table.ID table, ArrayList<Mutation> tableFailures) { init().addAll(table, tableFailures); }
private void addMutations(MutationSet mutationsToSend) { Map<String,TabletServerMutations<Mutation>> binnedMutations = new HashMap<>(); Span span = Trace.start("binMutations"); try { long t1 = System.currentTimeMillis(); binMutations(mutationsToSend, binnedMutations); long t2 = System.currentTimeMillis(); updateBinningStats(mutationsToSend.size(), (t2 - t1), binnedMutations); } finally { span.stop(); } addMutations(binnedMutations); }
public void addAll(Table.ID table, List<Mutation> mutations) { for (Mutation mutation : mutations) { addMutation(table, mutation); } }
/** * Add mutations that previously failed back into the mix */ private synchronized void addFailedMutations(MutationSet failedMutations) { mutations.addAll(failedMutations); if (mutations.getMemoryUsed() >= maxMem / 2 || closed || flushing) { startProcessing(); } }
private synchronized void startProcessing() { if (mutations.getMemoryUsed() == 0) return; lastProcessingStartTime = System.currentTimeMillis(); writer.queueMutations(mutations); mutations = new MutationSet(); }