stateCounts = new MapCounter<>(); counts.put(tableID, stateCounts); stateCounts.increment(srs, 1);
MapCounter<TServerInstance> serversToFlush = new MapCounter<>(); long t1 = System.currentTimeMillis(); tabletsToWaitFor++; if (tablet.hasCurrent()) { serversToFlush.increment(new TServerInstance(tablet.getLocation()), 1); TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null); if (serversToFlush.size() == 0 && Tables.getTableState(master.getContext(), tableId) == TableState.OFFLINE) throw new AcceptableThriftTableOperationException(tableId.canonicalID(), null, return 0; for (TServerInstance tsi : serversToFlush.keySet()) { try { final TServerConnection server = master.getConnection(tsi); if (serversToFlush.size() > 0) sleepTime = Collections.max(serversToFlush.values()) * sleepTime;
MapCounter<String> groupCounts = new MapCounter<>(); Map<TServerInstance,TserverGroupInfo> tservers = new HashMap<>(); groupCounts.increment(group, 1); TserverGroupInfo tgi = tservers.get(loc.getTserverInstance()); tgi.addGroup(group); for (String group : groupCounts.keySet()) { long groupCount = groupCounts.get(group); totalExtra += groupCount % current.size(); expectedCounts.put(group, (int) (groupCount / current.size()));
public long decrement(KT key, long l) { return increment(key, -1 * l); }
@Override void start(Path bulkDir, Master master, long tid, boolean setTime) throws Exception { super.start(bulkDir, master, tid, setTime); timeInMillis = master.getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT); fmtTid = String.format("%016x", tid); loadMsgs = new MapCounter<>(); loadQueue = new HashMap<>(); }
void returnFilesForScan(Long reservationId) { final Set<FileRef> filesToDelete = new HashSet<>(); synchronized (tablet) { Set<FileRef> absFilePaths = scanFileReservations.remove(reservationId); if (absFilePaths == null) throw new IllegalArgumentException("Unknown scan reservation id " + reservationId); boolean notify = false; for (FileRef path : absFilePaths) { long refCount = fileScanReferenceCounts.decrement(path, 1); if (refCount == 0) { if (filesToDeleteAfterScan.remove(path)) filesToDelete.add(path); notify = true; } else if (refCount < 0) throw new IllegalStateException("Scan ref count for " + path + " is " + refCount); } if (notify) tablet.notifyAll(); } if (filesToDelete.size() > 0) { log.debug("Removing scan refs from metadata {} {}", tablet.getExtent(), filesToDelete); MetadataTableUtil.removeScanFiles(tablet.getExtent(), filesToDelete, tablet.getContext(), tablet.getTabletServer().getLock()); } }
int holes = 0; Text continueRow = null; MapCounter<String> serverCounts = new MapCounter<>(); serverCounts.increment(loc.getId(), 1); long waitTime; long maxPerServer = 0; if (serverCounts.size() > 0) { maxPerServer = Collections.max(serverCounts.values()); waitTime = maxPerServer * 10; } else
MapCounter<String> groupCounts = new MapCounter<>(); Map<TServerInstance,TserverGroupInfo> tservers = new HashMap<>(); groupCounts.increment(group, 1); TserverGroupInfo tgi = tservers.get(loc.getTserverInstance()); tgi.addGroup(group); for (String group : groupCounts.keySet()) { long groupCount = groupCounts.get(group); totalExtra += groupCount % current.size(); expectedCounts.put(group, (int) (groupCount / current.size()));
private void addToQueue(HostAndPort server, KeyExtent extent, Map<String,MapFileInfo> thriftImports) { if (!thriftImports.isEmpty()) { loadMsgs.increment(server, 1); Map<String,MapFileInfo> prev = loadQueue.computeIfAbsent(server, k -> new HashMap<>()) .putIfAbsent(extent.toThrift(), thriftImports); Preconditions.checkState(prev == null, "Unexpectedly saw extent %s twice", extent); // keep a very rough estimate of how much is memory so we can send if over a few megs is // buffered queuedDataSize += thriftImports.keySet().stream().mapToInt(String::length).sum() + server.getHost().length() + 4 + thriftImports.size() * 32; } }
@Override void start(Path bulkDir, Master master, long tid, boolean setTime) throws Exception { Preconditions.checkArgument(!setTime); super.start(bulkDir, master, tid, setTime); bw = master.getContext().createBatchWriter(MetadataTable.NAME); unloadingTablets = new MapCounter<>(); }
void returnFilesForScan(Long reservationId) { final Set<FileRef> filesToDelete = new HashSet<>(); synchronized (tablet) { Set<FileRef> absFilePaths = scanFileReservations.remove(reservationId); if (absFilePaths == null) throw new IllegalArgumentException("Unknown scan reservation id " + reservationId); boolean notify = false; for (FileRef path : absFilePaths) { long refCount = fileScanReferenceCounts.decrement(path, 1); if (refCount == 0) { if (filesToDeleteAfterScan.remove(path)) filesToDelete.add(path); notify = true; } else if (refCount < 0) throw new IllegalStateException("Scan ref count for " + path + " is " + refCount); } if (notify) tablet.notifyAll(); } if (filesToDelete.size() > 0) { log.debug("Removing scan refs from metadata " + tablet.getExtent() + " " + filesToDelete); MetadataTableUtil.removeScanFiles(tablet.getExtent(), filesToDelete, tablet.getTabletServer(), tablet.getTabletServer().getLock()); } }
MapCounter<TServerInstance> serversToFlush = new MapCounter<>(); Connector conn = master.getConnector(); tabletsToWaitFor++; if (server != null) serversToFlush.increment(server, 1); TableOperationExceptionType.NOTFOUND, null); if (serversToFlush.size() == 0 && Tables.getTableState(instance, tableId) == TableState.OFFLINE) throw new AcceptableThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OFFLINE, null); return 0; for (TServerInstance tsi : serversToFlush.keySet()) { try { final TServerConnection server = master.getConnection(tsi); if (serversToFlush.size() > 0) sleepTime = Collections.max(serversToFlush.values()) * sleepTime; // make wait time depend on
stateCounts = new MapCounter<>(); counts.put(tableID, stateCounts); stateCounts.increment(srs, 1);
Pair<Long,Map<FileRef,DataFileValue>> reserveFilesForScan() { synchronized (tablet) { while (reservationsBlocked) { try { tablet.wait(50); } catch (InterruptedException e) { log.warn("{}", e.getMessage(), e); } } Set<FileRef> absFilePaths = new HashSet<>(datafileSizes.keySet()); long rid = nextScanReservationId++; scanFileReservations.put(rid, absFilePaths); Map<FileRef,DataFileValue> ret = new HashMap<>(); for (FileRef path : absFilePaths) { fileScanReferenceCounts.increment(path, 1); ret.put(path, datafileSizes.get(path)); } return new Pair<>(rid, ret); } }
@Override long finish() { sendQueued(0); long sleepTime = 0; if (loadMsgs.size() > 0) { // find which tablet server had the most load messages sent to it and sleep 13ms for each // load message sleepTime = Collections.max(loadMsgs.values()) * 13; } if (locationLess > 0) { sleepTime = Math.max(Math.max(100L, locationLess), sleepTime); } return sleepTime; }
void removeFilesAfterScan(Set<FileRef> scanFiles) { if (scanFiles.size() == 0) return; Set<FileRef> filesToDelete = new HashSet<>(); synchronized (tablet) { for (FileRef path : scanFiles) { if (fileScanReferenceCounts.get(path) == 0) filesToDelete.add(path); else filesToDeleteAfterScan.add(path); } } if (filesToDelete.size() > 0) { log.debug("Removing scan refs from metadata {} {}", tablet.getExtent(), filesToDelete); MetadataTableUtil.removeScanFiles(tablet.getExtent(), filesToDelete, tablet.getContext(), tablet.getTabletServer().getLock()); } }