BackpressureRegulator(HazelcastProperties properties, ILogger logger) { this.enabled = properties.getBoolean(BACKPRESSURE_ENABLED); this.disabled = !enabled; this.partitionCount = properties.getInteger(PARTITION_COUNT); this.syncWindow = getSyncWindow(properties); this.syncCountdown.set(syncWindow); this.maxConcurrentInvocations = getMaxConcurrentInvocations(properties); this.backoffTimeoutMs = getBackoffTimeoutMs(properties); if (enabled) { logger.info("Backpressure is enabled" + ", maxConcurrentInvocations:" + maxConcurrentInvocations + ", syncWindow: " + syncWindow); int backupTimeoutMillis = properties.getInteger(OPERATION_BACKUP_TIMEOUT_MILLIS); if (backupTimeoutMillis < MINUTES.toMillis(1)) { logger.warning( format("Back pressure is enabled, but '%s' is too small. ", OPERATION_BACKUP_TIMEOUT_MILLIS.getName())); } } else { logger.info("Backpressure is disabled"); } }
/** * Checks if a sync is forced for the given BackupAwareOperation. * <p> * Once and a while for every BackupAwareOperation with one or more async backups, these async backups are transformed * into a sync backup. * * @param backupAwareOp the BackupAwareOperation to check * @return {@code true} if a sync needs to be forced, {@code false} otherwise */ boolean isSyncForced(BackupAwareOperation backupAwareOp) { if (disabled) { return false; } // if there are no asynchronous backups, there is nothing to regulate. if (backupAwareOp.getAsyncBackupCount() == 0) { return false; } if (backupAwareOp instanceof UrgentSystemOperation) { return false; } for (; ; ) { int current = syncCountdown.decrementAndGet(); if (current > 0) { return false; } if (syncCountdown.compareAndSet(current, randomSyncDelay())) { return true; } } }
this.failOnIndeterminateOperationState = nodeEngine.getProperties().getBoolean(FAIL_ON_INDETERMINATE_OPERATION_STATE); this.backpressureRegulator = new BackpressureRegulator( node.getProperties(), node.getLogger(BackpressureRegulator.class)); node.getLogger(OperationServiceImpl.class), backpressureRegulator.newCallIdSequence());
int sendBackups0(BackupAwareOperation backupAwareOp) throws Exception { int requestedSyncBackups = requestedSyncBackups(backupAwareOp); int requestedAsyncBackups = requestedAsyncBackups(backupAwareOp); int requestedTotalBackups = requestedTotalBackups(backupAwareOp); if (requestedTotalBackups == 0) { return 0; } Operation op = (Operation) backupAwareOp; PartitionReplicaVersionManager versionManager = node.getPartitionService().getPartitionReplicaVersionManager(); ServiceNamespace namespace = versionManager.getServiceNamespace(op); long[] replicaVersions = versionManager.incrementPartitionReplicaVersions(op.getPartitionId(), namespace, requestedTotalBackups); boolean syncForced = backpressureRegulator.isSyncForced(backupAwareOp); int syncBackups = syncBackups(requestedSyncBackups, requestedAsyncBackups, syncForced); int asyncBackups = asyncBackups(requestedSyncBackups, requestedAsyncBackups, syncForced); // TODO: This could cause a problem with back pressure if (!op.returnsResponse()) { asyncBackups += syncBackups; syncBackups = 0; } if (syncBackups + asyncBackups == 0) { return 0; } return makeBackups(backupAwareOp, op.getPartitionId(), replicaVersions, syncBackups, asyncBackups); }
this.failOnIndeterminateOperationState = nodeEngine.getProperties().getBoolean(FAIL_ON_INDETERMINATE_OPERATION_STATE); this.backpressureRegulator = new BackpressureRegulator( node.getProperties(), node.getLogger(BackpressureRegulator.class)); node.getLogger(OperationServiceImpl.class), backpressureRegulator.newCallIdSequence());
int sendBackups0(BackupAwareOperation backupAwareOp) throws Exception { int requestedSyncBackups = requestedSyncBackups(backupAwareOp); int requestedAsyncBackups = requestedAsyncBackups(backupAwareOp); int requestedTotalBackups = requestedTotalBackups(backupAwareOp); if (requestedTotalBackups == 0) { return 0; } Operation op = (Operation) backupAwareOp; PartitionReplicaVersionManager versionManager = node.getPartitionService().getPartitionReplicaVersionManager(); ServiceNamespace namespace = versionManager.getServiceNamespace(op); long[] replicaVersions = versionManager.incrementPartitionReplicaVersions(op.getPartitionId(), namespace, requestedTotalBackups); boolean syncForced = backpressureRegulator.isSyncForced(backupAwareOp); int syncBackups = syncBackups(requestedSyncBackups, requestedAsyncBackups, syncForced); int asyncBackups = asyncBackups(requestedSyncBackups, requestedAsyncBackups, syncForced); // TODO: This could cause a problem with back pressure if (!op.returnsResponse()) { asyncBackups += syncBackups; syncBackups = 0; } if (syncBackups + asyncBackups == 0) { return 0; } return makeBackups(backupAwareOp, op.getPartitionId(), replicaVersions, syncBackups, asyncBackups); }
BackpressureRegulator(HazelcastProperties properties, ILogger logger) { this.enabled = properties.getBoolean(BACKPRESSURE_ENABLED); this.disabled = !enabled; this.partitionCount = properties.getInteger(PARTITION_COUNT); this.syncWindow = getSyncWindow(properties); this.syncCountdown.set(syncWindow); this.maxConcurrentInvocations = getMaxConcurrentInvocations(properties); this.backoffTimeoutMs = getBackoffTimeoutMs(properties); if (enabled) { logger.info("Backpressure is enabled" + ", maxConcurrentInvocations:" + maxConcurrentInvocations + ", syncWindow: " + syncWindow); int backupTimeoutMillis = properties.getInteger(OPERATION_BACKUP_TIMEOUT_MILLIS); if (backupTimeoutMillis < MINUTES.toMillis(1)) { logger.warning( format("Back pressure is enabled, but '%s' is too small. ", OPERATION_BACKUP_TIMEOUT_MILLIS.getName())); } } else { logger.info("Backpressure is disabled"); } }
/** * Checks if a sync is forced for the given BackupAwareOperation. * <p> * Once and a while for every BackupAwareOperation with one or more async backups, these async backups are transformed * into a sync backup. * * @param backupAwareOp the BackupAwareOperation to check * @return {@code true} if a sync needs to be forced, {@code false} otherwise */ boolean isSyncForced(BackupAwareOperation backupAwareOp) { if (disabled) { return false; } // if there are no asynchronous backups, there is nothing to regulate. if (backupAwareOp.getAsyncBackupCount() == 0) { return false; } if (backupAwareOp instanceof UrgentSystemOperation) { return false; } for (; ; ) { int current = syncCountdown.decrementAndGet(); if (current > 0) { return false; } if (syncCountdown.compareAndSet(current, randomSyncDelay())) { return true; } } }