private JCQueue createQueue(String name, int batchSize, int queueSize) { return new JCQueue(name, queueSize, 0, batchSize, waitStrategy, "test", "test", 1000, 1000); }
private static void ackingProducerSimulation() { WaitStrategyPark ws = new WaitStrategyPark(100); JCQueue spoutQ = new JCQueue("spoutQ", 1024, 0, 100, ws, "test", "test", 1000, 1000); JCQueue ackQ = new JCQueue("ackQ", 1024, 0, 100, ws, "test", "test", 1000, 1000); final AckingProducer ackingProducer = new AckingProducer(spoutQ, ackQ); final Acker acker = new Acker(ackQ, spoutQ); runAllThds(ackingProducer, acker); }
private static void producerFwdConsumer(int prodBatchSz) { WaitStrategyPark ws = new WaitStrategyPark(100); JCQueue q1 = new JCQueue("q1", 1024, 0, prodBatchSz, ws, "test", "test", 1000, 1000); JCQueue q2 = new JCQueue("q2", 1024, 0, prodBatchSz, ws, "test", "test", 1000, 1000); final Producer prod = new Producer(q1); final Forwarder fwd = new Forwarder(q1, q2); final Consumer cons = new Consumer(q2); runAllThds(prod, fwd, cons); }
private static void oneProducer2Consumers(int prodBatchSz) { WaitStrategyPark ws = new WaitStrategyPark(100); JCQueue q1 = new JCQueue("q1", 1024, 0, prodBatchSz, ws, "test", "test", 1000, 1000); JCQueue q2 = new JCQueue("q2", 1024, 0, prodBatchSz, ws, "test", "test", 1000, 1000); final Producer2 prod1 = new Producer2(q1, q2); final Consumer cons1 = new Consumer(q1); final Consumer cons2 = new Consumer(q2); runAllThds(prod1, cons1, cons2); }
private Map<List<Long>, JCQueue> mkReceiveQueueMap(Map<String, Object> topologyConf, Set<List<Long>> executors) { Integer recvQueueSize = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE)); Integer recvBatchSize = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_PRODUCER_BATCH_SIZE)); Integer overflowLimit = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_EXECUTOR_OVERFLOW_LIMIT)); if (recvBatchSize > recvQueueSize / 2) { throw new IllegalArgumentException(Config.TOPOLOGY_PRODUCER_BATCH_SIZE + ":" + recvBatchSize + " is greater than half of " + Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE + ":" + recvQueueSize); } IWaitStrategy backPressureWaitStrategy = IWaitStrategy.createBackPressureWaitStrategy(topologyConf); Map<List<Long>, JCQueue> receiveQueueMap = new HashMap<>(); for (List<Long> executor : executors) { int port = this.getPort(); receiveQueueMap.put(executor, new JCQueue("receive-queue" + executor.toString(), recvQueueSize, overflowLimit, recvBatchSize, backPressureWaitStrategy, this.getTopologyId(), Constants.SYSTEM_COMPONENT_ID, -1, this.getPort())); } return receiveQueueMap; }
public WorkerTransfer(WorkerState workerState, Map<String, Object> topologyConf, int maxTaskIdInTopo) { this.workerState = workerState; this.backPressureWaitStrategy = IWaitStrategy.createBackPressureWaitStrategy(topologyConf); this.drainer = new TransferDrainer(); this.remoteBackPressureStatus = new AtomicBoolean[maxTaskIdInTopo + 1]; for (int i = 0; i < remoteBackPressureStatus.length; i++) { remoteBackPressureStatus[i] = new AtomicBoolean(false); } Integer xferQueueSz = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE)); Integer xferBatchSz = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_TRANSFER_BATCH_SIZE)); if (xferBatchSz > xferQueueSz / 2) { throw new IllegalArgumentException(Config.TOPOLOGY_TRANSFER_BATCH_SIZE + ":" + xferBatchSz + " must be no more than half of " + Config.TOPOLOGY_TRANSFER_BUFFER_SIZE + ":" + xferQueueSz); } this.transferQueue = new JCQueue("worker-transfer-queue", xferQueueSz, 0, xferBatchSz, backPressureWaitStrategy, workerState.getTopologyId(), Constants.SYSTEM_COMPONENT_ID, -1, workerState.getPort()); }
private static JCQueue createQueue(String name, int queueSize) { return new JCQueue(name, queueSize, 0, 1, new WaitStrategyPark(0), "test", "test", 1000, 1000); }