@Override public void run() { try { BucketInfo bucketInfo = storage.addLogRecord(new LogRecord(record)); bucketInfoMap.put(bucketInfo.getBucketId(), bucketInfo); addDeliveryFuture(bucketInfo, future); } catch (IOException ex) { LOG.warn("Can't serialize log record {}, exception catched: {}", record, ex); } uploadIfNeeded(); } });
@Test public void testEmptyLogRecord() { long bucketSize = 3; int recordCount = 3; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogBucket group = storage.getNextBucket(); Assert.assertTrue(group == null); storage.close(); }
protected void uploadIfNeeded() { processUploadDecision(strategy.isUploadNeeded(storage.getStatus())); }
@Test public void testLogRecordRemoval() { long bucketSize = 9; int recordCount = 3; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord(); int insertionCount = 7; /* * Size of each record is 3B */ int iter = insertionCount; while (iter-- > 0) { storage.addLogRecord(record); } LogBucket removingBlock = storage.getNextBucket(); insertionCount -= removingBlock.getRecords().size(); storage.removeBucket(removingBlock.getBucketId()); removingBlock = storage.getNextBucket(); insertionCount -= removingBlock.getRecords().size(); storage.removeBucket(removingBlock.getBucketId()); LogBucket leftBlock = storage.getNextBucket(); Assert.assertTrue(leftBlock.getRecords().size() == insertionCount); storage.close(); }
@Test public void testUniqueIdGeneration() { long bucketSize = 3; int recordCount = 3; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord(); int insertionCount = 3; /* * Size of each record is 3B */ int iter = insertionCount; while (iter-- > 0) { storage.addLogRecord(record); } LogBucket group1 = storage.getNextBucket(); LogBucket group2 = storage.getNextBucket(); Assert.assertNotEquals(group1.getBucketId(), group2.getBucketId()); storage.close(); }
logCollector.setStorage(storage); Mockito.when(storage.getStatus()).thenReturn(new LogStorageStatus() { Mockito.when(storage.addLogRecord(Mockito.any(LogRecord.class))).thenReturn(bucketInfo); Mockito.when(storage.getNextBucket()).thenReturn(logBlock);
bucketInfoMap.remove(requestId); if (response.getResult() == SyncResponseResultType.SUCCESS) { storage.removeBucket(response.getRequestId()); storage.rollbackBucket(response.getRequestId()); processUploadDecision(strategy.isUploadNeeded(storage.getStatus()));
@Test public void testRecordCountAndConsumedBytes() { long bucketSize = 3; int recordCount = 3; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord(); int insertionCount = 3; /* * Size of each record is 3B */ int iter = insertionCount; while (iter-- > 0) { storage.addLogRecord(record); } Assert.assertTrue(storage.getStatus().getRecordCount() == insertionCount); Assert.assertTrue(storage.getStatus().getConsumedVolume() == (insertionCount * record.getSize())); storage.close(); }
Log record = new Log(); Mockito.when(storage.addLogRecord(Mockito.any(LogRecord.class))).thenReturn(new BucketInfo(1, 1)); Mockito.when(storage.getStatus()).thenReturn(new LogStorageStatus() { Mockito.when(storage.getStatus()).thenReturn(new LogStorageStatus() {
@Override public void stop() { LOG.debug("Closing storage"); storage.close(); LOG.debug("Clearing timeouts map"); for (Future<?> timeoutFuture : timeouts.values()) { timeoutFuture.cancel(true); } timeouts.clear(); }
private void checkDeliveryTimeout(final int bucketId) { LOG.debug("Checking for a delivery timeout of the bucket with id: [{}] ", bucketId); Future<?> timeoutFuture = timeouts.remove(bucketId); if (timeoutFuture != null) { LOG.info("Log delivery timeout detected for the bucket with id: [{}]", bucketId); storage.rollbackBucket(bucketId); final LogFailoverCommand controller = this.controller; executorContext.getCallbackExecutor().execute(new Runnable() { @Override public void run() { strategy.onTimeout(controller); } }); if (logDeliveryListener != null) { executorContext.getCallbackExecutor().execute(new Runnable() { @Override public void run() { logDeliveryListener.onLogDeliveryTimeout(bucketInfoMap.get(bucketId)); } }); } timeoutFuture.cancel(true); } else { LOG.trace("No log delivery timeout for the bucket with id [{}] was detected", bucketId); } }
LogBucket bucket = storage.getNextBucket();
storage.addLogRecord(record); LogBucket removingBlock1 = storage.getNextBucket(); insertionCount -= removingBlock1.getRecords().size(); LogBucket removingBlock2 = storage.getNextBucket(); insertionCount -= removingBlock2.getRecords().size(); LogBucket removingBlock3 = storage.getNextBucket(); insertionCount -= removingBlock3.getRecords().size(); storage.removeBucket(removingBlock2.getBucketId()); storage.rollbackBucket(removingBlock1.getBucketId()); insertionCount += removingBlock1.getRecords().size(); LogBucket leftBlock1 = storage.getNextBucket(); LogBucket leftBlock2 = storage.getNextBucket(); int leftSize = leftBlock1.getRecords().size(); if (leftBlock2 != null) { storage.close();
@Test public void testGetBigRecordBlock() { long bucketSize = 8192; int recordCount = 1000; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord(); int insertionCount = 7; /* * Size of each record is 3B */ int iter = insertionCount; while (iter-- > 0) { storage.addLogRecord(record); } LogBucket logBlock = storage.getNextBucket(); Assert.assertEquals(insertionCount, logBlock.getRecords().size()); storage.close(); }
Mockito.when(storage.addLogRecord(Mockito.any(LogRecord.class))).thenReturn(new BucketInfo(1, 1)); Mockito.when(storage.getStatus()).thenReturn(new LogStorageStatus() { @Override public long getRecordCount() { logCollector.addLogRecord(record); logCollector.addLogRecord(record); Mockito.when(storage.getStatus()).thenReturn(new LogStorageStatus() { logCollector.addLogRecord(record); Mockito.when(storage.getNextBucket()).thenReturn( new LogBucket(1, Arrays.asList(new LogRecord(record), new LogRecord(record), new LogRecord(record))));
storage.addLogRecord(record); LogBucket logBlock = storage.getNextBucket(); receivedCount = addIfNotEmpty(receivedCount, logBlock); Assert.assertEquals(insertionCount - receivedCount, storage.getStatus().getRecordCount()); Assert.assertEquals((insertionCount - receivedCount) * 3, storage.getStatus().getConsumedVolume()); logBlock = storage.getNextBucket(); receivedCount = addIfNotEmpty(receivedCount, logBlock); Assert.assertEquals(insertionCount - receivedCount, storage.getStatus().getRecordCount()); Assert.assertEquals((insertionCount - receivedCount) * 3, storage.getStatus().getConsumedVolume()); logBlock = storage.getNextBucket(); receivedCount = addIfNotEmpty(receivedCount, logBlock); Assert.assertEquals(insertionCount - receivedCount, storage.getStatus().getRecordCount()); Assert.assertEquals((insertionCount - receivedCount) * 3, storage.getStatus().getConsumedVolume()); storage.rollbackBucket(logBlock.getBucketId()); receivedCount -= logBlock.getRecords().size(); Assert.assertEquals(insertionCount - receivedCount, storage.getStatus().getRecordCount()); Assert.assertEquals((insertionCount - receivedCount) * 3, storage.getStatus().getConsumedVolume()); storage.close();
private void testAddHelper(int addedN, int bucketSize, int recordCount, int expectedN) { LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); List<LogRecord> expectedList = new LinkedList<>(); LogRecord record = new LogRecord(); while (addedN-- > 0) { storage.addLogRecord(record); } while (expectedN-- > 0) { expectedList.add(record); } LogBucket group = storage.getNextBucket(); List<LogRecord> actualList = group.getRecords(); Assert.assertTrue("Expected: " + expectedList.size() + ", actual: " + actualList.size() , expectedList.size() == actualList.size()); Iterator<LogRecord> expectedIt = expectedList.iterator(); Iterator<LogRecord> actualIt = actualList.iterator(); while (expectedIt.hasNext()) { LogRecord expected = expectedIt.next(); LogRecord actual = actualIt.next(); Assert.assertTrue(expected.getSize() == actual.getSize()); Assert.assertArrayEquals(expected.getData(), actual.getData()); } storage.close(); }
Mockito.when(storage.addLogRecord(Mockito.any(LogRecord.class))).thenReturn(new BucketInfo(1, 1)); Mockito.when(storage.getStatus()).thenReturn(new LogStorageStatus() { @Override public long getRecordCount() { logCollector.addLogRecord(record); logCollector.addLogRecord(record); Mockito.when(storage.addLogRecord(Mockito.any(LogRecord.class))).thenReturn(new BucketInfo(1, 1)); Mockito.when(storage.getStatus()).thenReturn(new LogStorageStatus() { logCollector.addLogRecord(record); Mockito.when(storage.getNextBucket()).thenReturn( new LogBucket(1, Arrays.asList(new LogRecord(record), new LogRecord(record), new LogRecord(record))));
private void processUploadDecision(LogUploadStrategyDecision decision) { switch (decision) { case UPLOAD: if (isUploadAllowed()) { LOG.debug("Going to upload logs"); transport.sync(); } break; case NOOP: if (strategy.getUploadCheckPeriod() > 0 && storage.getStatus().getRecordCount() > 0) { scheduleUploadCheck(); } break; default: break; } }
@Test public void testGetSameLogBlock() { long bucketSize = 3; int recordCount = 3; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord(); int insertionCount = 3; /* * Size of each record is 3B */ int iter = insertionCount; while (iter-- > 0) { storage.addLogRecord(record); } LogBucket group1 = storage.getNextBucket(); storage.rollbackBucket(group1.getBucketId()); LogBucket group2 = storage.getNextBucket(); Assert.assertTrue("Expected: " + group1.getRecords().size() + ", actual: " + group2.getRecords().size() , group1.getRecords().size() == group2.getRecords().size()); Iterator<LogRecord> expectedIt = group1.getRecords().iterator(); Iterator<LogRecord> actualIt = group2.getRecords().iterator(); while (expectedIt.hasNext()) { LogRecord expected = expectedIt.next(); LogRecord actual = actualIt.next(); Assert.assertTrue(expected.getSize() == actual.getSize()); Assert.assertArrayEquals(expected.getData(), actual.getData()); } storage.close(); }