private void testAddHelper(int addedN, int bucketSize, int recordCount, int expectedN) { LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); List<LogRecord> expectedList = new LinkedList<>(); LogRecord record = new LogRecord(); while (addedN-- > 0) { storage.addLogRecord(record); } while (expectedN-- > 0) { expectedList.add(record); } LogBucket group = storage.getNextBucket(); List<LogRecord> actualList = group.getRecords(); Assert.assertTrue("Expected: " + expectedList.size() + ", actual: " + actualList.size() , expectedList.size() == actualList.size()); Iterator<LogRecord> expectedIt = expectedList.iterator(); Iterator<LogRecord> actualIt = actualList.iterator(); while (expectedIt.hasNext()) { LogRecord expected = expectedIt.next(); LogRecord actual = actualIt.next(); Assert.assertTrue(expected.getSize() == actual.getSize()); Assert.assertArrayEquals(expected.getData(), actual.getData()); } storage.close(); }
long leftRecordCount = maxRecordCount - currentRecordCount; if (leftConsumedSize < record.getSize() || leftRecordCount == 0) { moveToNextBucket(); insertStatement.setBytes(2, record.getData()); int affectedRows = insertStatement.executeUpdate(); if (affectedRows == 1) { currentBucketSize += record.getSize(); currentRecordCount++; unmarkedConsumedSize += record.getSize(); unmarkedRecordCount++; totalRecordCount++; LOG.trace("Added a new log record, total record count: {}, data: {}, " + "unmarked record count: {}", totalRecordCount, record.getData(), unmarkedRecordCount); } else { LOG.warn("No log record was added");
@Override public void run() { try { BucketInfo bucketInfo = storage.addLogRecord(new LogRecord(record)); bucketInfoMap.put(bucketInfo.getBucketId(), bucketInfo); addDeliveryFuture(bucketInfo, future); } catch (IOException ex) { LOG.warn("Can't serialize log record {}, exception catched: {}", record, ex); } uploadIfNeeded(); } });
public void addLogRecordTestHelper(int maxSize, int maxRecordCount) { MemBucket bucket = new MemBucket(1, maxSize, maxRecordCount); int curSize = 0; int curRecordCount = 0; LogRecord record = new LogRecord(); while (curSize + record.getSize() <= maxSize && curRecordCount < maxRecordCount) { Assert.assertTrue(bucket.addRecord(record)); curRecordCount++; curSize += 3; } Assert.assertFalse(bucket.addRecord(record)); }
/** * Adds a record to a bucket. * * @param record record * @return true if record is added otherwise false */ public boolean addRecord(LogRecord record) { if (size + record.getSize() > maxSize) { LOG.trace("No space left in bucket. Current size: {}, record size: {}, max size: {}", size, record.getSize(), maxSize); return false; } if (getCount() + 1 > maxRecordCount) { LOG.trace("No space left in bucket. Current count: {}, max count: {}", getCount(), maxRecordCount); return false; } records.add(record); size += record.getSize(); return true; }
logs.add(new LogEntry(ByteBuffer.wrap(record.getData())));
private void addNRecordsToBucket(MemBucket bucket, int n) { while (n-- > 0) { bucket.addRecord(new LogRecord()); } } }
@Test public void testRecordCountAndConsumedBytes() { long bucketSize = 3; int recordCount = 3; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord(); int insertionCount = 3; /* * Size of each record is 3B */ int iter = insertionCount; while (iter-- > 0) { storage.addLogRecord(record); } Assert.assertTrue(storage.getStatus().getRecordCount() == insertionCount); Assert.assertTrue(storage.getStatus().getConsumedVolume() == (insertionCount * record.getSize())); storage.close(); }
long leftRecordCount = maxRecordCount - currentRecordCount; if (leftConsumedSize < record.getSize() || leftRecordCount == 0) { moveToNextBucket(); insertStatement.bindBlob(2, record.getData()); long insertedId = insertStatement.executeInsert(); if (insertedId >= 0) { currentBucketSize += record.getSize(); currentRecordCount++; unmarkedConsumedSize += record.getSize(); totalRecordCount++; unmarkedRecordCount++; Log.i(TAG, "Added a new log record, total record count: " + totalRecordCount + ", data: " + Arrays.toString(record.getData()) + "unmarked record count: " + unmarkedRecordCount); } else {
/** * Shrinks current bucket to the newSize. * * @param newSize expected max size of a bucket inclusively * @param newCount the new count * @return removed from the bucket records */ public List<LogRecord> shrinkToSize(long newSize, int newCount) { LOG.trace("Shrinking {} bucket to the new size: [{}] and count [{}]", this, newSize, newCount); if (newSize < 0 || newCount < 0) { throw new IllegalArgumentException("New size and count values must be non-negative"); } if (newSize >= size && newCount >= getCount()) { return Collections.emptyList(); } List<LogRecord> overSize = new ArrayList<>(); int lastIndex = records.size() - 1; while ((size > newSize || getCount() > newCount) && lastIndex > 0) { LogRecord curRecord = records.remove(lastIndex--); overSize.add(curRecord); size -= curRecord.getSize(); } LOG.trace("Shrink over-sized elements: [{}]. New bucket size: [{}] and count [{}]", overSize.size(), size, getCount()); return overSize; }
@Test public void testGetSameLogBlock() { long bucketSize = 3; int recordCount = 3; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord(); int insertionCount = 3; /* * Size of each record is 3B */ int iter = insertionCount; while (iter-- > 0) { storage.addLogRecord(record); } LogBucket group1 = storage.getNextBucket(); storage.rollbackBucket(group1.getBucketId()); LogBucket group2 = storage.getNextBucket(); Assert.assertTrue("Expected: " + group1.getRecords().size() + ", actual: " + group2.getRecords().size() , group1.getRecords().size() == group2.getRecords().size()); Iterator<LogRecord> expectedIt = group1.getRecords().iterator(); Iterator<LogRecord> actualIt = group2.getRecords().iterator(); while (expectedIt.hasNext()) { LogRecord expected = expectedIt.next(); LogRecord actual = actualIt.next(); Assert.assertTrue(expected.getSize() == actual.getSize()); Assert.assertArrayEquals(expected.getData(), actual.getData()); } storage.close(); }
break; logRecords.add(new LogRecord(recordData)); leftBlockSize -= recordData.length; } else {
@Override public BucketInfo addLogRecord(LogRecord record) { LOG.trace("Adding new log record with size {}", record.getSize()); if (record.getSize() > maxBucketSize) { throw new IllegalArgumentException("Record size(" + record.getSize() + ") is bigger than max bucket size (" + maxBucketSize + ")!"); } synchronized (buckets) { if (consumedVolume + record.getSize() > maxStorageSize) { throw new IllegalStateException("Storage is full!"); } if (currentBucket == null || currentBucket.getState() != MemBucketState.FREE) { currentBucket = new MemBucket(bucketIdSeq.getAndIncrement(), maxBucketSize, maxBucketRecordCount); buckets.put(currentBucket.getId(), currentBucket); } if (!currentBucket.addRecord(record)) { LOG.trace("Current bucket is full. Creating new one."); currentBucket.setState(MemBucketState.FULL); currentBucket = new MemBucket(bucketIdSeq.getAndIncrement(), maxBucketSize, maxBucketRecordCount); buckets.put(currentBucket.getId(), currentBucket); currentBucket.addRecord(record); } recordCount++; consumedVolume += record.getSize(); } LOG.trace("Added a new log record to bucket [{}]", currentBucket.getId()); return new BucketInfo(currentBucket.getId(), currentBucket.getCount()); }
byte[] recordData = cursor.getBlob(0); logRecords.add(new LogRecord(recordData)); leftBucketSize -= recordData.length;
@Test public void testGetBigRecordBlock() { long bucketSize = 8192; int recordCount = 1000; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord(); int insertionCount = 7; /* * Size of each record is 3B */ int iter = insertionCount; while (iter-- > 0) { storage.addLogRecord(record); } LogBucket logBlock = storage.getNextBucket(); Assert.assertEquals(insertionCount, logBlock.getRecords().size()); storage.close(); }
logRecords.add(new LogRecord()); LogBucket logBlock = new LogBucket(defaultId, logRecords); Mockito.when(storage.getNextBucket()).thenReturn(logBlock);
@Test public void testLogRecordRemoval() { long bucketSize = 9; int recordCount = 3; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord(); int insertionCount = 7; /* * Size of each record is 3B */ int iter = insertionCount; while (iter-- > 0) { storage.addLogRecord(record); } LogBucket removingBlock = storage.getNextBucket(); insertionCount -= removingBlock.getRecords().size(); storage.removeBucket(removingBlock.getBucketId()); removingBlock = storage.getNextBucket(); insertionCount -= removingBlock.getRecords().size(); storage.removeBucket(removingBlock.getBucketId()); LogBucket leftBlock = storage.getNextBucket(); Assert.assertTrue(leftBlock.getRecords().size() == insertionCount); storage.close(); }
int recordCount = 3; LogStorage storage = (LogStorage) getStorage(bucketSize, recordCount); LogRecord record = new LogRecord();