DLMTestUtil.completedLogSegment("/segment" + i, i, i, i * 100L, 100, i, 99L, 0L); String name = DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i); cache.add(name, metadata); DLMTestUtil.inprogressLogSegment("/inprogress-6", 6, 600L, 6); String name = DLMTestUtil.inprogressZNodeName(6); cache.add(name, inprogress); segmentRemoved.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i)); segmentRemoved.add((DLMTestUtil.inprogressZNodeName(6))); Set<String> segmentReceived = Sets.newHashSet(); Set<String> segmentAdded = Sets.newHashSet(); for (int i = 3; i <= 6; i++) { segmentReceived.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i)); if (i == 6) { segmentAdded.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
List<String> segments = new ArrayList<String>(); for (int i = 1; i <= 5; i++) { segments.add(DLMTestUtil.completedLedgerZNodeNameWithVersion(i, (i - 1) * 100, i * 100 - 1, i)); String segmentName = DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i); segments.add(segmentName); if (i == 10) { String segmentName = DLMTestUtil.completedLedgerZNodeNameWithTxID((i - 1) * 100, i * 100 - 1); segments.add(segmentName); expectedFilteredSegments.add(segmentName);
public URI createDLMURI(String path) throws Exception { return DLMTestUtil.createDLMURI(zkPort, path); }
@BeforeClass public static void setupDefaults() throws Exception { defaultUri = DLMTestUtil.createDLMURI(zkPort, defaultPath); DistributedLogManager dlm = DLMTestUtil.createNewDLM("DefaultStream", conf, defaultUri); bindStream(defaultUri, defaultLedgerPath, defaultHost); DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 8192); dlm.close(); }
@Test(timeout = 60000, expected = UnexpectedException.class) public void testGapDetection() throws Exception { LogSegmentCache cache = new LogSegmentCache("test-gap-detection"); cache.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(1L), DLMTestUtil.completedLogSegment("/segment-1", 1L, 1L, 100L, 100, 1L, 99L, 0L)); cache.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(3L), DLMTestUtil.completedLogSegment("/segment-3", 3L, 3L, 300L, 100, 3L, 99L, 0L)); cache.getLogSegments(LogSegmentMetadata.COMPARATOR); }
public static long generateLogSegmentNonPartitioned(DistributedLogManager dlm, int controlEntries, int userEntries, long startTxid) throws Exception { return generateLogSegmentNonPartitioned(dlm, controlEntries, userEntries, startTxid, 1L); }
@Test(timeout = 60000) public void testToolCreateZkAclId() throws Exception { createStream(defaultUri, "0", "CreateAclStream", defaultPrivilegedZkAclId); try { DistributedLogManager dlm = DLMTestUtil.createNewDLM("0CreateAclStream", conf, defaultUri); DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 1000); dlm.close(); } catch (ZKException ex) { assertEquals(KeeperException.Code.NOAUTH, ex.getKeeperExceptionCode()); } }
public static LogSegmentMetadata completedLogSegment(String ledgerPath, long ledgerId, long firstTxId, long lastTxId, int recordCount, long logSegmentSeqNo, long lastEntryId, long lastSlotId, int version) { LogSegmentMetadata metadata = new LogSegmentMetadata.LogSegmentMetadataBuilder( ledgerPath + "/" + inprogressZNodeName(logSegmentSeqNo), version, ledgerId, firstTxId) .setInprogress(false) .setLogSegmentSequenceNo(logSegmentSeqNo) .build(); return metadata.completeLogSegment(ledgerPath + "/" + completedLedgerZNodeNameWithLogSegmentSequenceNumber(logSegmentSeqNo), lastTxId, recordCount, lastEntryId, lastSlotId, firstTxId); }
DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 3, 10); LogRecordWithDLSN record = Await.result(reader.readNext()); assertEquals(expectedID++, record.getTransactionId()); DLMTestUtil.verifyLogRecord(record); ++numReads; writer.write(DLMTestUtil.getLogRecordInstance(31L)); writer.closeAndComplete(); DLMTestUtil.verifyLogRecord(record);
DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 4, 10); DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 5, 41, false, 10, true); dlm.close(); LogRecord record = Await.result(reader.readNext()); assertNotNull(record); DLMTestUtil.verifyLogRecord(record); assertEquals(expectedTxId, record.getTransactionId()); expectedTxId++; DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 3L, 5 * 10 + 1, true, 10, false); LogRecord record = Await.result(readFuture); assertNotNull(record); DLMTestUtil.verifyLogRecord(record); assertEquals(expectedTxId, record.getTransactionId()); expectedTxId++; record = Await.result(reader.readNext()); assertNotNull(record); DLMTestUtil.verifyLogRecord(record); assertEquals(expectedTxId, record.getTransactionId()); expectedTxId++;
zkc.get().create(ledgerPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); LogSegmentMetadata completedLogSegment = DLMTestUtil.completedLogSegment(ledgerPath, 1L, 0L, 99L, 100, 1L, 99L, 0L); completedLogSegment.write(zkc); LogSegmentMetadata inprogressLogSegment = DLMTestUtil.inprogressLogSegment(ledgerPath, 2L, 100L, 2L); inprogressLogSegment.write(zkc); DLSN goodLastDLSN2 = new DLSN(2L, 200L, 0L); LogRecordWithDLSN badRecord = DLMTestUtil.getLogRecordWithDLSNInstance(badLastDLSN, 100L); LogRecordWithDLSN goodRecord1 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN1, 100L); LogRecordWithDLSN goodRecord2 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN2, 200L);
@Test(timeout = 60000) public void testGetLogRecordCountWithCompletedAndInprogressLedgers() throws Exception { String streamName = runtime.getMethodName(); BKDistributedLogManager bkdlm = (BKDistributedLogManager) createNewDLM(conf, streamName); long txid = 1; txid += DLMTestUtil.generateLogSegmentNonPartitioned(bkdlm, 0, 5, txid); AsyncLogWriter out = bkdlm.startAsyncLogSegmentNonPartitioned(); Await.result(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false))); Await.result(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false))); Await.result(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false))); BKLogReadHandler readHandler = bkdlm.createReadHandler(); List<LogSegmentMetadata> ledgerList = readHandler.getLedgerList(false, false, LogSegmentMetadata.COMPARATOR, false); assertEquals(2, ledgerList.size()); assertFalse(ledgerList.get(0).isInProgress()); assertTrue(ledgerList.get(1).isInProgress()); Future<Long> count = null; count = readHandler.asyncGetLogRecordCount(new DLSN(1, 0, 0)); assertEquals(7, Await.result(count).longValue()); Utils.close(out); }
LogSegmentMetadata segment = DLMTestUtil.completedLogSegment(ledgerPath, i, (i - 1) * 100, i * 100 - 1, 100, i, 100, 0); completedLogSegments.put(((long)i), segment); LOG.info("Create completed segment {} : {}", segment.getZkPath(), segment); LogSegmentMetadata segment = DLMTestUtil.inprogressLogSegment(ledgerPath, inprogressSeqNo, 5 * 100, inprogressSeqNo); LOG.info("Create inprogress segment {} : {}", segment.getZkPath(), segment); segment.write(zkc);
DLMTestUtil.generateCompletedLogSegments(dlm, conf, numCompletedSegments, DEFAULT_SEGMENT_SIZE); long lastTxId = -1; while (null != record) { DLMTestUtil.verifyLogRecord(record); assert (lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId();
private static LogSegmentMetadata completedLogSegment( long logSegmentSequenceNumber, long fromTxnId, long toTxnId, int version) { return DLMTestUtil.completedLogSegment( "/logsegment/" + fromTxnId, fromTxnId, fromTxnId, toTxnId, 100, logSegmentSequenceNumber, 999L, 0L, version); }
@Test(timeout = 60000) public void testGetLogRecordCountAsync() throws Exception { DistributedLogManager dlm = createNewDLM(conf, testNames.getMethodName()); BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned(); DLMTestUtil.generateCompletedLogSegments(dlm, conf, 2, 10); Future<Long> futureCount = dlm.getLogRecordCountAsync(DLSN.InitialDLSN); Long count = Await.result(futureCount, Duration.fromSeconds(2)); assertEquals(20, count.longValue()); writer.close(); dlm.close(); }
public DLMTestUtil.BKLogPartitionWriteHandlerAndClients createNewBKDLM( DistributedLogConfiguration conf, String path) throws Exception { return DLMTestUtil.createNewBKDLM(conf, path, zkPort); }
public static void fenceStream(DistributedLogConfiguration conf, URI uri, String name) throws Exception { BKDistributedLogManager dlm = (BKDistributedLogManager) createNewDLM(name, conf, uri); try { BKLogReadHandler readHandler = dlm.createReadHandler(); List<LogSegmentMetadata> ledgerList = readHandler.getFullLedgerList(true, true); LogSegmentMetadata lastSegment = ledgerList.get(ledgerList.size() - 1); BookKeeperClient bkc = dlm.getWriterBKC(); LedgerHandle lh = bkc.get().openLedger(lastSegment.getLedgerId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8)); lh.close(); } finally { dlm.close(); } }
@Test(timeout = 60000) public void testMetadataAccessor() throws Exception { String name = "distrlog-metadata-accessor"; MetadataAccessor metadata = DLMTestUtil.createNewMetadataAccessor(conf, name, createDLMURI("/" + name)); assertEquals(name, metadata.getStreamName()); metadata.createOrUpdateMetadata(name.getBytes()); assertEquals(name, new String(metadata.getMetadata())); metadata.deleteMetadata(); assertEquals(null, metadata.getMetadata()); }
@Test(timeout = 60000) public void testGetLogRecordCountWithAllControlRecords() throws Exception { DistributedLogManager dlm = createNewDLM(conf, runtime.getMethodName()); long txid = 1; txid += DLMTestUtil.generateLogSegmentNonPartitioned(dlm, 5, 0, txid); txid += DLMTestUtil.generateLogSegmentNonPartitioned(dlm, 10, 0, txid); BKLogReadHandler readHandler = ((BKDistributedLogManager) dlm).createReadHandler(); Future<Long> count = null; count = readHandler.asyncGetLogRecordCount(new DLSN(1, 0, 0)); assertEquals(0, Await.result(count).longValue()); }