final ListenableFuture<ChangeRequestsSnapshot<DataSegmentChangeRequest>> future = announcer.getSegmentChangesSince( new ChangeRequestHistory.Counter( counter,
@Test public void testRun() throws Exception { segmentAnnouncer.announceSegments(testSegments); waitForSync(batchServerInventoryView, testSegments); DruidServer server = Iterables.get(batchServerInventoryView.getInventory(), 0); Set<DataSegment> segments = Sets.newHashSet(server.getSegments()); Assert.assertEquals(testSegments, segments); DataSegment segment1 = makeSegment(101); DataSegment segment2 = makeSegment(102); segmentAnnouncer.announceSegment(segment1); segmentAnnouncer.announceSegment(segment2); testSegments.add(segment1); testSegments.add(segment2); waitForSync(batchServerInventoryView, testSegments); Assert.assertEquals(testSegments, Sets.newHashSet(server.getSegments())); segmentAnnouncer.unannounceSegment(segment1); segmentAnnouncer.unannounceSegment(segment2); testSegments.remove(segment1); testSegments.remove(segment2); waitForSync(batchServerInventoryView, testSegments); Assert.assertEquals(testSegments, Sets.newHashSet(server.getSegments())); }
@Override public BatchDataSegmentAnnouncer call() BatchDataSegmentAnnouncer segmentAnnouncer = new BatchDataSegmentAnnouncer( new DruidServerMetadata( "id", segmentAnnouncer.announceSegments(segments); testSegments.addAll(segments);
@Test public void testSkipLoadSpec() throws Exception { skipLoadSpec = true; Iterator<DataSegment> segIter = testSegments.iterator(); DataSegment firstSegment = segIter.next(); segmentAnnouncer.announceSegment(firstSegment); List<String> zNodes = cf.getChildren().forPath(testSegmentsPath); for (String zNode : zNodes) { DataSegment announcedSegment = Iterables.getOnlyElement(segmentReader.read(joiner.join(testSegmentsPath, zNode))); Assert.assertEquals(announcedSegment, firstSegment); Assert.assertNull(announcedSegment.getLoadSpec()); } segmentAnnouncer.unannounceSegment(firstSegment); Assert.assertTrue(cf.getChildren().forPath(testSegmentsPath).isEmpty()); }
DataSegment secondSegment = segIter.next(); segmentAnnouncer.announceSegment(firstSegment); segmentAnnouncer.announceSegment(secondSegment); ChangeRequestsSnapshot<DataSegmentChangeRequest> snapshot = segmentAnnouncer.getSegmentChangesSince( new ChangeRequestHistory.Counter(-1, -1) ).get(); Assert.assertEquals(2, snapshot.getCounter().getCounter()); segmentAnnouncer.unannounceSegment(firstSegment); segmentAnnouncer.unannounceSegment(secondSegment); snapshot = segmentAnnouncer.getSegmentChangesSince( snapshot.getCounter() ).get(); Assert.assertEquals(4, snapshot.getCounter().getCounter()); snapshot = segmentAnnouncer.getSegmentChangesSince( new ChangeRequestHistory.Counter(-1, -1) ).get();
private void testBatchAnnounce(boolean testHistory) throws Exception segmentAnnouncer.announceSegments(testSegments); snapshot = segmentAnnouncer.getSegmentChangesSince( new ChangeRequestHistory.Counter(-1, -1) ).get(); segmentAnnouncer.unannounceSegments(testSegments); snapshot = segmentAnnouncer.getSegmentChangesSince( snapshot.getCounter() ).get(); Assert.assertEquals(2 * testSegments.size(), snapshot.getCounter().getCounter()); snapshot = segmentAnnouncer.getSegmentChangesSince( new ChangeRequestHistory.Counter(-1, -1) ).get();
final ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numThreads)); segmentAnnouncer.announceSegments(testSegments); public BatchDataSegmentAnnouncer call() BatchDataSegmentAnnouncer segmentAnnouncer = new BatchDataSegmentAnnouncer( new DruidServerMetadata( "id", segmentAnnouncer.unannounceSegment(segment); testSegments.remove(segment);
@Override public void unannounceSegments(Iterable<DataSegment> segments) { for (DataSegment segment : segments) { unannounceSegment(segment); } }
@Test public void testRunWithFilter() throws Exception { segmentAnnouncer.announceSegments(testSegments); waitForSync(filteredBatchServerInventoryView, testSegments); DruidServer server = Iterables.get(filteredBatchServerInventoryView.getInventory(), 0); Set<DataSegment> segments = Sets.newHashSet(server.getSegments()); Assert.assertEquals(testSegments, segments); int prevUpdateCount = inventoryUpdateCounter.get(); // segment outside the range of default filter DataSegment segment1 = makeSegment(101); segmentAnnouncer.announceSegment(segment1); testSegments.add(segment1); waitForUpdateEvents(prevUpdateCount + 1); Assert.assertNull( Iterables.getOnlyElement(filteredBatchServerInventoryView.getInventory()) .getSegment(segment1.getId()) ); }
private String makeServedSegmentPath() { // server.getName() is already in the zk path return makeServedSegmentPath( UUIDUtils.generateUuid( server.getHost(), server.getType().toString(), server.getTier(), DateTimes.nowUtc().toString() ) ); }
skipDimensionsAndMetrics = false; skipLoadSpec = false; segmentAnnouncer = new BatchDataSegmentAnnouncer( new DruidServerMetadata( "id",
@Test public void testSingleAnnounceManyTimes() throws Exception { int prevMax = maxBytesPerNode.get(); maxBytesPerNode.set(2048); // each segment is about 348 bytes long and that makes 2048 / 348 = 5 segments included per node // so 100 segments makes 100 / 5 = 20 nodes try { for (DataSegment segment : testSegments) { segmentAnnouncer.announceSegment(segment); } } finally { maxBytesPerNode.set(prevMax); } List<String> zNodes = cf.getChildren().forPath(testSegmentsPath); Assert.assertEquals(20, zNodes.size()); Set<DataSegment> segments = Sets.newHashSet(testSegments); for (String zNode : zNodes) { for (DataSegment segment : segmentReader.read(joiner.join(testSegmentsPath, zNode))) { Assert.assertTrue("Invalid segment " + segment, segments.remove(segment)); } } Assert.assertTrue("Failed to find segments " + segments, segments.isEmpty()); }
@Test public void testSkipDimensions() throws Exception { skipDimensionsAndMetrics = true; Iterator<DataSegment> segIter = testSegments.iterator(); DataSegment firstSegment = segIter.next(); segmentAnnouncer.announceSegment(firstSegment); List<String> zNodes = cf.getChildren().forPath(testSegmentsPath); for (String zNode : zNodes) { DataSegment announcedSegment = Iterables.getOnlyElement(segmentReader.read(joiner.join(testSegmentsPath, zNode))); Assert.assertEquals(announcedSegment, firstSegment); Assert.assertTrue(announcedSegment.getDimensions().isEmpty()); Assert.assertTrue(announcedSegment.getMetrics().isEmpty()); } segmentAnnouncer.unannounceSegment(firstSegment); Assert.assertTrue(cf.getChildren().forPath(testSegmentsPath).isEmpty()); }
@Override public void unannounceSegments(Iterable<DataSegment> segments) { for (DataSegment segment : segments) { unannounceSegment(segment); } }
@Override public void announceSegments(Iterable<DataSegment> segments) throws IOException SegmentZNode segmentZNode = new SegmentZNode(makeServedSegmentPath()); Set<DataSegment> batch = new HashSet<>(); List<DataSegmentChangeRequest> changesBatch = new ArrayList<>(); announcer.announce(segmentZNode.getPath(), segmentZNode.getBytes()); segmentZNode = new SegmentZNode(makeServedSegmentPath()); batch = new HashSet<>(); count = 0;
serverAnnouncer.announce(); segmentAnnouncer = new BatchDataSegmentAnnouncer( serverMetadata, new BatchDataSegmentAnnouncerConfig()
segmentAnnouncer.announceSegments(testSegments); segmentAnnouncer.announceSegment(segment2); testSegments.add(segment2); segmentAnnouncer.announceSegment(oldSegment); testSegments.add(oldSegment); segmentAnnouncer.unannounceSegment(oldSegment); testSegments.remove(oldSegment); segmentAnnouncer.unannounceSegment(segment2); testSegments.remove(segment2);
final ListenableFuture<ChangeRequestsSnapshot<DataSegmentChangeRequest>> future = announcer.getSegmentChangesSince( new ChangeRequestHistory.Counter( counter,
SegmentZNode availableZNode = new SegmentZNode(makeServedSegmentPath()); availableZNode.addSegment(toAnnounce);
private String makeServedSegmentPath() { // server.getName() is already in the zk path return makeServedSegmentPath( UUIDUtils.generateUuid( server.getHost(), server.getType().toString(), server.getTier(), DateTimes.nowUtc().toString() ) ); }