@Override public Set<DataSegment> announceHistoricalSegments(Set<DataSegment> segments) throws IOException { Set<DataSegment> result = super.announceHistoricalSegments(segments); Assert.assertFalse( "Segment latch not initialized, did you forget to call expectPublishSegments?", segmentLatch == null ); publishedSegments.addAll(result); segments.forEach(s -> segmentLatch.countDown()); return result; }
/** * Attempts to insert a set of segments to the database. Returns the set of segments actually added (segments * with identifiers already in the database will not be added). * * @param segments set of segments to add * * @return set of segments actually added */ @Override public Set<DataSegment> announceHistoricalSegments(final Set<DataSegment> segments) throws IOException { final SegmentPublishResult result = announceHistoricalSegments(segments, null, null); // Metadata transaction cannot fail because we are not trying to do one. if (!result.isSuccess()) { throw new ISE("WTF?! announceHistoricalSegments failed with null metadata, should not happen."); } return result.getSegments(); }
@Override public SegmentPublishResult announceHistoricalSegments( Set<DataSegment> segments, DataSourceMetadata startMetadata, DataSourceMetadata endMetadata ) throws IOException { SegmentPublishResult result = super.announceHistoricalSegments(segments, startMetadata, endMetadata); Assert.assertFalse( "Segment latch not initialized, did you forget to call expectPublishSegments?", segmentLatch == null ); publishedSegments.addAll(result.getSegments()); result.getSegments().forEach(s -> segmentLatch.countDown()); return result; } };
@Test public void testTransactionalAnnounceFailDbNullWantNotNull() throws IOException { final SegmentPublishResult result1 = coordinator.announceHistoricalSegments( ImmutableSet.of(defaultSegment), new ObjectMetadata(ImmutableMap.of("foo", "bar")), new ObjectMetadata(ImmutableMap.of("foo", "baz")) ); Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(), false), result1); // Should only be tried once. Assert.assertEquals(1, metadataUpdateCounter.get()); }
@Test public void testTransactionalAnnounceFailDbNotNullWantDifferent() throws IOException { final SegmentPublishResult result1 = coordinator.announceHistoricalSegments( ImmutableSet.of(defaultSegment), new ObjectMetadata(null), new ObjectMetadata(ImmutableMap.of("foo", "baz")) ); Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(defaultSegment), true), result1); final SegmentPublishResult result2 = coordinator.announceHistoricalSegments( ImmutableSet.of(defaultSegment2), new ObjectMetadata(ImmutableMap.of("foo", "qux")), new ObjectMetadata(ImmutableMap.of("foo", "baz")) ); Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(), false), result2); // Should only be tried once per call. Assert.assertEquals(2, metadataUpdateCounter.get()); }
@Test public void testUsedOverlapHigh() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), Intervals.of("2015-1-1T23:59:59.999Z/2015-02-01T00Z") ) ) ); }
@Test public void testSimpleUsedList() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), defaultSegment.getInterval() ) ) ); }
@Test public void testUsedOverlapLow() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Set<DataSegment> actualSegments = ImmutableSet.copyOf( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), Intervals.of("2014-12-31T23:59:59.999Z/2015-01-01T00:00:00.001Z") // end is exclusive ) ); Assert.assertEquals( SEGMENTS, actualSegments ); }
@Test public void testOvershadowingAnnounce() throws IOException { final ImmutableSet<DataSegment> segments = ImmutableSet.of(defaultSegment, defaultSegment2, defaultSegment4); coordinator.announceHistoricalSegments(segments); for (DataSegment segment : segments) { Assert.assertArrayEquals( mapper.writeValueAsString(segment).getBytes(StandardCharsets.UTF_8), derbyConnector.lookup( derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), "id", "payload", segment.getId().toString() ) ); } Assert.assertEquals(ImmutableList.of(defaultSegment4.getId().toString()), getUsedSegmentIds()); }
@Test public void testSimpleUnUsedList() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); unUseSegment(); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUnusedSegmentsForInterval( defaultSegment.getDataSource(), defaultSegment.getInterval() ) ) ); }
@Test public void testUnUsedBigOverlap() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); unUseSegment(); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUnusedSegmentsForInterval( defaultSegment.getDataSource(), Intervals.of("2000/2999") ) ) ); }
@Test public void testUsedOutOfBoundsHigh() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertTrue( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), new Interval(defaultSegment.getInterval().getEnd(), defaultSegment.getInterval().getEnd().plusDays(10)) ).isEmpty() ); }
@Test public void testUnUsedUnderlapLow() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); unUseSegment(); Assert.assertTrue( coordinator.getUnusedSegmentsForInterval( defaultSegment.getDataSource(), new Interval(defaultSegment.getInterval().getStart().plus(1), defaultSegment.getInterval().getEnd()) ).isEmpty() ); }
@Test public void testDeleteDataSourceMetadata() throws IOException { coordinator.announceHistoricalSegments( ImmutableSet.of(defaultSegment), new ObjectMetadata(null), new ObjectMetadata(ImmutableMap.of("foo", "bar")) ); Assert.assertEquals( new ObjectMetadata(ImmutableMap.of("foo", "bar")), coordinator.getDataSourceMetadata("fooDataSource") ); Assert.assertFalse("deleteInvalidDataSourceMetadata", coordinator.deleteDataSourceMetadata("nonExistentDS")); Assert.assertTrue("deleteValidDataSourceMetadata", coordinator.deleteDataSourceMetadata("fooDataSource")); Assert.assertNull("getDataSourceMetadataNullAfterDelete", coordinator.getDataSourceMetadata("fooDataSource")); }
@Test public void testUnUsedOverlapHigh() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); unUseSegment(); Assert.assertTrue( coordinator.getUnusedSegmentsForInterval( defaultSegment.getDataSource(), defaultSegment.getInterval().withStart(defaultSegment.getInterval().getEnd().minus(1)) ).isEmpty() ); }
@Test public void testUnUsedOverlapLow() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); unUseSegment(); Assert.assertTrue( coordinator.getUnusedSegmentsForInterval( defaultSegment.getDataSource(), new Interval( defaultSegment.getInterval().getStart().minus(1), defaultSegment.getInterval().getStart().plus(1) ) ).isEmpty() ); }
@Test public void testUsedOutOfBoundsLow() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertTrue( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), new Interval(defaultSegment.getInterval().getStart().minus(1), defaultSegment.getInterval().getStart()) ).isEmpty() ); }
@Test public void testUsedWithinBoundsEnd() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), defaultSegment.getInterval().withEnd(defaultSegment.getInterval().getEnd().minusMillis(1)) ) ) ); }
@Test public void testUsedOverlapEnd() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), defaultSegment.getInterval().withEnd(defaultSegment.getInterval().getEnd().plusMillis(1)) ) ) ); }
@Test public void testUnUsedUnderlapHigh() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); unUseSegment(); Assert.assertTrue( coordinator.getUnusedSegmentsForInterval( defaultSegment.getDataSource(), new Interval(defaultSegment.getInterval().getStart(), defaultSegment.getInterval().getEnd().minus(1)) ).isEmpty() ); }