@Test public void testJsonResponseWithoutSplitCommit() { SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSplitCommit(false).withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertNull(jsonNode.get("segmentLocation")); assertFalse(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertNull(jsonNode.get("controllerVipUrl")); }
@POST @Path(SegmentCompletionProtocol.MSG_TYPE_SEGMENT_UPLOAD) @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.MULTIPART_FORM_DATA) public String segmentUpload(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, FormDataMultiPart multiPart) { SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset); LOGGER.info("Processing segmentUpload:{}", requestParams.toString()); final String segmentLocation = uploadSegment(multiPart, instanceId, segmentName, true); if (segmentLocation == null) { return SegmentCompletionProtocol.RESP_FAILED.toJsonString(); } SegmentCompletionProtocol.Response.Params responseParams = new SegmentCompletionProtocol.Response.Params().withOffset(requestParams.getOffset()) .withSegmentLocation(segmentLocation) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.UPLOAD_SUCCESS); String response = new SegmentCompletionProtocol.Response(responseParams).toJsonString(); LOGGER.info("Response to segmentUpload:{}", response); return response; }
@Test public void testJsonNullSegmentLocationAndVip() { SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSplitCommit(false).withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertNull(jsonNode.get("segmentLocation")); assertFalse(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertNull(jsonNode.get("controllerVipUrl")); }
new SegmentCompletionProtocol.Response.Params() .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.HOLD). withOffset(firstOffset)); final SegmentCompletionProtocol.Response catchupResponse = new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params() .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.CATCH_UP).withOffset(catchupOffset)); final SegmentCompletionProtocol.Response holdResponse2 = new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params().withOffset(catchupOffset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.HOLD)); final SegmentCompletionProtocol.Response commitResponse = new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params().withOffset(catchupOffset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT));
@Test public void testCommitAfterHold() throws Exception { FakeLLRealtimeSegmentDataManager segmentDataManager = createFakeSegmentManager(); LLRealtimeSegmentDataManager.PartitionConsumer consumer = segmentDataManager.createPartitionConsumer(); final long endOffset = _startOffset + 500; // We should consume initially... segmentDataManager._consumeOffsets.add(endOffset); final SegmentCompletionProtocol.Response holdResponse = new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params().withOffset(endOffset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.HOLD)); final SegmentCompletionProtocol.Response commitResponse = new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params().withOffset(endOffset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT)); // And then never consume as long as we get a hold response, 100 times. segmentDataManager._responses.add(holdResponse); segmentDataManager._responses.add(commitResponse); consumer.run(); Assert.assertTrue(segmentDataManager._responses.isEmpty()); Assert.assertTrue(segmentDataManager._consumeOffsets.isEmpty()); Assert.assertTrue(segmentDataManager._buildSegmentCalled); Assert.assertFalse(segmentDataManager._buildAndReplaceCalled); Assert.assertFalse(segmentDataManager._downloadAndReplaceCalled); Assert.assertTrue(segmentDataManager._commitSegmentCalled); Assert .assertEquals(segmentDataManager._state.get(segmentDataManager), LLRealtimeSegmentDataManager.State.COMMITTED); }
@Test public void testHolding() throws Exception { FakeLLRealtimeSegmentDataManager segmentDataManager = createFakeSegmentManager(); LLRealtimeSegmentDataManager.PartitionConsumer consumer = segmentDataManager.createPartitionConsumer(); final long endOffset = _startOffset + 500; // We should consume initially... segmentDataManager._consumeOffsets.add(endOffset); final SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params() .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.HOLD).withOffset(endOffset)); // And then never consume as long as we get a hold response, 100 times. for (int i = 0; i < 100; i++) { segmentDataManager._responses.add(response); } consumer.run(); Assert.assertTrue(segmentDataManager._responses.isEmpty()); Assert.assertTrue(segmentDataManager._consumeOffsets.isEmpty()); Assert.assertFalse(segmentDataManager._buildAndReplaceCalled); Assert.assertFalse(segmentDataManager._buildSegmentCalled); Assert.assertFalse(segmentDataManager._commitSegmentCalled); Assert.assertFalse(segmentDataManager._downloadAndReplaceCalled); Assert.assertEquals(segmentDataManager._state.get(segmentDataManager), LLRealtimeSegmentDataManager.State.HOLDING); }
@Test public void testNotLeader() throws Exception { FakeLLRealtimeSegmentDataManager segmentDataManager = createFakeSegmentManager(); LLRealtimeSegmentDataManager.PartitionConsumer consumer = segmentDataManager.createPartitionConsumer(); final long endOffset = _startOffset + 500; // We should consume initially... segmentDataManager._consumeOffsets.add(endOffset); final SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params().withOffset(endOffset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER)); // And then never consume as long as we get a Not leader response, 100 times. for (int i = 0; i < 100; i++) { segmentDataManager._responses.add(response); } consumer.run(); Assert.assertTrue(segmentDataManager._responses.isEmpty()); Assert.assertTrue(segmentDataManager._consumeOffsets.isEmpty()); Assert.assertFalse(segmentDataManager._buildAndReplaceCalled); Assert.assertFalse(segmentDataManager._buildSegmentCalled); Assert.assertFalse(segmentDataManager._commitSegmentCalled); Assert.assertFalse(segmentDataManager._downloadAndReplaceCalled); Assert.assertEquals(segmentDataManager._state.get(segmentDataManager), LLRealtimeSegmentDataManager.State.HOLDING); }
@Test public void testRetained() throws Exception { FakeLLRealtimeSegmentDataManager segmentDataManager = createFakeSegmentManager(); LLRealtimeSegmentDataManager.PartitionConsumer consumer = segmentDataManager.createPartitionConsumer(); final long endOffset = _startOffset + 500; segmentDataManager._consumeOffsets.add(endOffset); SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params(); params.withOffset(endOffset).withStatus(SegmentCompletionProtocol.ControllerResponseStatus.KEEP); final SegmentCompletionProtocol.Response keepResponse = new SegmentCompletionProtocol.Response(params); segmentDataManager._responses.add(keepResponse); consumer.run(); Assert.assertTrue(segmentDataManager._responses.isEmpty()); Assert.assertTrue(segmentDataManager._consumeOffsets.isEmpty()); Assert.assertFalse(segmentDataManager._buildSegmentCalled); Assert.assertFalse(segmentDataManager._downloadAndReplaceCalled); Assert.assertTrue(segmentDataManager._buildAndReplaceCalled); Assert.assertFalse(segmentDataManager._commitSegmentCalled); Assert.assertEquals(segmentDataManager._state.get(segmentDataManager), LLRealtimeSegmentDataManager.State.RETAINED); }
@Test public void testDiscarded() throws Exception { FakeLLRealtimeSegmentDataManager segmentDataManager = createFakeSegmentManager(); LLRealtimeSegmentDataManager.PartitionConsumer consumer = segmentDataManager.createPartitionConsumer(); final long endOffset = _startOffset + 500; segmentDataManager._consumeOffsets.add(endOffset); final SegmentCompletionProtocol.Response discardResponse = new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params().withOffset(endOffset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.DISCARD)); segmentDataManager._responses.add(discardResponse); consumer.run(); Assert.assertTrue(segmentDataManager._responses.isEmpty()); Assert.assertTrue(segmentDataManager._consumeOffsets.isEmpty()); Assert.assertFalse(segmentDataManager._buildSegmentCalled); Assert.assertFalse(segmentDataManager._buildAndReplaceCalled); Assert.assertFalse(segmentDataManager._downloadAndReplaceCalled); Assert.assertFalse(segmentDataManager._commitSegmentCalled); Assert .assertEquals(segmentDataManager._state.get(segmentDataManager), LLRealtimeSegmentDataManager.State.DISCARDED); }
@Test public void testSegmentBuildException() throws Exception { FakeLLRealtimeSegmentDataManager segmentDataManager = createFakeSegmentManager(); LLRealtimeSegmentDataManager.PartitionConsumer consumer = segmentDataManager.createPartitionConsumer(); final long endOffset = _startOffset + 500; // We should consume initially... segmentDataManager._consumeOffsets.add(endOffset); final SegmentCompletionProtocol.Response commitResponse = new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params().withOffset(endOffset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT)); segmentDataManager._responses.add(commitResponse); segmentDataManager._failSegmentBuild = true; consumer.run(); Assert.assertTrue(segmentDataManager._buildSegmentCalled); Assert.assertEquals(segmentDataManager._state.get(segmentDataManager), LLRealtimeSegmentDataManager.State.ERROR); }
@Test public void testJsonResponseWithVipAndNullSegmentLocation() { // Should never happen because if split commit, should have both location and VIP, but testing deserialization regardless SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withControllerVipUrl(CONTROLLER_VIP_URL).withSplitCommit(false) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertNull(jsonNode.get("segmentLocation")); assertFalse(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertEquals(jsonNode.get("controllerVipUrl").asText(), CONTROLLER_VIP_URL); } }
@Test public void testJsonResponseWithSegmentLocationNullVip() { // Should never happen because if split commit, should have both location and VIP, but testing deserialization regardless SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSegmentLocation(SEGMENT_LOCATION).withSplitCommit(false) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertEquals(jsonNode.get("segmentLocation").asText(), SEGMENT_LOCATION); assertFalse(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertNull(jsonNode.get("controllerVipUrl")); }
@Test public void testJsonResponseWithAllParams() { // Test with all params SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSegmentLocation(SEGMENT_LOCATION).withSplitCommit(true).withControllerVipUrl(CONTROLLER_VIP_URL) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertEquals(jsonNode.get("segmentLocation").asText(), SEGMENT_LOCATION); assertTrue(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertEquals(jsonNode.get("controllerVipUrl").asText(), CONTROLLER_VIP_URL); }
@Test public void testCompleteResponseParams() { // Test with all params SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSegmentLocation(SEGMENT_LOCATION).withSplitCommit(true) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); assertEquals(response.getBuildTimeSeconds(), BUILD_TIME_MILLIS); assertEquals(response.getOffset(), OFFSET); assertEquals(response.getSegmentLocation(), SEGMENT_LOCATION); assertTrue(response.isSplitCommit()); assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); }
private SegmentCompletionProtocol.Response commit(String instanceId, long offset) { long allowedBuildTimeSec = (_maxTimeAllowedToCommitMs - _startTimeMs) / 1000; LOGGER .info("{}:COMMIT for instance={} offset={} buldTimeSec={}", _state, instanceId, offset, allowedBuildTimeSec); SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withOffset(offset).withBuildTimeSeconds(allowedBuildTimeSec) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT) .withSplitCommit(_isSplitCommitEnabled); if (_isSplitCommitEnabled) { params.withControllerVipUrl(_controllerVipUrl); } return new SegmentCompletionProtocol.Response(params); }
@Test public void testIncompleteResponseParams() { // Test with reduced params SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); assertEquals(response.getBuildTimeSeconds(), BUILD_TIME_MILLIS); assertEquals(response.getOffset(), OFFSET); assertNull(response.getSegmentLocation()); assertFalse(response.isSplitCommit()); assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); }
private SegmentCompletionProtocol.Response catchup(String instanceId, long offset) { LOGGER.info("{}:CATCHUP for instance={} offset={}", _state, instanceId, offset); return new SegmentCompletionProtocol.Response( new SegmentCompletionProtocol.Response.Params().withOffset(_winningOffset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.CATCH_UP)); }
private SegmentCompletionProtocol.Response hold(String instanceId, long offset) { LOGGER.info("{}:HOLD for instance={} offset={}", _state, instanceId, offset); return new SegmentCompletionProtocol.Response(new SegmentCompletionProtocol.Response.Params() .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.HOLD).withOffset(offset)); }
private SegmentCompletionProtocol.Response keep(String instanceId, long offset) { LOGGER.info("{}:KEEP for instance={} offset={}", _state, instanceId, offset); return new SegmentCompletionProtocol.Response(new SegmentCompletionProtocol.Response.Params().withOffset(offset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.KEEP)); }
private SegmentCompletionProtocol.Response processCommitWhileUploading(String instanceId, long offset, long now) { LOGGER.info("Processing segmentCommit({}, {})", instanceId, offset); SegmentCompletionProtocol.Response response = abortIfTooLateAndReturnHold(now, instanceId, offset); if (response != null) { return response; } // Another committer (or same) came in while one was uploading. Ask them to hold in case this one fails. return new SegmentCompletionProtocol.Response(new SegmentCompletionProtocol.Response.Params().withOffset(offset) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.HOLD)); }