private boolean hasTaskPendingBeyondThreshold(Collection<? extends TaskRunnerWorkItem> pendingTasks) { long now = System.currentTimeMillis(); for (TaskRunnerWorkItem pendingTask : pendingTasks) { final Duration durationSinceInsertion = new Duration(pendingTask.getQueueInsertionTime().getMillis(), now); final Duration timeoutDuration = config.getPendingTaskTimeout().toStandardDuration(); if (durationSinceInsertion.isEqual(timeoutDuration) || durationSinceInsertion.isLongerThan(timeoutDuration)) { return true; } } return false; }
/** * @param dataInterval interval containing the actual data * @param period time will be offset by a multiple of the given period * until there is at least a full period ending within the data interval * @param origin origin to be used to align time periods * (e.g. to determine on what day of the week a weekly period starts) */ @JsonCreator public TimewarpOperator( @JsonProperty("dataInterval") Interval dataInterval, @JsonProperty("period") Period period, @JsonProperty("origin") DateTime origin ) { this.originMillis = origin.getMillis(); this.dataInterval = dataInterval; // this will fail for periods that do not map to millis (e.g. P1M) this.periodMillis = period.toStandardDuration().getMillis(); }
public static class TaskNotRunnableException extends RuntimeException { public TaskNotRunnableException(String message) { super(message); } }
public static String humanDuration(final Duration dur) { final Period p = dur.toPeriod().normalizedStandard(); if (dur.getStandardSeconds() == 0) { return "0 seconds"; } else if (dur.getStandardSeconds() < 60) { return format("%d second%s", p.getSeconds(), p.getSeconds() > 1 ? "s" : ""); } else if (dur.getStandardMinutes() < 60) { return format("%d minute%s", p.getMinutes(), p.getMinutes() > 1 ? "s" : ""); } else if (dur.getStandardHours() < 24) { return format("%d hour%s", p.getHours(), p.getHours() > 1 ? "s" : ""); } else { return format("%d day%s", dur.getStandardDays(), dur.getStandardDays() > 1 ? "s" : ""); } }
/** Tests that the watermark that guarantees firing is that of the subtrigger. */ @Test public void testFireDeadline() throws Exception { setUp(FixedWindows.of(Duration.millis(10))); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); Instant arbitraryInstant = new Instant(34957849); when(mockTrigger.getWatermarkThatGuaranteesFiring(Mockito.<IntervalWindow>any())) .thenReturn(arbitraryInstant); assertThat( Repeatedly.forever(mockTrigger).getWatermarkThatGuaranteesFiring(window), equalTo(arbitraryInstant)); }
@Test public void testOnElementCombiningDiscarding() throws Exception { // Test basic execution of a trigger using a non-combining window set and discarding mode. WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining( strategy, mockTriggerStateMachine, Sum.ofIntegers(), VarIntCoder.of()); injectElement(tester, 2); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); injectElement(tester, 3); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); injectElement(tester, 4); // This element shouldn't be seen, because the trigger has finished injectElement(tester, 6); assertThat( tester.extractOutput(), contains( isSingleWindowedValue(equalTo(5), 2, 0, 10), isSingleWindowedValue(equalTo(4), 4, 0, 10))); assertTrue(tester.isMarkedFinished(firstWindow)); tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow); }
@Test public void testCumulativeTimeOverflow() throws Exception { Dataflow.Projects.Locations.Jobs.Get statusRequest = mock(Dataflow.Projects.Locations.Jobs.Get.class); Job statusResponse = new Job(); statusResponse.setCurrentState("JOB_STATE_RUNNING"); when(mockJobs.get(eq(PROJECT_ID), eq(REGION_ID), eq(JOB_ID))).thenReturn(statusRequest); when(statusRequest.execute()).thenReturn(statusResponse); FastNanoClockAndFuzzySleeper clock = new FastNanoClockAndFuzzySleeper(); DataflowPipelineJob job = new DataflowPipelineJob(DataflowClient.create(options), JOB_ID, options, ImmutableMap.of()); long startTime = clock.nanoTime(); State state = job.waitUntilFinish(Duration.millis(4), null, clock, clock); assertEquals(null, state); long timeDiff = TimeUnit.NANOSECONDS.toMillis(clock.nanoTime() - startTime); // Should only have slept for the 4 ms allowed. assertThat(timeDiff, lessThanOrEqualTo(4L)); }
@Test public void testOnElementBufferingAccumulating() throws Exception { // Test basic execution of a trigger using a non-combining window set and accumulating mode. ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY); injectElement(tester, 1); // Fires {1, 2} when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); injectElement(tester, 2); // Fires {1, 2, 3} because we are in accumulating mode when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); injectElement(tester, 3); // This element shouldn't be seen, because the trigger has finished injectElement(tester, 4); assertThat( tester.extractOutput(), contains( isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), isSingleWindowedValue(containsInAnyOrder(1, 2, 3), 3, 0, 10))); assertTrue(tester.isMarkedFinished(firstWindow)); tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow); }
@Test public void retryer_should_wait_after_failure_before_trying_again() throws Exception { Retryer timeRetryer = new Retryer(1, Duration.millis(100)); Stopwatch stopwatch = Stopwatch.createStarted(); when(operation.call()).thenThrow(new DockerExecutionException()).thenAnswer(i -> { assertThat(stopwatch.elapsed(TimeUnit.MILLISECONDS), greaterThan(100L)); return "success"; }); String result = timeRetryer.runWithRetries(operation); assertThat(result, is("success")); }
@Test public void shouldCalculateTotalTimeFromFirstScheduledJobToLastCompletedJob() { final DateTime time0 = new DateTime(2008, 2, 22, 10, 21, 23, 0); timeProvider = new TimeProvider() { @Override stage = StageMother.custom("test", jobInstances); firstJob.assign("AGENT-1", time1.toDate()); firstJob.completing(JobResult.Passed, time2.toDate()); firstJob.completed(time2.toDate()); stage.setLastTransitionedTime(new Timestamp(time4.toDate().getTime())); RunDuration.ActualDuration expectedDuration = new RunDuration.ActualDuration(new Duration(time0, time4)); RunDuration.ActualDuration duration = (RunDuration.ActualDuration) stage.getDuration(); assertThat(duration, is(expectedDuration)); assertThat(duration.getTotalSeconds(), is(7263L));
@Test public void should_not_pause_after_last_failure() throws Exception { Retryer failFast = new Retryer(0, Duration.standardSeconds(1)); when(operation.call()).thenThrow(new DockerExecutionException()); Stopwatch stopwatch = Stopwatch.createStarted(); try { failFast.runWithRetries(operation); } catch (DockerExecutionException e) { // expected } assertThat(stopwatch.elapsed(TimeUnit.MILLISECONDS), lessThan(1000L)); }
String maxMsg = "at " + new DateTime(curTimeStamp).toString(); log.info("Kafka pull time limit reached"); statusMsg += " max read " + maxMsg; this.periodFormatter.print(new Duration(this.startTime, System.currentTimeMillis()).toPeriod()); String timeSpentOnTopicMsg = String.format("Time spent on topic %s:%d = %s", key.getTopic(), key.getPartition(), timeSpentOnPartition); if (this.numRecordsReadForCurrentPartition != 0) { String timeSpentOnPartition = this.periodFormatter.print(new Duration(this.startTime, System.currentTimeMillis()).toPeriod()); log.info("Time spent on this partition = " + timeSpentOnPartition); log.info("Num of records read for this partition = " + this.numRecordsReadForCurrentPartition); DateTime time = new DateTime(curTimeStamp); statusMsg += " begin read at " + time.toString(); context.setStatus(statusMsg);
@Test public void testDurationToDateTime() { final DateTime origin = DateTimes.of("2012-01-02T05:00:00.000-08:00"); Granularity gran = new DurationGranularity( new Period("PT12H5M").toStandardDuration().getMillis(), origin ); Assert.assertEquals( DateTimes.of("2012-01-01T05:00:04.123-08:00"), gran.toDateTime(DateTimes.of("2012-01-01T05:00:04.123-08:00").getMillis()) ); }
final long windowMillis = windowPeriod.toStandardDuration().getMillis(); DateTimes.nowUtc().plus( new Duration( System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis Duration initialDelay = new Duration( System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis ); Duration rate = new Duration(truncatedNow, segmentGranularity.increment(truncatedNow)); ScheduledExecutors.scheduleAtFixedRate(scheduledExecutor, initialDelay, rate, threadRenamingCallable);
private static void printScriptRunTime(DateTime startTime) { DateTime endTime = new DateTime(); Duration duration = new Duration(startTime, endTime); Period period = duration.toPeriod().normalizedStandard(PeriodType.time()); log.info("Pig script completed in " + PeriodFormat.getDefault().print(period) + " (" + duration.getMillis() + " ms)"); }
@Test public void testSlightlyOverlapping() throws Exception { Map<IntervalWindow, Set<String>> expected = new HashMap<>(); expected.put(new IntervalWindow(new Instant(-5), new Instant(2)), set(1)); expected.put(new IntervalWindow(new Instant(0), new Instant(7)), set(1, 2, 5)); expected.put(new IntervalWindow(new Instant(5), new Instant(12)), set(5, 9, 10, 11)); expected.put(new IntervalWindow(new Instant(10), new Instant(17)), set(10, 11)); SlidingWindows windowFn = SlidingWindows.of(new Duration(7)).every(new Duration(5)); assertEquals(expected, runWindowFn(windowFn, Arrays.asList(1L, 2L, 5L, 9L, 10L, 11L))); assertThat(windowFn.assignsToOneWindow(), is(false)); }
@Test @Category(NeedsRunner.class) public void testUnboundedInputRate() { long numElements = 5000; long elemsPerPeriod = 10L; Duration periodLength = Duration.millis(8); PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements).withRate(elemsPerPeriod, periodLength)); addCountingAsserts(input, 0, numElements); long expectedRuntimeMillis = (periodLength.getMillis() * numElements) / elemsPerPeriod; Instant startTime = Instant.now(); p.run(); Instant endTime = Instant.now(); assertThat(endTime.isAfter(startTime.plus(expectedRuntimeMillis)), is(true)); }
@Override public Boolean apply(@Nullable DateTime startTime) { if (startTime == null) { return false; } taskData.startTime = startTime; long millisRemaining = ioConfig.getTaskDuration().getMillis() - (System.currentTimeMillis() - taskData.startTime.getMillis()); if (millisRemaining > 0) { scheduledExec.schedule( buildRunTask(), millisRemaining + MAX_RUN_FREQUENCY_MILLIS, TimeUnit.MILLISECONDS ); } return true; } }, workerExec
@Test public void unrelatedStepShouldNotFire() throws Exception { CountDownLatch latch = new CountDownLatch(1); WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10)); IntervalWindow window = new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10))); executor.callOnGuaranteedFiring( sum, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch)); executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(20))); assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(false)); }