@Override protected boolean matchesSafely(final TaskStatusEvent item) { return item.getStatus().getContainerId() != null; } };
@Override public int compare(TaskStatusEvent arg0, TaskStatusEvent arg1) { if (arg1.getTimestamp() > arg0.getTimestamp()) { return -1; } else if (arg1.getTimestamp() == arg0.getTimestamp()) { return 0; } else { return 1; } } };
/** * Set the {@link TaskStatus} for the job identified by {@code jobId}. */ @Override public void setTaskStatus(final JobId jobId, final TaskStatus status) throws InterruptedException { log.debug("setting task status: {}", status); taskStatuses.put(jobId.toString(), status.toJsonBytes()); if (historyWriter != null) { try { historyWriter.saveHistoryItem(status); } catch (Exception e) { // Log error here and keep going as saving task history is not critical. // This is to prevent bad data in the queue from screwing up the actually important Helios // agent operations. log.error("Error saving task status {} to ZooKeeper: {}", status, e); } } final TaskStatusEvent event = new TaskStatusEvent(status, System.currentTimeMillis(), agent); final byte[] message = event.toJsonBytes(); for (final EventSender sender : eventSenders) { sender.send(taskStatusEventTopic, message); } }
final String host = checkNotNull(event.getHost()); final long timestamp = event.getTimestamp(); final TaskStatus status = checkNotNull(event.getStatus()); final State state = checkNotNull(status.getState()); String containerId = status.getContainerId();
final JobId jobId = item.getStatus().getJob().getId(); final String historyPath = Paths.historyJobHostEventsTimestamp( jobId, hostname, item.getTimestamp()); log.debug("writing queued item to zookeeper {} {}", item.getStatus().getJob().getId(), item.getTimestamp()); client.createAndSetData(historyPath, item.getStatus().toJsonBytes());
public void saveHistoryItem(final TaskStatus status, long timestamp) throws InterruptedException { add(new TaskStatusEvent(status, timestamp, hostname)); }
String host = checkNotNull(event.getHost()); long timestamp = checkNotNull(event.getTimestamp()); TaskStatus status = checkNotNull(event.getStatus()); State state = checkNotNull(status.getState()); String containerId = status.getContainerId();
jobId, h, Long.valueOf(event))); final TaskStatus status = Json.read(data, TaskStatus.class); jsEvents.add(new TaskStatusEvent(status, Long.valueOf(event), h)); } catch (NoNodeException e) { // ignore, it went away before we read it } catch (KeeperException | IOException e) {
@Override protected boolean matchesSafely(final TaskStatusEvent event) { final State actual = event.getStatus().getState(); for (final State state : possibleStates) { if (state == actual) { return true; } } return false; } };
private TaskStatusEvent findEldestEvent() { // We don't lock anything because in the worst case, we just put things in out of order which // while not perfect, won't cause any actual harm. Out of order meaning between jobids, not // within the same job id. Whether this is the best strategy (as opposed to fullest deque) // is arguable. TaskStatusEvent current = null; for (final Deque<TaskStatusEvent> queue : items.values()) { if (queue == null) { continue; } final TaskStatusEvent item = queue.peek(); if (current == null || (item.getTimestamp() < current.getTimestamp())) { current = item; } } return current; }
/** * Set the {@link TaskStatus} for the job identified by {@code jobId}. */ @Override public void setTaskStatus(final JobId jobId, final TaskStatus status) throws InterruptedException { log.debug("setting task status: {}", status); taskStatuses.put(jobId.toString(), status.toJsonBytes()); try { historyWriter.saveHistoryItem(status); } catch (Exception e) { // Log error here and keep going as saving task history is not critical. // This is to prevent bad data in the queue from screwing up the actually important Helios // agent operations. log.error("Error saving task status {} to ZooKeeper: {}", status, e); } final TaskStatusEvent event = new TaskStatusEvent(status, System.currentTimeMillis(), agent); kafkaSender.send(KafkaRecord.of(TaskStatusEvent.KAFKA_TOPIC, event.toJsonBytes())); }
public void saveHistoryItem(final TaskStatus status, long timestamp) throws InterruptedException { add(new TaskStatusEvent(status, timestamp, hostname)); }
@Override public TaskStatus.State apply(@Nullable TaskStatusEvent input) { if (input != null) { return input.getStatus().getState(); } return null; } };
final String eventDate = DATE_FORMATTER.print(event.getTimestamp()); final long unusedDurationMillis = clock.now().getMillis() - event.getTimestamp();
jobId, host, Long.valueOf(event))); final TaskStatus status = Json.read(data, TaskStatus.class); jsEvents.add(new TaskStatusEvent(status, Long.valueOf(event), host)); } catch (NoNodeException e) { // ignore, it went away before we read it } catch (KeeperException | IOException e) {
private void putBack(TaskStatusEvent event) { final JobId key = event.getStatus().getJob().getId(); final Deque<TaskStatusEvent> queue = getDeque(key); synchronized (queue) { if (queue.size() >= MAX_QUEUE_SIZE) { // already full, just toss the event return; } queue.push(event); count.incrementAndGet(); } }
@Override public int compare(TaskStatusEvent arg0, TaskStatusEvent arg1) { if (arg1.getTimestamp() > arg0.getTimestamp()) { return -1; } else if (arg1.getTimestamp() == arg0.getTimestamp()) { return 0; } else { return 1; } } };
final JobId id = current.getStatus().getJob().getId(); final Deque<TaskStatusEvent> deque = items.get(id); if (deque == null) {
@Override protected long getTimestamp(final TaskStatusEvent event) { return event.getTimestamp(); }
private void add(TaskStatusEvent item) throws InterruptedException { // If too many "globally", toss them while (count.get() >= MAX_TOTAL_SIZE) { getNext(); } final JobId key = item.getStatus().getJob().getId(); final Deque<TaskStatusEvent> deque = getDeque(key); synchronized (deque) { // if too many in the particular deque, toss them while (deque.size() >= MAX_QUEUE_SIZE) { deque.remove(); count.decrementAndGet(); } deque.add(item); count.incrementAndGet(); } try { backingStore.set(items); } catch (ClosedByInterruptException e) { log.debug("Writing task status event to backing store was interrupted"); } catch (IOException e) { // We are best effort after all... log.warn("Failed to write task status event to backing store", e); } }