/** * Filter for {@link GobblinTrackingEvent}. Used to quickly determine whether an event should be used to produce * a {@link JobSpec}. */ protected boolean acceptEvent(GobblinTrackingEvent event) { if (!event.getMetadata().containsKey(SlaEventKeys.DATASET_URN_KEY)) { return false; } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); if (this.urnFilter.isPresent() && !this.urnFilter.get().matcher(datasetURN).find()) { return false; } if (this.nameFilter.isPresent() && !this.nameFilter.get().matcher(event.getName()).find()) { return false; } return true; } }
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getNamespace().equals(expectedNamespace); } };
private static GobblinTrackingEvent sanitizeEvent(GobblinTrackingEvent event) { Map<String, String> newMetadata = Maps.newHashMap(); for (Map.Entry<String, String> metadata : event.getMetadata().entrySet()) { newMetadata.put(metadata.getKey() == null ? NULL_STRING : metadata.getKey(), metadata.getValue() == null ? NULL_STRING : metadata.getValue()); } event.setMetadata(newMetadata); return event; } }
/** * Submit {@link gobblin.metrics.GobblinTrackingEvent} to all notification listeners attached to this or any * ancestor {@link gobblin.metrics.MetricContext}s. The argument for this method is mutated by the method, so it * should not be reused by the caller. * * @param nonReusableEvent {@link GobblinTrackingEvent} to submit. This object will be mutated by the method, * so it should not be reused by the caller. */ public void submitEvent(GobblinTrackingEvent nonReusableEvent) { nonReusableEvent.setTimestamp(System.currentTimeMillis()); // Inject metric context tags into event metadata. Map<String, String> originalMetadata = nonReusableEvent.getMetadata(); Map<String, Object> tags = getTagMap(); Map<String, String> newMetadata = Maps.newHashMap(); for(Map.Entry<String, Object> entry : tags.entrySet()) { newMetadata.put(entry.getKey(), entry.getValue().toString()); } newMetadata.putAll(originalMetadata); nonReusableEvent.setMetadata(newMetadata); EventNotification notification = new EventNotification(nonReusableEvent); sendNotification(notification); }
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getMetadata().containsKey(metadataKey); } };
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getName().equals(expectedName); } };
private void submitTrackingEvent(URI jobSpecURI, String jobSpecVersion, String operType) { GobblinTrackingEvent e = GobblinTrackingEvent.newBuilder() .setName(TRACKING_EVENT_NAME) .setNamespace(JobCatalog.class.getName()) .setMetadata(ImmutableMap.<String, String>builder() .put(GobblinMetricsKeys.OPERATION_TYPE_META, operType) .put(GobblinMetricsKeys.JOB_SPEC_URI_META, jobSpecURI.toString()) .put(GobblinMetricsKeys.JOB_SPEC_VERSION_META, jobSpecVersion) .build()) .build(); this.numAddedJobs.getContext().submitEvent(e); }
/** * Submits the {@link gobblin.metrics.GobblinTrackingEvent} to the {@link gobblin.metrics.MetricContext}. * @param name Name of the event. * @param additionalMetadata Additional metadata to be added to the event. */ public void submit(String name, Map<String, String> additionalMetadata) { if(this.metricContext.isPresent()) { Map<String, String> finalMetadata = Maps.newHashMap(this.metadata); if(!additionalMetadata.isEmpty()) { finalMetadata.putAll(additionalMetadata); } // Timestamp is set by metric context. this.metricContext.get().submitEvent(new GobblinTrackingEvent(0l, this.namespace, name, finalMetadata)); } }
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getMetadata().get(metadataKey).equals(metadataValue); } };
private void submitTrackingEvent(URI specSpecURI, String specSpecVersion, String operType) { GobblinTrackingEvent e = GobblinTrackingEvent.newBuilder() .setName(TRACKING_EVENT_NAME) .setNamespace(SpecCatalog.class.getName()) .setMetadata(ImmutableMap.<String, String>builder() .put(GobblinMetricsKeys.OPERATION_TYPE_META, operType) .put(GobblinMetricsKeys.SPEC_URI_META, specSpecURI.toString()) .put(GobblinMetricsKeys.SPEC_VERSION_META, specSpecVersion) .build()) .build(); this.numAddedSpecs.getContext().submitEvent(e); }
@Override public GobblinTrackingEvent build() { try { GobblinTrackingEvent record = new GobblinTrackingEvent(); record.timestamp = fieldSetFlags()[0] ? this.timestamp : (java.lang.Long) defaultValue(fields()[0]); record.namespace = fieldSetFlags()[1] ? this.namespace : (java.lang.String) defaultValue(fields()[1]); record.name = fieldSetFlags()[2] ? this.name : (java.lang.String) defaultValue(fields()[2]); record.metadata = fieldSetFlags()[3] ? this.metadata : (java.util.Map<java.lang.String,java.lang.String>) defaultValue(fields()[3]); return record; } catch (Exception e) { throw new org.apache.avro.AvroRuntimeException(e); } } }
@Override public Collection<Either<JobSpec, URI>> parseJobSpec(GobblinTrackingEvent event) { if (!acceptEvent(event)) { this.rejectedEvents.inc(); return Lists.newArrayList(); } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); URI jobSpecURI = PathUtils.mergePaths(new Path(this.baseURI), new Path(datasetURN)).toUri(); Map<String, String> jobConfigMap = Maps.newHashMap(); for (Map.Entry<String, String> entry : this.extractKeys.entrySet()) { if (event.getMetadata().containsKey(entry.getKey())) { jobConfigMap.put(entry.getValue(), event.getMetadata().get(entry.getKey())); } } Config jobConfig = ConfigFactory.parseMap(jobConfigMap); JobSpec jobSpec = JobSpec.builder(jobSpecURI).withTemplate(this.template).withConfig(jobConfig).build(); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }