@Test public void testUserProvidedHashing() { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); List<String> userHashes = Arrays.asList("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); env.addSource(new NoOpSourceFunction(), "src").setUidHash(userHashes.get(0)) .map(new NoOpMapFunction()) .filter(new NoOpFilterFunction()) .keyBy(new NoOpKeySelector()) .reduce(new NoOpReduceFunction()).name("reduce").setUidHash(userHashes.get(1)); StreamGraph streamGraph = env.getStreamGraph(); int idx = 1; for (JobVertex jobVertex : streamGraph.getJobGraph().getVertices()) { List<JobVertexID> idAlternatives = jobVertex.getIdAlternatives(); Assert.assertEquals(idAlternatives.get(idAlternatives.size() - 1).toString(), userHashes.get(idx)); --idx; } }
@Override protected String convertToString(JobVertexID value) { return value.toString(); } }
@Override public void serialize(JobVertexID value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeFieldName(value.toString()); } }
@Override protected void putVariables(Map<String, String> variables) { variables.put(ScopeFormat.SCOPE_TASK_VERTEX_ID, vertexId.toString()); variables.put(ScopeFormat.SCOPE_TASK_NAME, taskName); variables.put(ScopeFormat.SCOPE_TASK_ATTEMPT_ID, executionId.toString()); variables.put(ScopeFormat.SCOPE_TASK_ATTEMPT_NUM, String.valueOf(attemptNumber)); variables.put(ScopeFormat.SCOPE_TASK_SUBTASK_INDEX, String.valueOf(subtaskIndex)); }
@Override public void serialize(JobVertexID value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeString(value.toString()); } }
@Override public void serialize(JobVertexID value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeFieldName(value.toString()); } }
@Override public void serialize(JobVertexID value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeString(value.toString()); } }
@Override public void serialize(JobVertexID value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeFieldName(value.toString()); } }
@Override protected void putVariables(Map<String, String> variables) { variables.put(ScopeFormat.SCOPE_TASK_VERTEX_ID, vertexId.toString()); variables.put(ScopeFormat.SCOPE_TASK_NAME, taskName); variables.put(ScopeFormat.SCOPE_TASK_ATTEMPT_ID, executionId.toString()); variables.put(ScopeFormat.SCOPE_TASK_ATTEMPT_NUM, String.valueOf(attemptNumber)); variables.put(ScopeFormat.SCOPE_TASK_SUBTASK_INDEX, String.valueOf(subtaskIndex)); }
@Override public void serialize(JobVertexID value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeString(value.toString()); } }
@Override protected void putVariables(Map<String, String> variables) { variables.put(ScopeFormat.SCOPE_TASK_VERTEX_ID, vertexId.toString()); variables.put(ScopeFormat.SCOPE_TASK_NAME, taskName); variables.put(ScopeFormat.SCOPE_TASK_ATTEMPT_ID, executionId.toString()); variables.put(ScopeFormat.SCOPE_TASK_ATTEMPT_NUM, String.valueOf(attemptNumber)); variables.put(ScopeFormat.SCOPE_TASK_SUBTASK_INDEX, String.valueOf(subtaskIndex)); }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { Collection<? extends AccessExecutionJobVertex> allVertices = graph.getAllVertices().values(); List<ArchivedJson> archive = new ArrayList<>(allVertices.size()); for (AccessExecutionJobVertex task : allVertices) { ResponseBody json = createSubtaskTimesInfo(task); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } return archive; }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { Collection<? extends AccessExecutionJobVertex> vertices = graph.getAllVertices().values(); List<ArchivedJson> archive = new ArrayList<>(vertices.size()); for (AccessExecutionJobVertex task : vertices) { ResponseBody json = createJobVertexDetailsInfo(task, graph.getJobID(), null); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } return archive; }
@Override protected QueryScopeInfo.OperatorQueryScopeInfo createQueryServiceMetricInfo(CharacterFilter filter) { return new QueryScopeInfo.OperatorQueryScopeInfo( this.parent.parent.jobId.toString(), this.parent.vertexId.toString(), this.parent.subtaskIndex, filter.filterCharacters(this.operatorName)); }
@Override protected QueryScopeInfo.OperatorQueryScopeInfo createQueryServiceMetricInfo(CharacterFilter filter) { return new QueryScopeInfo.OperatorQueryScopeInfo( this.parent.parent.jobId.toString(), this.parent.vertexId.toString(), this.parent.subtaskIndex, filter.filterCharacters(this.operatorName)); }
@Nullable @Override protected MetricStore.ComponentMetricStore getComponentMetricStore( HandlerRequest<EmptyRequestBody, SubtaskMetricsMessageParameters> request, MetricStore metricStore) { final JobID jobId = request.getPathParameter(JobIDPathParameter.class); final JobVertexID vertexId = request.getPathParameter(JobVertexIdPathParameter.class); final int subtaskIndex = request.getPathParameter(SubtaskIndexPathParameter.class); return metricStore.getSubtaskMetricStore(jobId.toString(), vertexId.toString(), subtaskIndex); }
@Nullable @Override protected MetricStore.ComponentMetricStore getComponentMetricStore( HandlerRequest<EmptyRequestBody, SubtaskMetricsMessageParameters> request, MetricStore metricStore) { final JobID jobId = request.getPathParameter(JobIDPathParameter.class); final JobVertexID vertexId = request.getPathParameter(JobVertexIdPathParameter.class); final int subtaskIndex = request.getPathParameter(SubtaskIndexPathParameter.class); return metricStore.getSubtaskMetricStore(jobId.toString(), vertexId.toString(), subtaskIndex); }
@Override protected MetricStore.ComponentMetricStore getComponentMetricStore( HandlerRequest<EmptyRequestBody, JobVertexMetricsMessageParameters> request, MetricStore metricStore) { final JobID jobId = request.getPathParameter(JobIDPathParameter.class); final JobVertexID vertexId = request.getPathParameter(JobVertexIdPathParameter.class); return metricStore.getTaskMetricStore(jobId.toString(), vertexId.toString()); }
@Override protected MetricStore.ComponentMetricStore getComponentMetricStore( HandlerRequest<EmptyRequestBody, JobVertexMetricsMessageParameters> request, MetricStore metricStore) { final JobID jobId = request.getPathParameter(JobIDPathParameter.class); final JobVertexID vertexId = request.getPathParameter(JobVertexIdPathParameter.class); return metricStore.getTaskMetricStore(jobId.toString(), vertexId.toString()); }
@Override protected QueryScopeInfo.OperatorQueryScopeInfo createQueryServiceMetricInfo(CharacterFilter filter) { return new QueryScopeInfo.OperatorQueryScopeInfo( this.parent.parent.jobId.toString(), this.parent.vertexId.toString(), this.parent.subtaskIndex, filter.filterCharacters(this.operatorName), this.operatorID.toString()); }