@GET @Path("/namespaces/{namespace-id}/datasets/{dataset-id}/lineage/fields/{field-name}/operations") public void datasetFieldLineageDetails(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("dataset-id") String datasetId, @PathParam("field-name") String field, @QueryParam("direction") @DefaultValue("both") String directionStr, @QueryParam("start") String startStr, @QueryParam("end") String endStr) throws BadRequestException { TimeRange range = parseRange(startStr, endStr); Constants.FieldLineage.Direction direction = parseDirection(directionStr); EndPointField endPointField = new EndPointField(EndPoint.of(namespaceId, datasetId), field); FieldLineageDetails details = fieldLineageAdmin.getOperationDetails(direction, endPointField, range.getStart(), range.getEnd()); responder.sendJson(HttpResponseStatus.OK, GSON.toJson(details)); }
private void addEndPoint(MDSKey.Builder keyBuilder, EndPoint endPoint) { keyBuilder.add(endPoint.getNamespace()) .add(endPoint.getName()); }
/** * Return the EndPoint as defined by the provided namespace and name. * * @param namespace the name of the namespace * @param name the name of the EndPoint * @return the EndPoint */ public static EndPoint of(String namespace, String name) { return new EndPoint(namespace, name); }
for (WriteOperation write : writeOperations) { if (!write.getDestination().equals(destinationField.getEndPoint())) { continue;
for (WriteOperation write : writeOperations) { if (!write.getDestination().equals(destinationField.getEndPoint())) { continue;
@GET @Path("/namespaces/{namespace-id}/datasets/{dataset-id}/lineage/fields/{field-name}/operations") public void datasetFieldLineageDetails(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("dataset-id") String datasetId, @PathParam("field-name") String field, @QueryParam("direction") @DefaultValue("both") String directionStr, @QueryParam("start") String startStr, @QueryParam("end") String endStr) throws BadRequestException { TimeRange range = parseRange(startStr, endStr); Constants.FieldLineage.Direction direction = parseDirection(directionStr); EndPointField endPointField = new EndPointField(EndPoint.of(namespaceId, datasetId), field); FieldLineageDetails details = fieldLineageAdmin.getOperationDetails(direction, endPointField, range.getStart(), range.getEnd()); responder.sendJson(HttpResponseStatus.OK, GSON.toJson(details)); }
private void addEndPoint(MDSKey.Builder keyBuilder, EndPoint endPoint) { keyBuilder.add(endPoint.getNamespace()) .add(endPoint.getName()); }
if (!(readOperation.getSource().equals(sourceField.getEndPoint()) && readOperation.getOutputs().contains(sourceField.getField()))) { continue;
/** * Return the EndPoint as defined by the provided name. * * @param name the name of the EndPoint * @return the EndPoint */ public static EndPoint of(String name) { return new EndPoint(name); }
@GET @Path("/namespaces/{namespace-id}/datasets/{dataset-id}/lineage/fields") public void datasetFields(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("dataset-id") String datasetId, @QueryParam("start") String startStr, @QueryParam("end") String endStr, @QueryParam("prefix") String prefix, @QueryParam("includeCurrent") boolean includeCurrent) throws BadRequestException, IOException { TimeRange range = parseRange(startStr, endStr); Set<Field> result = fieldLineageAdmin.getFields(EndPoint.of(namespaceId, datasetId), range.getStart(), range.getEnd(), prefix, includeCurrent); // CDAP-14168: From 5.1 this endpoint supports returning a Set of Field object rather Set of String. For backward // compatibility in 5.1 the default behavior is to return a Set of String (field names). This default behavior // can be overridden by passing the query parameter 'includeCurrent' set to 'true' which will return set of // Field object. if (includeCurrent) { responder.sendJson(HttpResponseStatus.OK, GSON.toJson(result)); } else { responder.sendJson(HttpResponseStatus.OK, GSON.toJson(result.stream().map(Field::getName).collect(Collectors.toSet()))); } }
private Set<DatasetField> convertSummaryToDatasetField(Set<EndPointField> summary) { Map<EndPoint, Set<String>> endPointFields = new HashMap<>(); for (EndPointField endPointField : summary) { EndPoint endPoint = endPointField.getEndPoint(); Set<String> fields = endPointFields.computeIfAbsent(endPoint, k -> new HashSet<>()); fields.add(endPointField.getField()); } Set<DatasetField> result = new HashSet<>(); for (Map.Entry<EndPoint, Set<String>> entry : endPointFields.entrySet()) { DatasetId datasetId = new DatasetId(entry.getKey().getNamespace(), entry.getKey().getName()); result.add(new DatasetField(datasetId, entry.getValue())); } return result; }
if (!(readOperation.getSource().equals(sourceField.getEndPoint()) && readOperation.getOutputs().contains(sourceField.getField()))) { continue;
@GET @Path("/namespaces/{namespace-id}/datasets/{dataset-id}/lineage/fields/{field-name}") public void datasetFieldLineageSummary(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("dataset-id") String datasetId, @PathParam("field-name") String field, @QueryParam("direction") String directionStr, @QueryParam("start") String startStr, @QueryParam("end") String endStr) throws BadRequestException { TimeRange range = parseRange(startStr, endStr); Constants.FieldLineage.Direction direction = parseDirection(directionStr); EndPointField endPointField = new EndPointField(EndPoint.of(namespaceId, datasetId), field); FieldLineageSummary summary = fieldLineageAdmin.getSummary(direction, endPointField, range.getStart(), range.getEnd()); responder.sendJson(HttpResponseStatus.OK, GSON.toJson(summary)); }
private Set<DatasetField> convertSummaryToDatasetField(Set<EndPointField> summary) { Map<EndPoint, Set<String>> endPointFields = new HashMap<>(); for (EndPointField endPointField : summary) { EndPoint endPoint = endPointField.getEndPoint(); Set<String> fields = endPointFields.computeIfAbsent(endPoint, k -> new HashSet<>()); fields.add(endPointField.getField()); } Set<DatasetField> result = new HashSet<>(); for (Map.Entry<EndPoint, Set<String>> entry : endPointFields.entrySet()) { DatasetId datasetId = new DatasetId(entry.getKey().getNamespace(), entry.getKey().getName()); result.add(new DatasetField(datasetId, entry.getValue())); } return result; }
@GET @Path("/namespaces/{namespace-id}/datasets/{dataset-id}/lineage/fields") public void datasetFields(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("dataset-id") String datasetId, @QueryParam("start") String startStr, @QueryParam("end") String endStr, @QueryParam("prefix") String prefix, @QueryParam("includeCurrent") boolean includeCurrent) throws BadRequestException, IOException { TimeRange range = parseRange(startStr, endStr); Set<Field> result = fieldLineageAdmin.getFields(EndPoint.of(namespaceId, datasetId), range.getStart(), range.getEnd(), prefix, includeCurrent); // CDAP-14168: From 5.1 this endpoint supports returning a Set of Field object rather Set of String. For backward // compatibility in 5.1 the default behavior is to return a Set of String (field names). This default behavior // can be overridden by passing the query parameter 'includeCurrent' set to 'true' which will return set of // Field object. if (includeCurrent) { responder.sendJson(HttpResponseStatus.OK, GSON.toJson(result)); } else { responder.sendJson(HttpResponseStatus.OK, GSON.toJson(result.stream().map(Field::getName).collect(Collectors.toSet()))); } }
private boolean matchesEndpoint(byte[] column, EndPoint endPoint) { // RAW_OPERATION_MARKER key is not written as MDSKey with byte length encoding so don't try to create an object // out of it if (Arrays.equals(column, RAW_OPERATION_MARKER)) { return false; } MDSKey.Splitter keySplitter = new MDSKey(column).split(); return Arrays.equals(OUTGOING_DIRECTION_MARKER, keySplitter.getBytes()) && keySplitter.getString().equals(endPoint.getNamespace()) && keySplitter.getString().equals(endPoint.getName()); }
@GET @Path("/namespaces/{namespace-id}/datasets/{dataset-id}/lineage/fields/{field-name}") public void datasetFieldLineageSummary(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("dataset-id") String datasetId, @PathParam("field-name") String field, @QueryParam("direction") String directionStr, @QueryParam("start") String startStr, @QueryParam("end") String endStr) throws BadRequestException { TimeRange range = parseRange(startStr, endStr); Constants.FieldLineage.Direction direction = parseDirection(directionStr); EndPointField endPointField = new EndPointField(EndPoint.of(namespaceId, datasetId), field); FieldLineageSummary summary = fieldLineageAdmin.getSummary(direction, endPointField, range.getStart(), range.getEnd()); responder.sendJson(HttpResponseStatus.OK, GSON.toJson(summary)); }
private boolean matchesEndpoint(byte[] column, EndPoint endPoint) { // RAW_OPERATION_MARKER key is not written as MDSKey with byte length encoding so don't try to create an object // out of it if (Arrays.equals(column, RAW_OPERATION_MARKER)) { return false; } MDSKey.Splitter keySplitter = new MDSKey(column).split(); return Arrays.equals(OUTGOING_DIRECTION_MARKER, keySplitter.getBytes()) && keySplitter.getString().equals(endPoint.getNamespace()) && keySplitter.getString().equals(endPoint.getName()); }
@Override public void prepareRun(BatchSourceContext context) throws DatasetManagementException { super.prepareRun(context); Schema schema = tableConfig.getSchema(); if (schema != null && schema.getFields() != null) { FieldOperation operation = new FieldReadOperation("Read", "Read from Table dataset", EndPoint.of(context.getNamespace(), tableConfig.getName()), schema.getFields().stream().map(Schema.Field::getName) .collect(Collectors.toList())); context.record(Collections.singletonList(operation)); } }
MetadataEntity.ofDataset(endPoint.getNamespace(), endPoint.getName()));