final SchemaMetadata metadata = info.getSchemaMetadata(); schemaName = metadata.getName();
private AggregatedSchemaMetadataInfo buildAggregatedSchemaMetadataInfo(SchemaMetadataInfo schemaMetadataInfo) throws SchemaNotFoundException, SchemaBranchNotFoundException { if (schemaMetadataInfo == null) { return null; } List<SerDesInfo> serDesInfos = getSerDesInfos(schemaMetadataInfo.getSchemaMetadata().getName()); return new AggregatedSchemaMetadataInfo(schemaMetadataInfo.getSchemaMetadata(), schemaMetadataInfo.getId(), schemaMetadataInfo.getTimestamp(), getAggregatedSchemaBranch(schemaMetadataInfo.getSchemaMetadata().getName()), serDesInfos); }
@GET @Path("/subjects") @ApiOperation(value = "Get all registered subjects", response = String.class, responseContainer = "List", tags = OPERATION_GROUP_CONFLUENT_SR) @Timed @UnitOfWork public Response getSubjects() { Response response; try { List<String> registeredSubjects = schemaRegistry.findSchemaMetadata(Collections.emptyMap()) .stream() .map(x -> x.getSchemaMetadata().getName()) .collect(Collectors.toList()); response = WSUtils.respondEntity(registeredSubjects, Response.Status.OK); } catch (Exception ex) { LOG.error("Encountered error while retrieving all subjects", ex); response = serverError(); } return response; }
@GET @Path("/subjects") @ApiOperation(value = "Get all registered subjects", response = String.class, responseContainer = "List", tags = OPERATION_GROUP_CONFLUENT_SR) @Timed @UnitOfWork public Response getSubjects() { Response response; try { List<String> registeredSubjects = schemaRegistry.findSchemaMetadata(Collections.emptyMap()) .stream() .map(x -> x.getSchemaMetadata().getName()) .collect(Collectors.toList()); response = WSUtils.respondEntity(registeredSubjects, Response.Status.OK); } catch (Exception ex) { LOG.error("Encountered error while retrieving all subjects", ex); response = serverError(); } return response; }
@GET @Path("/schemas/{name}/serdes") @ApiOperation(value = "Get list of Serializers registered for the given schema name", response = SerDesInfo.class, responseContainer = "List", tags = OPERATION_GROUP_SERDE) @Timed @UnitOfWork public Response getSerializers(@ApiParam(value = "Schema name", required = true) @PathParam("name") String schemaName) { Response response; try { SchemaMetadataInfo schemaMetadataInfoStorable = schemaRegistry.getSchemaMetadataInfo(schemaName); if (schemaMetadataInfoStorable != null) { Collection<SerDesInfo> schemaSerializers = schemaRegistry.getSerDes(schemaMetadataInfoStorable.getSchemaMetadata().getName()); response = WSUtils.respondEntities(schemaSerializers, Response.Status.OK); } else { LOG.info("No schemas found with schemakey: [{}]", schemaName); response = WSUtils.respond(Response.Status.NOT_FOUND, CatalogResponse.ResponseMessage.ENTITY_NOT_FOUND, schemaName); } } catch (Exception ex) { LOG.error("Encountered error while getting serializers for schemaKey [{}]", schemaName, ex); response = WSUtils.respond(Response.Status.INTERNAL_SERVER_ERROR, CatalogResponse.ResponseMessage.EXCEPTION, ex.getMessage()); } return response; }
@GET @Path("/schemas/{name}/serdes") @ApiOperation(value = "Get list of Serializers registered for the given schema name", response = SerDesInfo.class, responseContainer = "List", tags = OPERATION_GROUP_SERDE) @Timed @UnitOfWork public Response getSerializers(@ApiParam(value = "Schema name", required = true) @PathParam("name") String schemaName) { Response response; try { SchemaMetadataInfo schemaMetadataInfoStorable = schemaRegistry.getSchemaMetadataInfo(schemaName); if (schemaMetadataInfoStorable != null) { Collection<SerDesInfo> schemaSerializers = schemaRegistry.getSerDes(schemaMetadataInfoStorable.getSchemaMetadata().getName()); response = WSUtils.respondEntities(schemaSerializers, Response.Status.OK); } else { LOG.info("No schemas found with schemakey: [{}]", schemaName); response = WSUtils.respond(Response.Status.NOT_FOUND, CatalogResponse.ResponseMessage.ENTITY_NOT_FOUND, schemaName); } } catch (Exception ex) { LOG.error("Encountered error while getting serializers for schemaKey [{}]", schemaName, ex); response = WSUtils.respond(Response.Status.INTERNAL_SERVER_ERROR, CatalogResponse.ResponseMessage.EXCEPTION, ex.getMessage()); } return response; }
public SchemaVersionInfo getSchemaVersionInfo(String schemaName, String schemaText) throws SchemaNotFoundException, InvalidSchemaException, SchemaBranchNotFoundException { SchemaMetadataInfo schemaMetadataInfo = getSchemaMetadataInfo(schemaName); if (schemaMetadataInfo == null) { throw new SchemaNotFoundException("No schema found for schema metadata key: " + schemaName); } return findSchemaVersion(SchemaBranch.MASTER_BRANCH, schemaMetadataInfo.getSchemaMetadata() .getType(), schemaText, schemaName); }
List<AggregatedSchemaMetadataInfo> aggregatedSchemaMetadataInfos = new ArrayList<>(); for (SchemaMetadataInfo schemaMetadataInfo : schemaMetadataInfos) { SchemaMetadata schemaMetadata = schemaMetadataInfo.getSchemaMetadata(); List<SerDesInfo> serDesInfos = new ArrayList<>(schemaRegistry.getSerDes(schemaMetadataInfo .getSchemaMetadata() .getName())); aggregatedSchemaMetadataInfos.add(
List<AggregatedSchemaMetadataInfo> aggregatedSchemaMetadataInfos = new ArrayList<>(); for (SchemaMetadataInfo schemaMetadataInfo : schemaMetadataInfos) { SchemaMetadata schemaMetadata = schemaMetadataInfo.getSchemaMetadata(); List<SerDesInfo> serDesInfos = new ArrayList<>(schemaRegistry.getSerDes(schemaMetadataInfo .getSchemaMetadata() .getName())); aggregatedSchemaMetadataInfos.add(
private SchemaVersionInfo doGetSchemaVersionInfo(SchemaIdVersion schemaIdVersion) throws SchemaNotFoundException { if (schemaIdVersion.getSchemaVersionId() != null) { LOG.info("Getting schema version from target registry for [{}]", schemaIdVersion.getSchemaVersionId()); return getEntity(currentSchemaRegistryTargets() .schemaVersionsByIdTarget .path(schemaIdVersion.getSchemaVersionId().toString()), SchemaVersionInfo.class); } else if (schemaIdVersion.getSchemaMetadataId() != null) { SchemaMetadataInfo schemaMetadataInfo = getSchemaMetadataInfo(schemaIdVersion.getSchemaMetadataId()); SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaMetadataInfo.getSchemaMetadata() .getName(), schemaIdVersion.getVersion()); LOG.info("Getting schema version from target registry for key [{}]", schemaVersionKey); return doGetSchemaVersionInfo(schemaVersionKey); } throw new IllegalArgumentException("Given argument not valid: " + schemaIdVersion); }
public static SchemaMetadataStorable fromSchemaMetadataInfo(SchemaMetadataInfo schemaMetadataInfo) { SchemaMetadata schemaMetadata = schemaMetadataInfo.getSchemaMetadata(); SchemaMetadataStorable schemaMetadataStorable = updateSchemaMetadata(new SchemaMetadataStorable(), schemaMetadata); schemaMetadataStorable.setId(schemaMetadataInfo.getId()); schemaMetadataStorable.setTimestamp(schemaMetadataInfo.getTimestamp()); return schemaMetadataStorable; }
@Override public byte[] serialize(String topic, StreamlineEvent streamlineEvent) { SchemaMetadata schemaMetadata = getSchemaKey(topic, false); SchemaVersionInfo schemaVersionInfo; try { schemaMetadata = schemaRegistryClient.getSchemaMetadataInfo(schemaMetadata.getName()).getSchemaMetadata(); if (writerSchemaVersion != null) { schemaVersionInfo = schemaRegistryClient.getSchemaVersionInfo(new SchemaVersionKey(schemaMetadata.getName(), writerSchemaVersion)); } else { schemaVersionInfo = schemaRegistryClient.getLatestSchemaVersionInfo(schemaMetadata.getName()); } } catch (SchemaNotFoundException e) { LOG.error("Exception occured while getting SchemaVersionInfo for " + schemaMetadata, e); throw new RuntimeException(e); } if (streamlineEvent == null || streamlineEvent.isEmpty()) { return null; } else { return avroSnapshotSerializer.serialize(getAvroRecord(streamlineEvent, new Schema.Parser().parse(schemaVersionInfo.getSchemaText())), schemaMetadata); } }
private SchemaVersionInfo retrieveSchemaVersionInfo(SchemaIdVersion key) throws SchemaNotFoundException { SchemaVersionInfo schemaVersionInfo = null; if (key.getSchemaVersionId() != null) { schemaVersionInfo = fetchSchemaVersionInfo(key.getSchemaVersionId()); } else if (key.getSchemaMetadataId() != null) { SchemaMetadataInfo schemaMetadataInfo = schemaMetadataFetcher.getSchemaMetadataInfo(key.getSchemaMetadataId()); Integer version = key.getVersion(); schemaVersionInfo = fetchSchemaVersionInfo(schemaMetadataInfo.getSchemaMetadata().getName(), version); } else { throw new IllegalArgumentException("Invalid SchemaIdVersion: " + key); } return schemaVersionInfo; }
public SchemaIdVersion addSchemaVersion(String schemaBranchName, SchemaMetadataInfo schemaMetadataInfo, SchemaVersion schemaVersion) throws SchemaNotFoundException, IncompatibleSchemaException, InvalidSchemaException, SchemaBranchNotFoundException { Preconditions.checkNotNull(schemaBranchName, "Schema branch name can't be null"); checkSchemaText(schemaVersion.getSchemaText()); SchemaVersionInfo schemaVersionInfo; // check whether there exists schema-metadata for schema-metadata-key SchemaMetadata schemaMetadata = schemaMetadataInfo.getSchemaMetadata(); // check whether the same schema text exists schemaVersionInfo = findSchemaVersion(schemaBranchName, schemaMetadata.getType(), schemaVersion.getSchemaText(), schemaMetadataInfo .getSchemaMetadata().getName()); if (schemaVersionInfo == null) { schemaVersionInfo = createSchemaVersion(schemaBranchName, schemaMetadata, schemaMetadataInfo.getId(), schemaVersion); } return new SchemaIdVersion(schemaMetadataInfo.getId(), schemaVersionInfo.getVersion(), schemaVersionInfo.getId()); }
private SchemaIdVersion handleSchemaIdVersionResponse(SchemaMetadataInfo schemaMetadataInfo, Response response) throws IncompatibleSchemaException, InvalidSchemaException { int status = response.getStatus(); String msg = response.readEntity(String.class); if (status == Response.Status.BAD_REQUEST.getStatusCode() || status == Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) { CatalogResponse catalogResponse = readCatalogResponse(msg); if (CatalogResponse.ResponseMessage.INCOMPATIBLE_SCHEMA.getCode() == catalogResponse.getResponseCode()) { throw new IncompatibleSchemaException(catalogResponse.getResponseMessage()); } else if (CatalogResponse.ResponseMessage.INVALID_SCHEMA.getCode() == catalogResponse.getResponseCode()) { throw new InvalidSchemaException(catalogResponse.getResponseMessage()); } else { throw new RuntimeException(catalogResponse.getResponseMessage()); } } Integer version = readEntity(msg, Integer.class); SchemaVersionInfo schemaVersionInfo = doGetSchemaVersionInfo(new SchemaVersionKey(schemaMetadataInfo.getSchemaMetadata() .getName(), version)); return new SchemaIdVersion(schemaMetadataInfo.getId(), version, schemaVersionInfo.getId()); }
@Override public Object deserialize(MessageAndMetadata context, Integer readerSchemaVersion) throws SerDesException { byte protocolId = retrieveProtocolId(context); SchemaIdVersion schemaIdVersion = retrieveSchemaIdVersion(protocolId, context); SchemaMetadata schemaMetadata; SchemaVersionInfo schemaVersionInfo; try { schemaVersionInfo = schemaRegistryClient.getSchemaVersionInfo(schemaIdVersion); schemaMetadata = schemaRegistryClient.getSchemaMetadataInfo(schemaVersionInfo.getName()).getSchemaMetadata(); } catch (Exception e) { throw new RegistryException(e); } return doDeserialize(context, protocolId, schemaMetadata, schemaVersionInfo.getVersion(), readerSchemaVersion); }
@Override public O deserialize(I input, Integer readerSchemaVersion) throws SerDesException { ensureInitialized(); // it can be enhanced to have respective protocol handlers for different versions byte protocolId = retrieveProtocolId(input); SchemaIdVersion schemaIdVersion = retrieveSchemaIdVersion(protocolId, input); SchemaVersionInfo schemaVersionInfo; SchemaMetadata schemaMetadata; try { schemaVersionInfo = schemaRegistryClient.getSchemaVersionInfo(schemaIdVersion); schemaMetadata = schemaRegistryClient.getSchemaMetadataInfo(schemaVersionInfo.getName()).getSchemaMetadata(); } catch (Exception e) { throw new RegistryException(e); } return doDeserialize(input, protocolId, schemaMetadata, schemaVersionInfo.getVersion(), readerSchemaVersion); }
@Test public void testSchemaMetadataOps() throws Exception { for (SchemaCompatibility schemaCompatibility : SchemaCompatibility.values()) { SchemaMetadata schemaMetadata = new SchemaMetadata.Builder("compatibility-" + schemaCompatibility) .type(AvroSchemaProvider.TYPE) .description("devices schema") .compatibility(schemaCompatibility) .schemaGroup(SCHEMA_GROUP).build(); Long schemaMetadataId = schemaRegistry.registerSchemaMetadata(schemaMetadata); SchemaMetadata schemaMetadataReturned = schemaRegistry.getSchemaMetadataInfo(schemaMetadataId).getSchemaMetadata(); Assert.assertEquals(schemaMetadata, schemaMetadataReturned); } }
@Test public void testHASanity() throws Exception { SchemaRegistryTestServerClientWrapper followerServer = followerSchemaRegistryServer(); SchemaRegistryClient schemaRegistryClient = followerServer.getClient(); // registering schema metadata on follower, this should have been redirected to leader. String schemaName = "foo"; SchemaMetadata schemaMetadata = new SchemaMetadata.Builder(schemaName).type("avro").build(); Long schemaId = schemaRegistryClient.registerSchemaMetadata(schemaMetadata); Assert.assertNotNull(schemaId); // registering a new schema on follower, this should have been redirected to leader. String schema1 = IOUtils.toString(LocalRegistryServerTest.class.getResourceAsStream("/schema-1.avsc"), "UTF-8"); SchemaIdVersion v1 = schemaRegistryClient.addSchemaVersion(schemaName, new SchemaVersion(schema1, "Initial version of the schema")); // retrieve schema on leader as the schema data is stored in memory in leader. this data does not exist on // followers as the storage is inmemory. SchemaRegistryTestServerClientWrapper leaderServer = leaderSchemaRegistryServer(); int leaderPort = leaderServer.getLocalPort(); SchemaRegistryClient leaderClient = leaderServer.getClient(); SchemaMetadataInfo schemaMetadataInfo = leaderClient.getSchemaMetadataInfo(schemaName); Assert.assertEquals(schemaMetadata, schemaMetadataInfo.getSchemaMetadata()); // stop the leader server leaderServer.stopTestServer(); // get the new leader server and run operations. SchemaRegistryTestServerClientWrapper newLeaderServer = leaderSchemaRegistryServer(); Assert.assertNotEquals(leaderPort, newLeaderServer.getLocalPort()); leaderClient = newLeaderServer.getClient(); String receivedSchema = leaderClient.getSchemaVersionInfo(new SchemaVersionKey(schemaName, v1.getVersion())).getSchemaText(); Assert.assertEquals(schema1, receivedSchema); }
@Test public void testValidationLevels() throws Exception { SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BOTH); String schemaName = schemaMetadata.getName(); Long id = SCHEMA_REGISTRY_CLIENT.registerSchemaMetadata(schemaMetadata); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-1.avsc"), "Initial version of the schema")); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-2.avsc"), "Second version of the schema")); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-3.avsc"), "Third version of the schema, removes name field")); try { SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-4.avsc"), "Forth version of the schema, adds back name field, but different type")); Assert.fail("Should throw IncompatibleSchemaException as check against all schema's would find name field is not compatible with v1 and v2"); } catch (IncompatibleSchemaException ise) { //expected } SchemaMetadata currentSchemaMetadata = SCHEMA_REGISTRY_CLIENT.getSchemaMetadataInfo(schemaName) .getSchemaMetadata(); SchemaMetadata schemaMetadataToUpdateTo = new SchemaMetadata.Builder(currentSchemaMetadata).validationLevel(SchemaValidationLevel.LATEST) .build(); SchemaMetadataInfo updatedSchemaMetadata = SCHEMA_REGISTRY_CLIENT.updateSchemaMetadata(schemaName, schemaMetadataToUpdateTo); Assert.assertEquals(SchemaValidationLevel.LATEST, updatedSchemaMetadata.getSchemaMetadata() .getValidationLevel()); try { SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-4.avsc"), "Forth version of the schema, adds back name field, but different type")); } catch (IncompatibleSchemaException ise) { Assert.fail("Should not throw IncompatibleSchemaException as check against only latest schema as such should ignore v1 and v2"); } }