@Override public SchemaIdVersion addSchemaVersion(final String schemaName, final SchemaVersion schemaVersion) throws InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, SchemaBranchNotFoundException { return addSchemaVersion(SchemaBranch.MASTER_BRANCH, schemaName, schemaVersion); }
@Override public SchemaIdVersion addSchemaVersion(SchemaMetadata schemaMetadata, SchemaVersion schemaVersion) throws InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, SchemaBranchNotFoundException { return addSchemaVersion(SchemaBranch.MASTER_BRANCH, schemaMetadata, schemaVersion); }
@Override public SchemaIdVersion addSchemaVersion(String schemaBranchName, SchemaMetadata schemaMetadata, SchemaVersion schemaVersion) throws InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, SchemaBranchNotFoundException { // get it, if it exists in cache SchemaDigestEntry schemaDigestEntry = buildSchemaTextEntry(schemaVersion, schemaMetadata.getName()); SchemaIdVersion schemaIdVersion = schemaTextCache.getIfPresent(schemaDigestEntry); if (schemaIdVersion == null) { //register schema metadata if it does not exist Long metadataId = registerSchemaMetadata(schemaMetadata); if (metadataId == null) { LOG.error("Schema Metadata [{}] is not registered successfully", schemaMetadata); throw new RuntimeException("Given SchemaMetadata could not be registered: " + schemaMetadata); } // add schemaIdVersion schemaIdVersion = addSchemaVersion(schemaBranchName, schemaMetadata.getName(), schemaVersion); } return schemaIdVersion; }
schemaIdVersion = schemaRegistryClient.addSchemaVersion(schemaName, avroSchemaVersion); } catch (InvalidSchemaException e) { String errMsg = String.format("Invalid schema received for schema with name [%s] : [%s]", schemaName, streamsSchemaText);
public void runCustomSerDesApi() throws Exception { // upload jar file String serdesJarName = "/serdes-examples.jar"; InputStream serdesJarInputStream = SampleSchemaRegistryClientApp.class.getResourceAsStream(serdesJarName); if (serdesJarInputStream == null) { throw new RuntimeException("Jar " + serdesJarName + " could not be loaded"); } String fileId = schemaRegistryClient.uploadFile(serdesJarInputStream); SchemaMetadata schemaMetadata = createSchemaMetadata("serdes-device-" + System.currentTimeMillis()); SchemaIdVersion v1 = schemaRegistryClient.addSchemaVersion(schemaMetadata, new SchemaVersion(getSchema("/device.avsc"), "Initial version of the schema")); // register serializer/deserializer Long serDesId = registerSimpleSerDes(fileId); // map serializer and deserializer with schemakey // for each schema, one serializer/deserializer is sufficient unless someone want to maintain multiple implementations of serializers/deserializers String schemaName = schemaMetadata.getName(); schemaRegistryClient.mapSchemaWithSerDes(schemaName, serDesId); SnapshotSerializer<Object, byte[], SchemaMetadata> snapshotSerializer = getSnapshotSerializer(schemaMetadata); String payload = "Random text: " + new Random().nextLong(); byte[] serializedBytes = snapshotSerializer.serialize(payload, schemaMetadata); SnapshotDeserializer<byte[], Object, Integer> snapshotdeserializer = getSnapshotDeserializer(schemaMetadata); Object deserializedObject = snapshotdeserializer.deserialize(serializedBytes, null); LOG.info("Given payload and deserialized object are equal: " + payload.equals(deserializedObject)); }
SchemaIdVersion v1 = schemaRegistryClient.addSchemaVersion(schemaMetadata, new SchemaVersion(schema1, "Initial version of the schema")); LOG.info("Registered schema [{}] and returned version [{}]", schema1, v1); SchemaIdVersion v2 = schemaRegistryClient.addSchemaVersion(schemaMetadata, schemaInfo2); LOG.info("Registered schema [{}] and returned version [{}]", schema2, v2); SchemaIdVersion version = schemaRegistryClient.addSchemaVersion(schemaMetadata, schemaInfo2); LOG.info("Received version [{}] for schema metadata [{}]", version, schemaMetadata);
private SchemaIdVersion addSchemaVersion(String schemaBranchName, SchemaMetadata schemaMetadata, String pathToSchema) throws IOException, SchemaNotFoundException, InvalidSchemaException, IncompatibleSchemaException { String schema = AvroSchemaRegistryClientUtil.getSchema(pathToSchema); SchemaIdVersion schemaIdVersion = schemaRegistryClient.addSchemaVersion(schemaBranchName, schemaMetadata, new SchemaVersion(schema, "schema version description")); return schemaIdVersion; }
@Test(expected = IncompatibleSchemaException.class) public void testIncompatibleSchemas() throws Exception { String schema = AvroSchemaRegistryClientUtil.getSchema("/device.avsc"); String incompatSchema = AvroSchemaRegistryClientUtil.getSchema("/device-incompat.avsc"); SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BACKWARD); // registering a new schema SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(schema, "Initial version of the schema")); // adding a new version of the schema SchemaVersion incompatSchemaInfo = new SchemaVersion(incompatSchema, "second version"); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, incompatSchemaInfo); }
@Test(expected = InvalidSchemaException.class) public void testInvalidSchema() throws Exception { String schema = "--- invalid schema ---"; SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BACKWARD); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(schema, "Initial version of the schema")); }
@Test public void testSerializerOps() throws Exception { String fileId = uploadFile(); SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BOTH); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/device.avsc"), "Initial version of the schema")); SerDesPair serDesPair = createSerDesInfo(fileId); Long serDesId = SCHEMA_REGISTRY_CLIENT.addSerDes(serDesPair); Assert.assertNotNull("Returned serDesId can not be null", serDesId); String schemaName = schemaMetadata.getName(); SCHEMA_REGISTRY_CLIENT.mapSchemaWithSerDes(schemaName, serDesId); Collection<SerDesInfo> serializers = SCHEMA_REGISTRY_CLIENT.getSerDes(schemaName); Assert.assertTrue(serializers.stream() .map(x -> x.getSerDesPair()) .collect(Collectors.toList()) .contains(serDesPair)); }
private void _testAvroSerDesGenericObj(Byte protocolId) throws IOException, InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, SchemaBranchNotFoundException { Map<String, Object> config = Maps.newHashMap(); config.putAll(SCHEMA_REGISTRY_CLIENT_CONF); config.put(SERDES_PROTOCOL_VERSION, protocolId); AvroSnapshotSerializer avroSnapshotSerializer = new AvroSnapshotSerializer(); avroSnapshotSerializer.init(config); AvroSnapshotDeserializer avroSnapshotDeserializer = new AvroSnapshotDeserializer(); avroSnapshotDeserializer.init(config); String deviceSchema = AvroSchemaRegistryClientUtil.getSchema("/device.avsc"); SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BOTH); SchemaIdVersion v1 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(deviceSchema, "Initial version of the schema")); Assert.assertNotNull(v1); Object deviceObject = AvroSchemaRegistryClientUtil.createGenericRecordForDevice(); byte[] serializedData = avroSnapshotSerializer.serialize(deviceObject, schemaMetadata); Object deserializedObj = avroSnapshotDeserializer.deserialize(new ByteArrayInputStream(serializedData), null); Assert.assertEquals(deviceObject, deserializedObj); }
private SchemaVersionKey addAndDeleteSchemaVersion(String schemaName) throws InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, IOException, SchemaBranchNotFoundException, SchemaLifecycleException { SchemaMetadata schemaMetadata = createSchemaMetadata(schemaName, SchemaCompatibility.BOTH); SchemaIdVersion schemaIdVersion = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(AvroSchemaRegistryClientUtil .getSchema("/device.avsc"), "Initial version of the schema")); SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaMetadata.getName(), schemaIdVersion.getVersion()); SCHEMA_REGISTRY_CLIENT.deleteSchemaVersion(schemaVersionKey); return schemaVersionKey; }
@Test public void testValidationLevels() throws Exception { SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BOTH); String schemaName = schemaMetadata.getName(); Long id = SCHEMA_REGISTRY_CLIENT.registerSchemaMetadata(schemaMetadata); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-1.avsc"), "Initial version of the schema")); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-2.avsc"), "Second version of the schema")); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-3.avsc"), "Third version of the schema, removes name field")); try { SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-4.avsc"), "Forth version of the schema, adds back name field, but different type")); Assert.fail("Should throw IncompatibleSchemaException as check against all schema's would find name field is not compatible with v1 and v2"); } catch (IncompatibleSchemaException ise) { //expected } SchemaMetadata currentSchemaMetadata = SCHEMA_REGISTRY_CLIENT.getSchemaMetadataInfo(schemaName) .getSchemaMetadata(); SchemaMetadata schemaMetadataToUpdateTo = new SchemaMetadata.Builder(currentSchemaMetadata).validationLevel(SchemaValidationLevel.LATEST) .build(); SchemaMetadataInfo updatedSchemaMetadata = SCHEMA_REGISTRY_CLIENT.updateSchemaMetadata(schemaName, schemaMetadataToUpdateTo); Assert.assertEquals(SchemaValidationLevel.LATEST, updatedSchemaMetadata.getSchemaMetadata() .getValidationLevel()); try { SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-4.avsc"), "Forth version of the schema, adds back name field, but different type")); } catch (IncompatibleSchemaException ise) { Assert.fail("Should not throw IncompatibleSchemaException as check against only latest schema as such should ignore v1 and v2"); } }
@Test public void testGetSchemaVersionFromStates() throws IOException, SchemaNotFoundException, InvalidSchemaException, IncompatibleSchemaException, SchemaLifecycleException, SchemaBranchAlreadyExistsException { SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.NONE); String schemaName = schemaMetadata.getName(); Long id = SCHEMA_REGISTRY_CLIENT.registerSchemaMetadata(schemaMetadata); SchemaIdVersion v1 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-1.avsc"), "Initial version of the schema")); SchemaIdVersion v2 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-2.avsc"), "Second version of the schema")); SchemaIdVersion v3 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-3.avsc"), "Third version of the schema, removes name field")); SchemaBranch schemaBranch = SCHEMA_REGISTRY_CLIENT.createSchemaBranch(v3.getSchemaVersionId(), new SchemaBranch("Branch-1", schemaName)); SchemaIdVersion v4 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaBranch.getName(), schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-4.avsc"), "Forth version of the schema, adds back name field, but different type")); SCHEMA_REGISTRY_CLIENT.startSchemaVersionReview(v4.getSchemaVersionId()); SCHEMA_REGISTRY_CLIENT.transitionState(v4.getSchemaVersionId(), SchemaVersionLifecycleStates.REVIEWED.getId(), null); SCHEMA_REGISTRY_CLIENT.archiveSchemaVersion(v2.getSchemaVersionId()); SCHEMA_REGISTRY_CLIENT.archiveSchemaVersion(v3.getSchemaVersionId()); Assert.assertEquals( transformToSchemaIdVersions(SCHEMA_REGISTRY_CLIENT.getAllVersions(SchemaBranch.MASTER_BRANCH, schemaName, Collections.singletonList(SchemaVersionLifecycleStates.ENABLED.getId()))), new HashSet<>(Arrays.asList(v1))); Assert.assertEquals( transformToSchemaIdVersions(SCHEMA_REGISTRY_CLIENT.getAllVersions(SchemaBranch.MASTER_BRANCH, schemaName, Collections.singletonList(SchemaVersionLifecycleStates.ARCHIVED.getId()))), new HashSet<>(Arrays.asList(v2,v3))); Assert.assertEquals( transformToSchemaIdVersions(SCHEMA_REGISTRY_CLIENT.getAllVersions(schemaBranch.getName(), schemaName, Collections.singletonList(SchemaVersionLifecycleStates.REVIEWED.getId()))), new HashSet<>(Arrays.asList(v4))); SCHEMA_REGISTRY_CLIENT.disableSchemaVersion(v1.getSchemaVersionId()); Assert.assertEquals( transformToSchemaIdVersions(SCHEMA_REGISTRY_CLIENT.getAllVersions(SchemaBranch.MASTER_BRANCH, schemaName, Arrays.asList(SchemaVersionLifecycleStates.ARCHIVED.getId(), SchemaVersionLifecycleStates.DISABLED.getId()))), new HashSet<>(Arrays.asList(v1,v2,v3))); }
@Test public void testSanity() throws Exception { SchemaRegistryTestServerClientWrapper schemaRegistryTestServerClientWrapper = new SchemaRegistryTestServerClientWrapper(SCHEMA_REGISTRY_TEST_CONFIGURATION); schemaRegistryTestServerClientWrapper.startTestServer(); SchemaRegistryClient schemaRegistryClient = schemaRegistryTestServerClientWrapper.getClient(); // registering schema metadata SchemaMetadata schemaMetadata = new SchemaMetadata.Builder("foo").type("avro").build(); Long schemaId = schemaRegistryClient.registerSchemaMetadata(schemaMetadata); Assert.assertNotNull(schemaId); // registering a new schema String schemaName = schemaMetadata.getName(); String schema1 = IOUtils.toString(LocalRegistryServerTest.class.getResourceAsStream("/schema-1.avsc"), "UTF-8"); SchemaIdVersion v1 = schemaRegistryClient.addSchemaVersion(schemaName, new SchemaVersion(schema1, "Initial version of the schema")); schemaRegistryTestServerClientWrapper.stopTestServer(); } }
@Test (expected = InvalidSchemaException.class) public void addInvalidSchemaToBranch() throws IOException, SchemaBranchNotFoundException, InvalidSchemaException, SchemaNotFoundException, IncompatibleSchemaException, SchemaBranchAlreadyExistsException { SchemaMetadata schemaMetadata = addSchemaMetadata(testNameRule.getMethodName(), SchemaCompatibility.BACKWARD); SchemaIdVersion masterSchemaIdVersion = addSchemaVersion(SchemaBranch.MASTER_BRANCH, schemaMetadata, "/device.avsc"); SchemaBranch schemaBranch1 = addSchemaBranch("BRANCH1", schemaMetadata, masterSchemaIdVersion.getSchemaVersionId()); String schema2 = "--- invalid schema ---"; schemaRegistryClient.addSchemaVersion(schemaBranch1.getName(), schemaMetadata.getName(), new SchemaVersion(schema2, "second version")); }
result = new SchemaMetadataInfo(schemaMetadata); minTimes=0; maxTimes=1; mockSchemaRegistryClient.addSchemaVersion(withInstanceOf(SchemaMetadata.class), withInstanceOf(SchemaVersion.class)); result = schemaIdVersion; minTimes=0; maxTimes=1;
mockSchemaRegistryClient.addSchemaVersion(withInstanceOf(SchemaMetadata.class), withInstanceOf(SchemaVersion.class)); result = schemaIdVersion;
public void doTestSchemaVersionEnableState(String schemaName) throws Exception { String schema = AvroSchemaRegistryClientUtil.getSchema("/device.avsc"); String incompatSchema = AvroSchemaRegistryClientUtil.getSchema("/device-compat.avsc"); SchemaMetadata schemaMetadata = createSchemaMetadata(schemaName, SchemaCompatibility.BACKWARD); // registering a new schema SchemaIdVersion schemaIdVersion_1 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(schema, "Initial version of the schema")); // adding a new version of the schema SchemaVersion incompatSchemaInfo = new SchemaVersion(incompatSchema, "second version"); SchemaIdVersion schemaIdVersion_2 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, incompatSchemaInfo); Long schemaVersionId_2 = schemaIdVersion_2.getSchemaVersionId(); SCHEMA_REGISTRY_CLIENT.disableSchemaVersion(schemaVersionId_2); SchemaVersionInfo schemaVersionInfo = SCHEMA_REGISTRY_CLIENT.getSchemaVersionInfo(schemaIdVersion_2); System.out.println("disable :: schemaVersionInfo.getStateId() = " + schemaVersionInfo.getStateId()); Assert.assertEquals(SchemaVersionLifecycleStates.DISABLED.getId(), schemaVersionInfo.getStateId()); SCHEMA_REGISTRY_CLIENT.enableSchemaVersion(schemaVersionId_2); schemaVersionInfo = SCHEMA_REGISTRY_CLIENT.getSchemaVersionInfo(schemaIdVersion_2); System.out.println("enable :: schemaVersionInfo.getStateId() = " + schemaVersionInfo.getStateId()); Assert.assertEquals(SchemaVersionLifecycleStates.ENABLED.getId(), schemaVersionInfo.getStateId()); SCHEMA_REGISTRY_CLIENT.disableSchemaVersion(schemaVersionId_2); schemaVersionInfo = SCHEMA_REGISTRY_CLIENT.getSchemaVersionInfo(schemaIdVersion_2); System.out.println("disable :: schemaVersionInfo.getStateId() = " + schemaVersionInfo.getStateId()); Assert.assertEquals(SchemaVersionLifecycleStates.DISABLED.getId(), schemaVersionInfo.getStateId()); SCHEMA_REGISTRY_CLIENT.archiveSchemaVersion(schemaVersionId_2); schemaVersionInfo = SCHEMA_REGISTRY_CLIENT.getSchemaVersionInfo(schemaIdVersion_2); System.out.println("archive :: schemaVersionInfo.getStateId() = " + schemaVersionInfo.getStateId()); Assert.assertEquals(SchemaVersionLifecycleStates.ARCHIVED.getId(), schemaVersionInfo.getStateId()); }
@Test public void testHASanity() throws Exception { SchemaRegistryTestServerClientWrapper followerServer = followerSchemaRegistryServer(); SchemaRegistryClient schemaRegistryClient = followerServer.getClient(); // registering schema metadata on follower, this should have been redirected to leader. String schemaName = "foo"; SchemaMetadata schemaMetadata = new SchemaMetadata.Builder(schemaName).type("avro").build(); Long schemaId = schemaRegistryClient.registerSchemaMetadata(schemaMetadata); Assert.assertNotNull(schemaId); // registering a new schema on follower, this should have been redirected to leader. String schema1 = IOUtils.toString(LocalRegistryServerTest.class.getResourceAsStream("/schema-1.avsc"), "UTF-8"); SchemaIdVersion v1 = schemaRegistryClient.addSchemaVersion(schemaName, new SchemaVersion(schema1, "Initial version of the schema")); // retrieve schema on leader as the schema data is stored in memory in leader. this data does not exist on // followers as the storage is inmemory. SchemaRegistryTestServerClientWrapper leaderServer = leaderSchemaRegistryServer(); int leaderPort = leaderServer.getLocalPort(); SchemaRegistryClient leaderClient = leaderServer.getClient(); SchemaMetadataInfo schemaMetadataInfo = leaderClient.getSchemaMetadataInfo(schemaName); Assert.assertEquals(schemaMetadata, schemaMetadataInfo.getSchemaMetadata()); // stop the leader server leaderServer.stopTestServer(); // get the new leader server and run operations. SchemaRegistryTestServerClientWrapper newLeaderServer = leaderSchemaRegistryServer(); Assert.assertNotEquals(leaderPort, newLeaderServer.getLocalPort()); leaderClient = newLeaderServer.getClient(); String receivedSchema = leaderClient.getSchemaVersionInfo(new SchemaVersionKey(schemaName, v1.getVersion())).getSchemaText(); Assert.assertEquals(schema1, receivedSchema); }