@Override public void process(InputStream in) throws IOException { try (DataFileStream<Record> stream = new DataFileStream<>( in, AvroUtil.newDatumReader(schema, Record.class))) { IncompatibleSchemaException.check( SchemaValidationUtil.canRead(stream.getSchema(), schema), "Incompatible file schema %s, expected %s", stream.getSchema(), schema); long written = 0L; try (DatasetWriter<Record> writer = target.newWriter()) { for (Record record : stream) { writer.write(record); written += 1; } } finally { session.adjustCounter("Stored records", written, true /* cannot roll back the write */); } } } });
@Override public boolean compatible(EntitySchema entitySchema) { if (!mappingCompatible(this, entitySchema)) { return false; } AvroEntitySchema avroEntitySchema = (AvroEntitySchema) entitySchema; if (!SchemaValidationUtil.canRead(schema, avroEntitySchema.getAvroSchema())) { return false; } if (!SchemaValidationUtil.canRead(avroEntitySchema.getAvroSchema(), schema)) { return false; } return true; }
@Override public boolean compatible(EntitySchema entitySchema) { if (!mappingCompatible(this, entitySchema)) { return false; } AvroEntitySchema avroEntitySchema = (AvroEntitySchema) entitySchema; if (!SchemaValidationUtil.canRead(schema, avroEntitySchema.getAvroSchema())) { return false; } if (!SchemaValidationUtil.canRead(avroEntitySchema.getAvroSchema(), schema)) { return false; } return true; }
if (!SchemaValidationUtil.canRead(oldSchema, schema)) { throw new IncompatibleSchemaException("Schema cannot read data " + "written using existing schema. Schema: " +
protected AbstractRefinableView(AbstractRefinableView<?> view, Schema schema, Class<E> type) { if (view.dataset instanceof AbstractDataset) { this.dataset = ((AbstractDataset<?>) view.dataset).asType(type); } else { this.dataset = Datasets.load(view.dataset.getUri(), type); } this.comparator = view.comparator; this.constraints = view.constraints; // thread-safe, so okay to reuse when views share a partition strategy this.keys = view.keys; // Resolve our type according to the given schema this.accessor = DataModelUtil.accessor(type, schema); this.entityTest = constraints.toEntityPredicate(accessor); Schema datasetSchema = dataset.getDescriptor().getSchema(); this.canRead = SchemaValidationUtil.canRead( datasetSchema, accessor.getReadSchema()); this.canWrite = SchemaValidationUtil.canRead( accessor.getWriteSchema(), datasetSchema); IncompatibleSchemaException.check(canRead || canWrite, "The type cannot be used to read from or write to the dataset:\n" + "Type schema: %s\nDataset schema: %s", getSchema(), datasetSchema); }
/** * Checks that the {@code existing} {@link DatasetDescriptor} is compatible * with {@code test}. * * @param existing the current {@code DatasetDescriptor} for a dataset * @param test a new {@code DatasetDescriptor} for the same dataset */ public static void checkCompatible(DatasetDescriptor existing, DatasetDescriptor test) { checkNotChanged("format", existing.getFormat(), test.getFormat()); checkNotChanged("partitioning", existing.isPartitioned(), test.isPartitioned()); if (existing.isPartitioned()) { checkStrategyUpdate( existing.getPartitionStrategy(), test.getPartitionStrategy(), test.getSchema()); } // check can read records written with old schema using new schema Schema oldSchema = existing.getSchema(); Schema testSchema = test.getSchema(); if (!SchemaValidationUtil.canRead(oldSchema, testSchema)) { throw new IncompatibleSchemaException("Schema cannot read data " + "written using existing schema. Schema: " + testSchema.toString(true) + "\nExisting schema: " + oldSchema.toString(true)); } }
@Override public void process(InputStream in) throws IOException { try (DataFileStream<Record> stream = new DataFileStream<>( in, AvroUtil.newDatumReader(schema, Record.class))) { IncompatibleSchemaException.check( SchemaValidationUtil.canRead(stream.getSchema(), schema), "Incompatible file schema %s, expected %s", stream.getSchema(), schema); long written = 0L; try (DatasetWriter<Record> writer = target.newWriter()) { for (Record record : stream) { writer.write(record); written += 1; } } finally { session.adjustCounter("Stored records", written, true /* cannot roll back the write */); } } } });
schema = Schemas.fromAvsc(open(avroSchemaFile)); ValidationException.check( SchemaValidationUtil.canRead(existingSchema, schema), "Schema %s cannot read existing data schema: %s", avroSchemaFile, existingSchema.toString(true));
@Test public void testAllAvroTypes() { Schema r = SchemaBuilder.record("r").fields() .requiredBoolean("boolF") .requiredInt("intF") .requiredLong("longF") .requiredFloat("floatF") .requiredDouble("doubleF") .requiredString("stringF") .requiredBytes("bytesF") .name("fixedF1").type().fixed("F1").size(1).noDefault() .name("enumF").type().enumeration("E1").symbols("S").noDefault() .name("mapF").type().map().values().stringType().noDefault() .name("arrayF").type().array().items().stringType().noDefault() .name("recordF").type().record("inner").fields() .name("f").type().intType().noDefault() .endRecord().noDefault() .optionalBoolean("boolO") .endRecord(); assertTrue(SchemaValidationUtil.canRead(r, r)); } }