public AbstractDataset(Class<E> type, Schema schema) { this.type = DataModelUtil.resolveType(type, schema); this.schema = DataModelUtil.getReaderSchema(this.type, schema); }
public CSVRecordParser(CSVProperties props, Schema schema, Class<E> type, @Nullable List<String> header) { this.parser = CSVUtil.newParser(props); this.builder = new CSVRecordBuilder<E>( DataModelUtil.getReaderSchema(type, schema), type, getHeader(props, header)); }
EntityAccessor(Class<E> type, Schema schema) { this.type = DataModelUtil.resolveType(type, schema); this.schema = DataModelUtil.getReaderSchema(this.type, schema); this.writeSchema = DataModelUtil.getWriterSchema(this.type, this.schema); this.model = DataModelUtil.getDataModelForType(this.type); }
/** * Get the DatumReader for the given type. * * @param <E> The entity type * @param type The Java class of the entity type * @param writerSchema The {@link Schema} for entities * @return The DatumReader for the given type */ @SuppressWarnings("unchecked") public static <E> DatumReader<E> getDatumReaderForType(Class<E> type, Schema writerSchema) { Schema readerSchema = getReaderSchema(type, writerSchema); GenericData dataModel = getDataModelForType(type); if (dataModel instanceof ReflectData) { return new ReflectDatumReader<E>(writerSchema, readerSchema, (ReflectData)dataModel); } else if (dataModel instanceof SpecificData) { return new SpecificDatumReader<E>(writerSchema, readerSchema, (SpecificData)dataModel); } else { return new GenericDatumReader<E>(writerSchema, readerSchema, dataModel); } }
public ParquetFileSystemDatasetReader(FileSystem fileSystem, Path path, Schema schema, Class<E> type) { Preconditions.checkArgument(fileSystem != null, "FileSystem cannot be null"); Preconditions.checkArgument(path != null, "Path cannot be null"); Preconditions.checkArgument(schema != null, "Schema cannot be null"); Preconditions.checkArgument(IndexedRecord.class.isAssignableFrom(type) || (Class<?>)type == Object.class, "The entity type must implement IndexedRecord"); this.fileSystem = fileSystem; this.path = path; this.schema = schema; this.type = type; this.readerSchema = DataModelUtil.getReaderSchema(type, schema); this.state = ReaderWriterState.NEW; }
@Test public void testGetReaderSchemaForGeneric() { Class<GenericData.Record> type = GenericData.Record.class; Schema writerSchema = StandardEvent.getClassSchema(); Schema expResult = writerSchema; Schema result = DataModelUtil.getReaderSchema(type, writerSchema); assertEquals(expResult, result); }
@Test public void testGetReaderSchemaForSpecific() { Class<StandardEvent> type = StandardEvent.class; Schema writerSchema = StandardEvent.getClassSchema(); Schema expResult = writerSchema; Schema result = DataModelUtil.getReaderSchema(type, writerSchema); assertEquals(expResult, result); }
@Test public void testGetReaderSchemaForCompatibleSpecific() { Class<SmallEvent> type = SmallEvent.class; Schema writerSchema = StandardEvent.getClassSchema(); Schema expResult = SmallEvent.getClassSchema(); Schema result = DataModelUtil.getReaderSchema(type, writerSchema); assertEquals(expResult, result); }
@Test public void testGetReaderSchemaForReflect() { Class<String> type = String.class; Schema writerSchema = ReflectData.get().getSchema(String.class); Schema expResult = writerSchema; Schema result = DataModelUtil.getReaderSchema(type, writerSchema); assertEquals(expResult, result); }
@Test public void testGetReaderSchemaForCompatibleReflect() { Class<ReflectSmallEvent> type = ReflectSmallEvent.class; Schema writerSchema = ReflectData.get().getSchema(ReflectStandardEvent.class); Schema expResult = DataModelUtil.AllowNulls.get().getSchema(ReflectSmallEvent.class); Schema result = DataModelUtil.getReaderSchema(type, writerSchema); assertEquals(expResult, result); }