/** * Creates a new instance. * @param dataClass the data model class * @param dataFormat the data format * @param <T> the data type * @return the created instance */ public static <T> DataDefinition<T> newInstance( Class<?> dataClass, DataFormat<T> dataFormat) { return newInstance(dataClass, dataFormat, null); }
/** * Creates a new instance. * @param format the format. */ public HadoopDataSourceCoreTest(DataFormat<StringBuilder> format) { this.definition = SimpleDataDefinition.newInstance(StringBuilder.class, format); }
private DataDefinition<?> createDataDefinition(Configuration configuration, DirectInputGroup group) { DataFormat<?> format = ReflectionUtils.newInstance(group.formatClass, configuration); DataFilter<?> filter = createFilter(group.filterClass, configuration); DataDefinition<?> definition = SimpleDataDefinition.newInstance(group.dataType, format, filter); return definition; }
private <T> DataDefinition<T> createDataDefinition(Class<? extends DataFormat<T>> formatClass) { DataFormat<T> format = ReflectionUtils.newInstance(formatClass, getConf()); return SimpleDataDefinition.newInstance(format.getSupportedType(), format); }
DataDefinition<T> definition = SimpleDataDefinition.newInstance(dataType, format); Counter counter = new Counter(); try {
description.getClass().getName())); DataDefinition<T> definition = SimpleDataDefinition.newInstance(dataType, format); try { dataSource.setupTransactionOutput(outputContext.getTransactionContext());
String.format("%04d", context.getTaskAttemptID().getTaskID().getId())); //$NON-NLS-1$ String resourcePath = variables.parse(unresolvedResourcePath); DataDefinition<? super T> definition = SimpleDataDefinition.newInstance(dataType, format);
try (ModelOutput<StringBuilder> output = ds.openOutput( aContext, SimpleDataDefinition.newInstance(StringBuilder.class, new MockFormat()), "", executionId,
@SuppressWarnings({ "unchecked", "rawtypes" }) @Override protected void reduce( AbstractDirectOutputKey key, Iterable<AbstractDirectOutputValue> values, Context context) throws IOException , InterruptedException { DirectOutputGroup group = (DirectOutputGroup) key.getGroupObject().getObject(); String portId = group.getOutputId(); String path = variables.parse(group.getPath(), false); String sourceId = repository.getRelatedId(path); OutputAttemptContext outputContext = BridgeOutputFormat.createContext(context, sourceId); DataDefinition definition = SimpleDataDefinition.newInstance( group.getDataType(), configure(context, group.getFormat())); DirectDataSource datasource = repository.getRelatedDataSource(path); String basePath = repository.getComponentPath(path); String resourcePath = variables.parse(group.getResourcePath()); Counter counter = new Counter(); long records = 0; try (ModelOutput output = datasource.openOutput(outputContext, definition, basePath, resourcePath, counter)) { for (Union union : values) { Object object = union.getObject(); output.write(object); records++; } } recordCounter.increment(records); Constants.putCounts(context, sourceId, portId, 1, records, counter.get()); }