/** * Build up an aggregation function over the input elements. * * <p>This method specifies an aggregation over multiple fields of the input. The union of all * calls to aggregateField and aggregateFields will determine the output schema. */ public <CombineInputT, AccumT, CombineOutputT> CombineFieldsGlobally<InputT> aggregateFields( FieldAccessDescriptor fieldsToAggregate, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, Field outputField) { return new CombineFieldsGlobally<>( SchemaAggregateFn.<InputT>create().aggregateFields(fieldsToAggregate, fn, outputField)); }
/** * Build up an aggregation function over the input elements. * * <p>This method specifies an aggregation over multiple fields of the input. The union of all * calls to aggregateField and aggregateFields will determine the output schema. */ public <CombineInputT, AccumT, CombineOutputT> CombineFieldsByFields<InputT> aggregateFields( FieldAccessDescriptor fieldsToAggregate, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, Field outputField) { return new CombineFieldsByFields<>( this, SchemaAggregateFn.<InputT>create().aggregateFields(fieldsToAggregate, fn, outputField)); }
/** * Build up an aggregation function over the input elements. * * <p>This method specifies an aggregation over multiple fields of the input. The union of all * calls to aggregateField and aggregateFields will determine the output schema. */ public <CombineInputT, AccumT, CombineOutputT> CombineFieldsGlobally<InputT> aggregateFields( FieldAccessDescriptor fieldAccessDescriptor, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, Field outputField) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields(fieldAccessDescriptor, fn, outputField)); }
public static <T> Cast<T> narrowing(Schema outputSchema) { return new AutoValue_Cast<>(outputSchema, Narrowing.of()); }
public static <T> Inner<T> create() { return new AutoValue_Unnest_Inner.Builder<T>().setFieldNameFunction(CONCAT_FIELD_NAMES).build(); } /**
static <T> Inner<T> create() { return new AutoValue_SchemaAggregateFn_Inner.Builder<T>() .setFieldAggregations(Lists.newArrayList()) .build(); }
/** * Build up an aggregation function over the input elements. * * <p>This method specifies an aggregation over multiple fields of the input. The union of all * calls to aggregateField and aggregateFields will determine the output schema. */ public <CombineInputT, AccumT, CombineOutputT> CombineFieldsByFields<InputT> aggregateFields( FieldAccessDescriptor fieldsToAggregate, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, Field outputField) { return new CombineFieldsByFields<>( byFields, schemaAggregateFn.aggregateFields(fieldsToAggregate, fn, outputField)); }
/** * Convert a {@link PCollection}{@literal <Row>} into a {@link PCollection}{@literal <OutputT>}. * * <p>The output schema will be inferred using the schema registry. A schema must be registered * for this type, or the conversion will fail. */ public static <OutputT> PTransform<PCollection<Row>, PCollection<OutputT>> fromRows( Class<OutputT> clazz) { return to(clazz); }
/** * Select the following fields for the specified PCollection using {@link FieldAccessDescriptor}. * * <p>Each PCollection in the input must have fields specified for the join key. */ public static Inner byFieldAccessDescriptor( TupleTag<?> tag, FieldAccessDescriptor fieldAccessDescriptor) { return new Inner().byFieldAccessDescriptor(tag, fieldAccessDescriptor); }
/** * Convert a {@link PCollection}{@literal <InputT>} to a {@link PCollection}{@literal <OutputT>}. * * <p>This function allows converting between two types as long as the two types have * <i>compatible</i> schemas. Two schemas are said to be <i>compatible</i> if they recursively * have fields with the same names, but possibly different orders. */ public static <InputT, OutputT> PTransform<PCollection<InputT>, PCollection<OutputT>> to( TypeDescriptor<OutputT> typeDescriptor) { return new ConvertTransform<>(typeDescriptor); }
FieldAggregation<FieldT, AccumT, OutputT> resolve(Schema schema) { return new FieldAggregation<>( fieldsToAggregate, outputField, fn, combineTag, aggregationSchema, schema); } }
/** * Select a set of fields described in a {@link FieldAccessDescriptor}. * * <p>This allows for nested fields to be selected as well. */ public static <T> Select<T> fieldAccess(FieldAccessDescriptor fieldAccessDescriptor) { return new Select(fieldAccessDescriptor); }
/** * Join by the following {@link FieldAccessDescriptor}. * * <p>The same access descriptor is used in all input PCollections. */ public static Inner byFieldAccessDescriptor(FieldAccessDescriptor fieldAccessDescriptor) { return new Inner(fieldAccessDescriptor); }
/** The same as {@link #aggregateField} but using field id. */ public <CombineInputT, AccumT, CombineOutputT> CombineFieldsGlobally<InputT> aggregateField( int inputFieldId, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, String outputFieldName) { return new CombineFieldsGlobally<>( SchemaAggregateFn.<InputT>create() .aggregateFields( FieldAccessDescriptor.withFieldIds(inputFieldId), fn, outputFieldName)); }
public <CombineInputT, AccumT, CombineOutputT> CombineFieldsByFields<InputT> aggregateField( int inputFieldId, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, String outputFieldName) { return new CombineFieldsByFields<>( this, SchemaAggregateFn.<InputT>create() .aggregateFields( FieldAccessDescriptor.withFieldIds(inputFieldId), fn, outputFieldName)); }
public <CombineInputT, AccumT, CombineOutputT> CombineFieldsGlobally<InputT> aggregateField( int inputFieldId, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, String outputFieldName) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields( FieldAccessDescriptor.withFieldIds(inputFieldId), fn, outputFieldName)); }
public <CombineInputT, AccumT, CombineOutputT> CombineFieldsByFields<InputT> aggregateField( int inputFieldId, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, String outputFieldName) { return new CombineFieldsByFields<>( byFields, schemaAggregateFn.aggregateFields( FieldAccessDescriptor.withFieldIds(inputFieldId), fn, outputFieldName)); }
/** * Convert a {@link PCollection}{@literal <Row>} into a {@link PCollection}{@literal <Row>}. * * <p>The output schema will be inferred using the schema registry. A schema must be registered * for this type, or the conversion will fail. */ public static <OutputT> PTransform<PCollection<Row>, PCollection<OutputT>> fromRows( TypeDescriptor<OutputT> typeDescriptor) { return to(typeDescriptor); }
/** The same as {@link #aggregateField} but using field id. */ public <CombineInputT, AccumT, CombineOutputT> CombineFieldsGlobally<InputT> aggregateField( int inputFielId, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, Field outputField) { return new CombineFieldsGlobally<>( SchemaAggregateFn.<InputT>create() .aggregateFields(FieldAccessDescriptor.withFieldIds(inputFielId), fn, outputField)); }
public <CombineInputT, AccumT, CombineOutputT> CombineFieldsByFields<InputT> aggregateField( int inputFieldId, CombineFn<CombineInputT, AccumT, CombineOutputT> fn, Field outputField) { return new CombineFieldsByFields<>( this, SchemaAggregateFn.<InputT>create() .aggregateFields(FieldAccessDescriptor.withFieldIds(inputFieldId), fn, outputField)); }