public static int[] projectAllVariables(IOperatorSchema opSchema) { return projectVariablesImpl(opSchema, opSchema, opSchema.getSize()); }
protected int[] getKeysAndDecs(IOperatorSchema inputSchema) { int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchema); int sz = inputSchema.getSize(); int fdSz = sz - columnList.size(); int[] fdColumns = new int[fdSz]; int j = 0; for (LogicalVariable v : inputSchema) { if (!columnList.contains(v)) { fdColumns[j++] = inputSchema.findVariable(v); } } int[] keysAndDecs = new int[keys.length + fdColumns.length]; for (int i = 0; i < keys.length; i++) { keysAndDecs[i] = keys[i]; } for (int i = 0; i < fdColumns.length; i++) { keysAndDecs[i + keys.length] = fdColumns[i]; } return keysAndDecs; }
@SuppressWarnings("rawtypes") public static RecordDescriptor mkRecordDescriptor(IVariableTypeEnvironment env, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { ISerializerDeserializer[] fields = new ISerializerDeserializer[opSchema.getSize()]; ITypeTraits[] typeTraits = new ITypeTraits[opSchema.getSize()]; ISerializerDeserializerProvider sdp = context.getSerializerDeserializerProvider(); ITypeTraitProvider ttp = context.getTypeTraitProvider(); int i = 0; for (LogicalVariable var : opSchema) { Object t = env.getVarType(var); if (t == null) { LOGGER.warn("No type for variable " + var); } fields[i] = sdp.getSerializerDeserializer(t); typeTraits[i] = ttp.getTypeTrait(t); i++; } return new RecordDescriptor(fields, typeTraits); }
@Override public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource, List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed, List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig) throws AlgebricksException { VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) dataSource; if (sourceFileMap != null) { final int len = ds.getPartitions().length; String[] collectionPartitions = new String[len]; for (int i = 0; i < len; ++i) { String partition = ds.getPartitions()[i]; File mapped = sourceFileMap.get(partition); collectionPartitions[i] = mapped != null ? mapped.toString() : partition; } ds.setPartitions(collectionPartitions); } RecordDescriptor rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]); IOperatorDescriptor scanner = new VXQueryCollectionOperatorDescriptor(jobSpec, ds, rDesc, this.hdfsConf, this.nodeControllerInfos); AlgebricksPartitionConstraint constraint = getClusterLocations(nodeList, ds.getPartitionCount()); return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint); }
int opSchemaSizePreSubplans = opSchema.getSize(); AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], winOp, opSchema, context); int aggregatorOutputSchemaSize = opSchema.getSize() - opSchemaSizePreSubplans; WindowAggregatorDescriptorFactory nestedAggFactory = new WindowAggregatorDescriptorFactory(subplans); nestedAggFactory.setSourceLocation(winOp.getSourceLocation());
break; case LEFT_OUTER: IMissingWriterFactory[] nonMatchWriterFactories = new IMissingWriterFactory[inputSchemas[1].getSize()]; for (int j = 0; j < nonMatchWriterFactories.length; j++) { nonMatchWriterFactories[j] = context.getMissingWriterFactory();
break; case LEFT_OUTER: IMissingWriterFactory[] nonMatchWriterFactories = new IMissingWriterFactory[inputSchemas[1].getSize()]; for (int j = 0; j < nonMatchWriterFactories.length; j++) { nonMatchWriterFactories[j] = context.getMissingWriterFactory();
private IOperatorDescriptor generateOptimizedHashJoinRuntime(JobGenContext context, IOperatorSchema[] inputSchemas, int[] keysLeft, int[] keysRight, IBinaryHashFunctionFamily[] hashFunFamilies, IBinaryComparatorFactory[] comparatorFactories, IPredicateEvaluatorFactory predEvaluatorFactory, RecordDescriptor recDescriptor, IOperatorDescriptorRegistry spec) throws AlgebricksException { switch (kind) { case INNER: return new OptimizedHybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(), maxInputBuildSizeInFrames, getFudgeFactor(), keysLeft, keysRight, hashFunFamilies, comparatorFactories, recDescriptor, new JoinMultiComparatorFactory(comparatorFactories, keysLeft, keysRight), new JoinMultiComparatorFactory(comparatorFactories, keysRight, keysLeft), predEvaluatorFactory); case LEFT_OUTER: IMissingWriterFactory[] nonMatchWriterFactories = new IMissingWriterFactory[inputSchemas[1].getSize()]; for (int j = 0; j < nonMatchWriterFactories.length; j++) { nonMatchWriterFactories[j] = context.getMissingWriterFactory(); } return new OptimizedHybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(), maxInputBuildSizeInFrames, getFudgeFactor(), keysLeft, keysRight, hashFunFamilies, comparatorFactories, recDescriptor, new JoinMultiComparatorFactory(comparatorFactories, keysLeft, keysRight), new JoinMultiComparatorFactory(comparatorFactories, keysRight, keysLeft), predEvaluatorFactory, true, nonMatchWriterFactories); default: throw new NotImplementedException(); } }