Refine search
@Override public List<FuncSpec> getArgToFuncMapping() { List<FuncSpec> funcList = new ArrayList<>(); Schema s = new Schema(); s.add(new Schema.FieldSchema(null, DataType.CHARARRAY)); funcList.add(new FuncSpec(this.getClass().getName(), s)); return funcList; }
public Schema outputSchema(Schema input) { try { // Column $0 is a bag of tuples that represent the left dataset Schema leftSchema = input.getField(0).schema.getField(0).schema; // Column $1 is a bag of tuples that represent the right dataset Schema rightSchema = input.getField(1).schema.getField(0).schema; Schema tupleSchema = new Schema(); for (FieldSchema leftAttr : leftSchema.getFields()) { leftAttr = leftAttr.clone(); leftAttr.alias = "left::"+leftAttr.alias; tupleSchema.add(leftAttr); } for (FieldSchema rightAttr : rightSchema.getFields()) { rightAttr = rightAttr.clone(); rightAttr.alias = "right::"+rightAttr.alias; tupleSchema.add(rightAttr); } FieldSchema outSchema = new Schema.FieldSchema("result", tupleSchema); outSchema.type = DataType.BAG; return new Schema(outSchema); } catch (Exception e) { return null; } }
@Override public List<FuncSpec> getArgToFuncMapping() throws FrontendException { List<FieldSchema> fields = new ArrayList<FieldSchema>(3); fields.add(new Schema.FieldSchema(null, DataType.INTEGER)); fields.add(new Schema.FieldSchema(null, DataType.INTEGER)); fields.add(new Schema.FieldSchema(null, DataType.BAG)); FuncSpec funcSpec = new FuncSpec(this.getClass().getName(), new Schema(fields)); List<FuncSpec> funcSpecs = new ArrayList<FuncSpec>(1); funcSpecs.add(funcSpec); return funcSpecs; }
Schema schema2) throws SchemaMergeException{ Schema mergedSchema = new Schema(); HashSet<FieldSchema> schema2colsAdded = new HashSet<FieldSchema>(); for(FieldSchema fs1 : schema1.getFields()){ checkNullAlias(fs1, schema1); FieldSchema fs2 = getFieldSubNameMatchThrowSchemaMergeException(schema2,fs1.alias); if(fs2 != null){ if(! schema2colsAdded.contains(fs2)){ try { mergedSchema.add(fs2.clone()); } catch (CloneNotSupportedException e) { throw new SchemaMergeException(
@Override public Schema outputSchema(Schema input) { try{ if(input.getField(0).type != DataType.MAP) { throw new RuntimeException("Expected map, received schema " +DataType.findTypeName(input.getField(0).type)); } } catch(FrontendException e) { throw new RuntimeException(e); } return new Schema(new Schema.FieldSchema(null, DataType.MAP)); }
private Schema getSchemaFromRequiredFieldList(Schema schema, List<RequiredField> fieldList) throws FrontendException { Schema s = new Schema(); for (RequiredField rf : fieldList) { FieldSchema f; try { f = schema.getField(rf.getAlias()).clone(); } catch (CloneNotSupportedException e) { throw new FrontendException("Clone not supported for the fieldschema", e); } if (rf.getSubFields() == null) { s.add(f); } else { Schema innerSchema = getSchemaFromRequiredFieldList(f.schema, rf.getSubFields()); if (innerSchema == null) { return null; } else { f.schema = innerSchema; s.add(f); } } } return s; }
if(schema.isTwoLevelAccessRequired() || other.isTwoLevelAccessRequired()) { if(schema.isTwoLevelAccessRequired()) { try { schema = schema.getField(0).schema; if (!FieldSchema.equals(myFs, otherFs, false, relaxAlias)) { return false ;
List<FieldSchema> sFields = s1.getFields(); List<FieldSchema> fsFields = s2.getFields(); FieldSchema fsFS = ((s2Type == SchemaType.VARARG) && i >= s2.size()) ? fsFields.get(s2.size() - 1) : fsFields.get(i); if(!FieldSchema.equals(sFS, fsFS, false, true)) return INF; if(FieldSchema.equals(sFS, fsFS, true, true)) continue; if(!castLookup.containsKey(sFS.type)) return INF;
if(isNullOrUnknownType(myFs) && isNullOrUnknownType(otherFs)) { int errCode = 1021; String msg = "Type mismatch. No useful type for merging. Field Schema: " + myFs + ". Other Field Schema: " + otherFs; } else if(myFs.type == otherFs.type) { mergedType = myFs.type; } else if (!isNullOrUnknownType(myFs) && isNullOrUnknownType(otherFs)) { mergedType = myFs.type; } else { if (allowMergeableTypes) { if (isNullOrUnknownType(myFs) && !isNullOrUnknownType(otherFs)) { mergedType = otherFs.type; } else if(otherFs.type == DataType.BYTEARRAY) { if(castable(otherFs, myFs)) { mergedType = otherFs.type; } else { mergedFs = new FieldSchema(mergedAlias, mergedType) ; mergedFs = new FieldSchema(mergedAlias, mergedSubSchema, mergedType) ; } catch (FrontendException fee) { int errCode = 1023;
@Override public ResourceFieldSchema getLoadSchema() throws IOException { return new ResourceFieldSchema(new FieldSchema(null, protobufToPig.toSchema(Protobufs .getMessageDescriptor(typeRef.getRawClass())))); }
private static FieldSchema union(FieldSchema mergedFieldSchema, FieldSchema newFieldSchema) { if (!mergedFieldSchema.alias.equals(newFieldSchema.alias) || mergedFieldSchema.type != newFieldSchema.type) { throw new IncompatibleSchemaModificationException("Incompatible Pig schema change: " + mergedFieldSchema + " can not accept"); } try { return new FieldSchema(mergedFieldSchema.alias, union(mergedFieldSchema.schema, newFieldSchema.schema), mergedFieldSchema.type); } catch (FrontendException e) { throw new SchemaConversionException(e); } }
@Override public FieldSchema convertFLOAT(PrimitiveTypeName primitiveTypeName) throws FrontendException { return new FieldSchema(fieldName, null, DataType.FLOAT); }
@Override public FieldSchema convertINT32(PrimitiveTypeName primitiveTypeName) throws FrontendException { return new FieldSchema(fieldName, null, DataType.INTEGER); }
/** * Two field schemas are equal if types and schemas * are equal in all levels. * * In order to relax alias equivalent requirement, * instead use equals(FieldSchema fschema, FieldSchema fother, boolean relaxInner, boolean relaxAlias) */ @Override public boolean equals(Object other) { if (!(other instanceof FieldSchema)) return false; FieldSchema otherfs = (FieldSchema)other; return FieldSchema.equals(this, otherfs, false, false) ; }
public static FuncSpec buildSimpleFuncSpec(String className, byte...types) { List<Schema.FieldSchema> fieldSchemas = Lists.newArrayListWithExpectedSize(types.length); for (byte type : types) { fieldSchemas.add(new Schema.FieldSchema(null, type)); } return new FuncSpec(className, new Schema(fieldSchemas)); }
@Override public Schema outputSchema(Schema input) { try { return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), input.getField(0).schema, DataType.BAG)); } catch (FrontendException e) { return null; } } }
if ((udfSchemaType == SchemaType.NORMAL) && (inputSchema.size() != udfSchema.size())) return false; if ((udfSchemaType == SchemaType.VARARG) && inputSchema.size() < udfSchema.size()) return false; ) { if (!FieldSchema.equals(inputFieldSchema, udfFieldSchema, false, true)) { new FieldSchema(inputFieldSchema); if (!FieldSchema.equals(inputFSWithBytearrayinTuple, udfFieldSchema, false, true)) { return false;
@Override public List<FuncSpec> getArgToFuncMapping() throws FrontendException { List<FuncSpec> funcList = new ArrayList<FuncSpec>(); Schema s = new Schema(); s.add(new Schema.FieldSchema(null, DataType.BAG)); funcList.add(new FuncSpec(this.getClass().getName(), s)); return funcList; }