public void setSchema(ResourceSchema schema) { schemaField = new ResourceFieldSchema(); schemaField.setType(DataType.TUPLE); try { schemaField.setSchema(schema); } catch (IOException ex) { throw new EsHadoopIllegalStateException(String.format("Cannot use schema [%s]", schema), ex); } }
bagSubFieldSchemas[0] = new ResourceFieldSchema().setName(innerTupleName) .setDescription("The tuple in the bag") .setType(DataType.TUPLE); } else { ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1]; innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName(innerFieldName) .setDescription("The inner field in the tuple in the bag") .setType(getPigType(arrayElementFieldSchema))
PDataType phoenixDataType = PDataType.fromTypeId(sqlType); byte pigType = TypeUtil.getPigDataTypeForPhoenixType(phoenixDataType); ResourceFieldSchema field = new ResourceFieldSchema(); field.setType(pigType).setName(cinfo.getDisplayName()); fields[i++] = field;
@Override public ResourceFieldSchema getLoadSchema() throws IOException { return new ResourceFieldSchema(new FieldSchema(null, expectedSchema)); }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.CHARARRAY); return schema; }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.BYTEARRAY); return schema; }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { return new ResourceFieldSchema(new FieldSchema(null, protobufToPig.toSchema(Protobufs .getMessageDescriptor(typeRef.getRawClass())))); }
public void setSchema(ResourceSchema schema) { schemaField = new ResourceFieldSchema(); schemaField.setType(DataType.TUPLE); try { schemaField.setSchema(schema); } catch (IOException ex) { throw new EsHadoopIllegalStateException(String.format("Cannot use schema [%s]", schema), ex); } }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.LONG); return schema; }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.BYTEARRAY); return schema; }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.INTEGER); return schema; }
private static TypeInfo getInputTypeInfo(Schema inputSchema) throws IOException { FieldSchema innerFieldSchema = inputSchema.getField(0).schema.getField(0); ResourceFieldSchema rfs = new ResourceFieldSchema(innerFieldSchema); TypeInfo inputTypeInfo = HiveUtils.getTypeInfo(rfs); return inputTypeInfo; }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.NULL); return schema; }
/** * Construct a ResourceSchema from a {@link LogicalSchema} * @param pigSchema Schema to use */ public ResourceSchema(LogicalSchema pigSchema) { List<LogicalFieldSchema> pigSchemaFields = pigSchema.getFields(); fields = new ResourceFieldSchema[pigSchemaFields.size()]; for (int i=0; i<fields.length; i++) { fields[i] = new ResourceFieldSchema(pigSchemaFields.get(i)); } }
private static ResourceFieldSchema convert(Type type) throws IOException { ResourceFieldSchema result = new ResourceFieldSchema(); result.setType(convertType(type)); if (!type.isPrimitiveType()) { result.setSchema(convertComplex(type)); } return result; }
/** * Construct a ResourceSchema from a {@link Schema} * @param pigSchema Schema to use */ public ResourceSchema(Schema pigSchema) { List<FieldSchema> pigSchemaFields = pigSchema.getFields(); fields = new ResourceFieldSchema[pigSchemaFields.size()]; for (int i=0; i<fields.length; i++) { fields[i] = new ResourceFieldSchema(pigSchemaFields.get(i)); } }
@Override public void checkSchema(ResourceSchema rs) throws IOException { ResourceFieldSchema fs = new ResourceFieldSchema(); fs.setType(DataType.TUPLE); fs.setSchema(rs); typeInfo = HiveUtils.getTypeInfo(fs); Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass()); p.setProperty(signature + SchemaSignatureSuffix, ObjectSerializer.serialize(typeInfo)); }
when(record.getResultMap()).thenReturn(values); ResourceFieldSchema field = new ResourceFieldSchema().setType(DataType.CHARARRAY); ResourceFieldSchema field1 = new ResourceFieldSchema().setType(DataType.INTEGER); ResourceFieldSchema field2 = new ResourceFieldSchema().setType(DataType.LONG); ResourceFieldSchema field3 = new ResourceFieldSchema().setType(DataType.BYTEARRAY); ResourceFieldSchema field4 = new ResourceFieldSchema().setType(DataType.TUPLE); ResourceFieldSchema[] projectedColumns = { field, field1, field2, field3 , field4 }; assertEquals(2,doubleArrayTuple.size()); field = new ResourceFieldSchema().setType(DataType.BIGDECIMAL); field1 = new ResourceFieldSchema().setType(DataType.BIGINTEGER); values.clear(); values.put("first", new BigDecimal(123123123.123213));
@Test public void testSchema() throws SQLException, IOException { final Configuration configuration = mock(Configuration.class); when(configuration.get(PhoenixConfigurationUtil.SCHEMA_TYPE)).thenReturn(SchemaType.TABLE.name()); final ResourceSchema actual = PhoenixPigSchemaUtil.getResourceSchema( configuration, new Dependencies() { List<ColumnInfo> getSelectColumnMetadataList( Configuration configuration) throws SQLException { return Lists.newArrayList(ID_COLUMN, NAME_COLUMN); } }); // expected schema. final ResourceFieldSchema[] fields = new ResourceFieldSchema[2]; fields[0] = new ResourceFieldSchema().setName("ID") .setType(DataType.LONG); fields[1] = new ResourceFieldSchema().setName("NAME") .setType(DataType.CHARARRAY); final ResourceSchema expected = new ResourceSchema().setFields(fields); assertEquals(expected.toString(), actual.toString()); }
private static ResourceFieldSchema getResourceSchemaFromFieldSchema(HCatFieldSchema hfs) throws IOException { ResourceFieldSchema rfSchema; // if we are dealing with a bag or tuple column - need to worry about subschema if (hfs.getType() == Type.STRUCT) { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(getTupleSubSchema(hfs)); } else if (hfs.getType() == Type.ARRAY) { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(getBagSubSchema(hfs)); } else { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(null); // no munging inner-schemas } return rfSchema; }