static boolean isComplexType(ResourceFieldSchema fieldSchema) { return (fieldSchema != null && fieldSchema.getType() >= 100); } }
public void setSchema(ResourceSchema schema) { schemaField = new ResourceFieldSchema(); schemaField.setType(DataType.TUPLE); try { schemaField.setSchema(schema); } catch (IOException ex) { throw new EsHadoopIllegalStateException(String.format("Cannot use schema [%s]", schema), ex); } }
switch (expr.getOpType()) { case OP_EQ: opStr = " = "; break; default: opStr = expr.getOpType().toString(); if (be.getRhs() instanceof Const) { UDFContext udfContext = UDFContext.getUDFContext(); Properties udfProps = udfContext.getUDFProperties(this.getClass(), new String[]{signature}); HCatSchema hcatTableSchema = (HCatSchema) udfProps.get(HCatConstants.HCAT_TABLE_SCHEMA); HCatFieldSchema fs = null; try { fs = hcatTableSchema.get(be.getLhs().toString()); } catch (HCatException e) { return "(" + getHCatComparisonString(be.getLhs()) + opStr + getHCatConstString((Const)be.getRhs(), fs.getType()) + ")"; return "(" + getHCatComparisonString(be.getLhs()) + opStr + getHCatComparisonString(be.getRhs()) + ")"; } else {
/** * Checks to see if the given field is a schema-less Map that has values. * @return true if Map has no schema but has values (mixed schema map). false if not a Map or if Map is just empty. */ private boolean isPopulatedMixedValueMap(ResourceFieldSchema schema, int field, Tuple object) { if (schema.getType() != DataType.MAP) { // Can't be a mixed value map if it's not a map at all. return false; } try { Object fieldValue = object.get(field); Map<?, ?> map = (Map<?, ?>) fieldValue; return schema.getSchema() == null && !(map == null || map.isEmpty()); } catch (ExecException e) { throw new EsHadoopIllegalStateException(e); } }
private static ResourceFieldSchema getResourceSchemaFromFieldSchema(HCatFieldSchema hfs) throws IOException { ResourceFieldSchema rfSchema; // if we are dealing with a bag or tuple column - need to worry about subschema if (hfs.getType() == Type.STRUCT) { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(getTupleSubSchema(hfs)); } else if (hfs.getType() == Type.ARRAY) { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(getBagSubSchema(hfs)); } else { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(null); // no munging inner-schemas } return rfSchema; }
HCatSchema getHCatSchema(List<RequiredField> fields, String signature, Class<?> classForUDFCLookup) throws IOException { if (fields == null) { return null; } Properties props = UDFContext.getUDFContext().getUDFProperties( classForUDFCLookup, new String[]{signature}); HCatSchema hcatTableSchema = (HCatSchema) props.get(HCatConstants.HCAT_TABLE_SCHEMA); ArrayList<HCatFieldSchema> fcols = new ArrayList<HCatFieldSchema>(); for (RequiredField rf : fields) { fcols.add(hcatTableSchema.getFields().get(rf.getIndex())); } return new HCatSchema(fcols); }
private ResourceSchema createSchema(String schema) { try { return new ResourceSchema(Utils.getSchemaFromString(schema)); } catch (Exception ex) { throw new RuntimeException(ex); } }
public static ResourceSchema getResourceSchema(HCatSchema hcatSchema) throws IOException { List<ResourceFieldSchema> rfSchemaList = new ArrayList<ResourceFieldSchema>(); for (HCatFieldSchema hfs : hcatSchema.getFields()) { ResourceFieldSchema rfSchema; rfSchema = getResourceSchemaFromFieldSchema(hfs); rfSchemaList.add(rfSchema); } ResourceSchema rSchema = new ResourceSchema(); rSchema.setFields(rfSchemaList.toArray(new ResourceFieldSchema[rfSchemaList.size()])); return rSchema; }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldsInfo) throws FrontendException { // Store the required fields information in the UDFContext so that we // can retrieve it later. storeInUDFContext(signature, PRUNE_PROJECTION_INFO, requiredFieldsInfo); // HCat will always prune columns based on what we ask of it - so the // response is true return new RequiredFieldResponse(true); }
private static void addField(RequiredField field, List<String> fields, FieldAlias fa, String currentNode) { if (field.getSubFields() != null && !field.getSubFields().isEmpty()) { for (RequiredField subField : field.getSubFields()) { addField(subField, fields, fa, currentNode + "." + fa.toES(subField.getAlias())); } } else { fields.add(fa.toES(field.getAlias())); } }
static String asProjection(RequiredFieldList list, Properties props) { List<String> fields = new ArrayList<String>(); FieldAlias alias = alias(new PropertiesSettings(props)); for (RequiredField field : list.getFields()) { addField(field, fields, alias, ""); } return StringUtils.concatenate(fields, ","); }
static public Pair<String, String> getDBTableNames(String location) throws IOException { // the location string will be of the form: // <database name>.<table name> - parse it and // communicate the information to HCatInputFormat try { return HCatUtil.getDbAndTableName(location); } catch (IOException e) { String locationErrMsg = "The input location in load statement " + "should be of the form " + "<databasename>.<table name> or <table name>. Got " + location; throw new PigException(locationErrMsg, PIG_EXCEPTION_CODE); } }
protected void logAndRegister(PigServer server, String query, int lineNumber) throws IOException { assert lineNumber > 0 : "(lineNumber > 0) is false"; LOG.info("Registering pig query: " + query); server.registerQuery(query, lineNumber); }
private ResourceSchema createSchema(String schema) { try { return new ResourceSchema(Utils.getSchemaFromString(schema)); } catch (Exception ex) { throw new RuntimeException(ex); } }
private static ResourceSchema getTupleSubSchema(HCatFieldSchema hfs) throws IOException { // for each struct subfield, create equivalent ResourceFieldSchema ResourceSchema s = new ResourceSchema(); List<ResourceFieldSchema> lrfs = new ArrayList<ResourceFieldSchema>(); for (HCatFieldSchema subField : hfs.getStructSubSchema().getFields()) { lrfs.add(getResourceSchemaFromFieldSchema(subField)); } s.setFields(lrfs.toArray(new ResourceFieldSchema[lrfs.size()])); return s; }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException { String fields = PigUtils.asProjection(requiredFieldList, properties); getUDFProperties().setProperty(InternalConfigurationOptions.INTERNAL_ES_TARGET_FIELDS, fields); if (log.isTraceEnabled()) { log.trace(String.format("Given push projection; saving field projection [%s]", fields)); } return new RequiredFieldResponse(true); }
throw new PigException("HCatalog column type '" + type.toString() + "' is not supported in Pig as a column type", PIG_EXCEPTION_CODE);
private ResourceSchema createSchema(String schema) { try { return new ResourceSchema(Utils.getSchemaFromString(schema)); } catch (Exception ex) { throw new RuntimeException(ex); } }
private ResourceSchema createSchema(String schema) { try { return new ResourceSchema(Utils.getSchemaFromString(schema)); } catch (Exception ex) { throw new RuntimeException(ex); } }