/** * Convert FieldSchemas to columnNames. */ public static String getColumnNamesFromFieldSchema(List<FieldSchema> fieldSchemas) { String delimiter = getColumnNameDelimiter(fieldSchemas); StringBuilder sb = new StringBuilder(); for (int i = 0; i < fieldSchemas.size(); i++) { if (i > 0) { sb.append(delimiter); } sb.append(fieldSchemas.get(i).getName()); } return sb.toString(); }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getIntermediateFileTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
private static Properties addCols(Properties schema, List<FieldSchema> cols) { StringBuilder colNameBuf = new StringBuilder(); StringBuilder colTypeBuf = new StringBuilder(); StringBuilder colComment = new StringBuilder(); boolean first = true; String columnNameDelimiter = getColumnNameDelimiter(cols); for (FieldSchema col : cols) { if (!first) { colNameBuf.append(columnNameDelimiter); colTypeBuf.append(":"); colComment.append('\0'); } colNameBuf.append(col.getName()); colTypeBuf.append(col.getType()); colComment.append((null != col.getComment()) ? col.getComment() : StringUtils.EMPTY); first = false; } schema.setProperty( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS, colNameBuf.toString()); schema.setProperty(ColumnType.COLUMN_NAME_DELIMITER, columnNameDelimiter); String colTypes = colTypeBuf.toString(); schema.setProperty( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES, colTypes); schema.setProperty("columns.comments", colComment.toString()); return schema; }
private static Properties getSerdeProperties(HCatTableInfo info, HCatSchema s) throws SerDeException { Properties props = new Properties(); List<FieldSchema> fields = HCatUtil.getFieldSchemaList(s.getFields()); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(fields)); props.setProperty(serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fields)); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils.getColumnTypesFromFieldSchema(fields)); props.setProperty("columns.comments", MetaStoreUtils.getColumnCommentsFromFieldSchema(fields)); // setting these props to match LazySimpleSerde props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N"); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1"); //add props from params set in table schema props.putAll(info.getStorerInfo().getProperties()); return props; }
/** * Generate the table descriptor for Map-side join value. */ public static TableDesc getMapJoinValueTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
/** * Generate the table descriptor for reduce key. */ public static TableDesc getReduceKeyTableDesc(List<FieldSchema> fieldSchemas, String order, String nullOrder) { return new TableDesc( SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties(serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.SERIALIZATION_SORT_ORDER, order, serdeConstants.SERIALIZATION_NULL_SORT_ORDER, nullOrder, serdeConstants.SERIALIZATION_LIB, BinarySortableSerDe.class.getName())); }
Utilities.makeProperties(serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas),
/** * Convert FieldSchemas to columnNames. */ public static String getColumnNamesFromFieldSchema(List<FieldSchema> fieldSchemas) { String delimiter = getColumnNameDelimiter(fieldSchemas); StringBuilder sb = new StringBuilder(); for (int i = 0; i < fieldSchemas.size(); i++) { if (i > 0) { sb.append(delimiter); } sb.append(fieldSchemas.get(i).getName()); } return sb.toString(); }
private static Properties addCols(Properties schema, List<FieldSchema> cols) { StringBuilder colNameBuf = new StringBuilder(); StringBuilder colTypeBuf = new StringBuilder(); StringBuilder colComment = new StringBuilder(); boolean first = true; String columnNameDelimiter = getColumnNameDelimiter(cols); for (FieldSchema col : cols) { if (!first) { colNameBuf.append(columnNameDelimiter); colTypeBuf.append(":"); colComment.append('\0'); } colNameBuf.append(col.getName()); colTypeBuf.append(col.getType()); colComment.append((null != col.getComment()) ? col.getComment() : StringUtils.EMPTY); first = false; } schema.setProperty( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS, colNameBuf.toString()); schema.setProperty(ColumnType.COLUMN_NAME_DELIMITER, columnNameDelimiter); String colTypes = colTypeBuf.toString(); schema.setProperty( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES, colTypes); schema.setProperty("columns.comments", colComment.toString()); return schema; }
private static Properties getSerdeProperties(HCatTableInfo info, HCatSchema s) throws SerDeException { Properties props = new Properties(); List<FieldSchema> fields = HCatUtil.getFieldSchemaList(s.getFields()); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(fields)); props.setProperty(serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fields)); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils.getColumnTypesFromFieldSchema(fields)); props.setProperty("columns.comments", MetaStoreUtils.getColumnCommentsFromFieldSchema(fields)); // setting these props to match LazySimpleSerde props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N"); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1"); //add props from params set in table schema props.putAll(info.getStorerInfo().getProperties()); return props; }