/** * Generate the table descriptor for intermediate files. */ public static TableDesc getReduceValueTableDesc(List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.SERIALIZATION_SORT_ORDER, order.toString(), serdeConstants.SERIALIZATION_NULL_SORT_ORDER, nullOrder.toString(), MetaStoreUtils.getColumnNamesFromFieldSchema(fieldSchemas), "columns.types", MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName()));
/** * Generate the table descriptor for Map-side join value. */ public static TableDesc getMapJoinValueTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
/** * Generate the table descriptor for reduce key. */ public static TableDesc getReduceKeyTableDesc(List<FieldSchema> fieldSchemas, String order, String nullOrder) { return new TableDesc( SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties(serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.SERIALIZATION_SORT_ORDER, order, serdeConstants.SERIALIZATION_NULL_SORT_ORDER, nullOrder, serdeConstants.SERIALIZATION_LIB, BinarySortableSerDe.class.getName())); }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getIntermediateFileTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
public static void addSchemaEvolutionToTableScanOperator(Table table, TableScanOperator tableScanOp) { String colNames = MetaStoreUtils.getColumnNamesFromFieldSchema(table.getSd().getCols()); String colTypes = MetaStoreUtils.getColumnTypesFromFieldSchema(table.getSd().getCols()); tableScanOp.setSchemaEvolution(colNames, colTypes); }
/** * Get Properties for configuring a SerDe * * @param fieldSchemas column information * @param serDeParams parameters to serde * @return Properties */ public static Properties getSerDeProperties( List<FieldSchema> fieldSchemas, Map<String, String> serDeParams) { Properties props = new Properties(); props.setProperty(serdeConstants.LIST_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(fieldSchemas)); props.setProperty(serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils.getColumnTypesFromFieldSchema(fieldSchemas)); props.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N"); props.setProperty(serdeConstants.SERIALIZATION_FORMAT, "1"); props.putAll(serDeParams); return props; }
/** * Get Properties for configuring a SerDe * * @param fieldSchemas column information * @param serDeParams parameters to serde * @return Properties */ public static Properties getSerDeProperties( List<FieldSchema> fieldSchemas, Map<String, String> serDeParams) { Properties props = new Properties(); props.setProperty(serdeConstants.LIST_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(fieldSchemas)); props.setProperty(serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils.getColumnTypesFromFieldSchema(fieldSchemas)); props.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N"); props.setProperty(serdeConstants.SERIALIZATION_FORMAT, "1"); props.putAll(serDeParams); return props; }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getReduceValueTableDesc(List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
/** * Generate the table descriptor for reduce key. */ public static TableDesc getReduceKeyTableDesc(List<FieldSchema> fieldSchemas, String order) { return new TableDesc( SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties(serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.SERIALIZATION_SORT_ORDER, order, serdeConstants.SERIALIZATION_LIB, BinarySortableSerDe.class.getName())); }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getIntermediateFileTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
/** * Generate the table descriptor for Map-side join value. */ public static TableDesc getMapJoinValueTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
private static Properties getSerdeProperties(HCatTableInfo info, HCatSchema s) throws SerDeException { Properties props = new Properties(); List<FieldSchema> fields = HCatUtil.getFieldSchemaList(s.getFields()); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(fields)); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils.getColumnTypesFromFieldSchema(fields)); props.setProperty("columns.comments", MetaStoreUtils.getColumnCommentsFromFieldSchema(fields)); // setting these props to match LazySimpleSerde props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N"); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1"); //add props from params set in table schema props.putAll(info.getStorerInfo().getProperties()); return props; }
private static Properties getSerdeProperties(HCatTableInfo info, HCatSchema s) throws SerDeException { Properties props = new Properties(); List<FieldSchema> fields = HCatUtil.getFieldSchemaList(s.getFields()); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(fields)); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils.getColumnTypesFromFieldSchema(fields)); props.setProperty("columns.comments", MetaStoreUtils.getColumnCommentsFromFieldSchema(fields)); // setting these props to match LazySimpleSerde props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N"); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1"); //add props from params set in table schema props.putAll(info.getStorerInfo().getProperties()); return props; }
private static Properties getSerdeProperties(HCatTableInfo info, HCatSchema s) throws SerDeException { Properties props = new Properties(); List<FieldSchema> fields = HCatUtil.getFieldSchemaList(s.getFields()); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(fields)); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils.getColumnTypesFromFieldSchema(fields)); props.setProperty("columns.comments", MetaStoreUtils.getColumnCommentsFromFieldSchema(fields)); // setting these props to match LazySimpleSerde props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N"); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1"); //add props from params set in table schema props.putAll(info.getStorerInfo().getProperties()); return props; }
/** * Generate the table descriptor for Map-side join key. */ public static TableDesc getMapJoinKeyTableDesc(List<FieldSchema> fieldSchemas) { return new TableDesc(LazyBinarySerDe.class, SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties("columns", MetaStoreUtils.getColumnNamesFromFieldSchema(fieldSchemas), "columns.types", MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), Constants.ESCAPE_CHAR, "\\")); }
/** * Generate the table descriptor for Map-side join key. */ public static TableDesc getMapJoinValueTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(LazyBinarySerDe.class, SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties("columns", MetaStoreUtils.getColumnNamesFromFieldSchema(fieldSchemas), "columns.types", MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), Constants.ESCAPE_CHAR, "\\")); }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getReduceValueTableDesc(List<FieldSchema> fieldSchemas) { return new TableDesc(LazyBinarySerDe.class, SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( Constants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), Constants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), Constants.ESCAPE_CHAR, "\\")); }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getIntermediateFileTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(LazyBinarySerDe.class, SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( Constants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), Constants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), Constants.ESCAPE_CHAR, "\\")); }
/** * Generate the table descriptor for reduce key. */ public static TableDesc getReduceKeyTableDesc(List<FieldSchema> fieldSchemas, String order) { return new TableDesc(BinarySortableSerDe.class, SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties(Constants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), Constants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), Constants.SERIALIZATION_SORT_ORDER, order)); }