public static Schema getSchemaFromCols(Properties properties, List<String> columnNames, List<TypeInfo> columnTypes, String columnCommentProperty) { List<String> columnComments; if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { columnComments = new ArrayList<String>(); } else { //Comments are separated by "\0" in columnCommentProperty, see method getSchema //in MetaStoreUtils where this string columns.comments is generated columnComments = Arrays.asList(columnCommentProperty.split("\0")); if (LOG.isDebugEnabled()) { LOG.debug("columnComments is " + columnCommentProperty); } } if (columnNames.size() != columnTypes.size()) { throw new IllegalArgumentException("AvroSerde initialization failed. Number of column " + "name and column type differs. columnNames = " + columnNames + ", columnTypes = " + columnTypes); } final String tableName = properties.getProperty(TABLE_NAME); final String tableComment = properties.getProperty(TABLE_COMMENT); TypeInfoToSchema typeInfoToSchema = new TypeInfoToSchema(); return typeInfoToSchema.convert(columnNames, columnTypes, columnComments, properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAMESPACE.getPropName()), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAME.getPropName(), tableName), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_DOC.getPropName(), tableComment)); }
.collect(Collectors.toList()); List<String> comments = fields.stream().map(fs -> fs.getComment()).collect(Collectors.toList()); Schema schema = new TypeInfoToSchema().convert(colNames, typeInfos, comments, null, null, null); return getOrGenerateSchemaFile(schema);
public Schema getSchema(String columnNamesStr, String columnTypesStr, String columnCommentsStr, String namespace, String name, String doc) { List<String> columnNames = Arrays.asList(columnNamesStr.split(",")); List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypesStr); List<String> columnComments; if (columnCommentsStr.isEmpty()) { columnComments = new ArrayList<String>(); } else { columnComments = Arrays.asList(columnCommentsStr.split(",")); } return typeInfoToSchema.convert(columnNames, columnTypes, columnComments, namespace, name, doc); } }
public static Schema getSchemaFromCols(Properties properties, List<String> columnNames, List<TypeInfo> columnTypes, String columnCommentProperty) { List<String> columnComments; if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { columnComments = new ArrayList<String>(); } else { //Comments are separated by "\0" in columnCommentProperty, see method getSchema //in MetaStoreUtils where this string columns.comments is generated columnComments = Arrays.asList(columnCommentProperty.split("\0")); LOG.info("columnComments is " + columnCommentProperty); } if (columnNames.size() != columnTypes.size()) { throw new IllegalArgumentException("AvroSerde initialization failed. Number of column " + "name and column type differs. columnNames = " + columnNames + ", columnTypes = " + columnTypes); } final String tableName = properties.getProperty(TABLE_NAME); final String tableComment = properties.getProperty(TABLE_COMMENT); TypeInfoToSchema typeInfoToSchema = new TypeInfoToSchema(); return typeInfoToSchema.convert(columnNames, columnTypes, columnComments, properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAMESPACE.getPropName()), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAME.getPropName(), tableName), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_DOC.getPropName(), tableComment)); }
public static Schema getSchemaFromCols(Properties properties, List<String> columnNames, List<TypeInfo> columnTypes, String columnCommentProperty) { List<String> columnComments; if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { columnComments = new ArrayList<String>(); } else { //Comments are separated by "\0" in columnCommentProperty, see method getSchema //in MetaStoreUtils where this string columns.comments is generated columnComments = Arrays.asList(columnCommentProperty.split("\0")); LOG.info("columnComments is " + columnCommentProperty); } if (columnNames.size() != columnTypes.size()) { throw new IllegalArgumentException("AvroSerde initialization failed. Number of column " + "name and column type differs. columnNames = " + columnNames + ", columnTypes = " + columnTypes); } final String tableName = properties.getProperty(TABLE_NAME); final String tableComment = properties.getProperty(TABLE_COMMENT); TypeInfoToSchema typeInfoToSchema = new TypeInfoToSchema(); return typeInfoToSchema.convert(columnNames, columnTypes, columnComments, properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAMESPACE.getPropName()), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAME.getPropName(), tableName), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_DOC.getPropName(), tableComment)); }
public static Schema getSchemaFromCols(Properties properties, List<String> columnNames, List<TypeInfo> columnTypes, String columnCommentProperty) { List<String> columnComments; if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { columnComments = new ArrayList<String>(); } else { //Comments are separated by "\0" in columnCommentProperty, see method getSchema //in MetaStoreUtils where this string columns.comments is generated columnComments = Arrays.asList(columnCommentProperty.split("\0")); if (LOG.isDebugEnabled()) { LOG.debug("columnComments is " + columnCommentProperty); } } if (columnNames.size() != columnTypes.size()) { throw new IllegalArgumentException("AvroSerde initialization failed. Number of column " + "name and column type differs. columnNames = " + columnNames + ", columnTypes = " + columnTypes); } final String tableName = properties.getProperty(TABLE_NAME); final String tableComment = properties.getProperty(TABLE_COMMENT); TypeInfoToSchema typeInfoToSchema = new TypeInfoToSchema(); return typeInfoToSchema.convert(columnNames, columnTypes, columnComments, properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAMESPACE.getPropName()), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAME.getPropName(), tableName), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_DOC.getPropName(), tableComment)); }
public static Schema getSchemaFromCols(Properties properties, List<String> columnNames, List<TypeInfo> columnTypes, String columnCommentProperty) { List<String> columnComments; if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { columnComments = new ArrayList<String>(); } else { //Comments are separated by "\0" in columnCommentProperty, see method getSchema //in MetaStoreUtils where this string columns.comments is generated columnComments = Arrays.asList(columnCommentProperty.split("\0")); LOG.info("columnComments is " + columnCommentProperty); } if (columnNames.size() != columnTypes.size()) { throw new IllegalArgumentException("AvroSerde initialization failed. Number of column " + "name and column type differs. columnNames = " + columnNames + ", columnTypes = " + columnTypes); } final String tableName = properties.getProperty(TABLE_NAME); final String tableComment = properties.getProperty(TABLE_COMMENT); TypeInfoToSchema typeInfoToSchema = new TypeInfoToSchema(); return typeInfoToSchema.convert(columnNames, columnTypes, columnComments, properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAMESPACE.getPropName()), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_NAME.getPropName(), tableName), properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_DOC.getPropName(), tableComment)); }
.collect(Collectors.toList()); List<String> comments = fields.stream().map(fs -> fs.getComment()).collect(Collectors.toList()); Schema schema = new TypeInfoToSchema().convert(colNames, typeInfos, comments, null, null, null); return getOrGenerateSchemaFile(schema);