Refine search
/** * Utility method which gets table or partition {@link InputFormat} class. First it * tries to get the class name from given StorageDescriptor object. If it doesn't contain it tries to get it from * StorageHandler class set in table properties. If not found throws an exception. * @param job {@link JobConf} instance needed incase the table is StorageHandler based table. * @param sd {@link StorageDescriptor} instance of currently reading partition or table (for non-partitioned tables). * @param table Table object */ public static Class<? extends InputFormat<?, ?>> getInputFormatClass(final JobConf job, final StorageDescriptor sd, final Table table) throws Exception { final String inputFormatName = sd.getInputFormat(); if (Strings.isNullOrEmpty(inputFormatName)) { final String storageHandlerClass = table.getParameters().get(META_TABLE_STORAGE); if (Strings.isNullOrEmpty(storageHandlerClass)) { throw new ExecutionSetupException("Unable to get Hive table InputFormat class. There is neither " + "InputFormat class explicitly specified nor StorageHandler class"); } final HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(job, storageHandlerClass); TableDesc tableDesc = new TableDesc(); tableDesc.setProperties(MetaStoreUtils.getTableMetadata(table)); storageHandler.configureInputJobProperties(tableDesc, table.getParameters()); return (Class<? extends InputFormat<?, ?>>) storageHandler.getInputFormatClass(); } else { return (Class<? extends InputFormat<?, ?>>) Class.forName(inputFormatName); } }
public TableWrapper(org.apache.hadoop.hive.metastore.api.Table apiTable) { org.apache.hadoop.hive.metastore.api.Table wrapperApiTable = apiTable.deepCopy(); if (wrapperApiTable.getTableType() == null){ // TableType specified was null, we need to figure out what type it was. if (MetaStoreUtils.isExternalTable(wrapperApiTable)){ wrapperApiTable.setTableType(TableType.EXTERNAL_TABLE.toString()); } else if (MetaStoreUtils.isIndexTable(wrapperApiTable)) { wrapperApiTable.setTableType(TableType.INDEX_TABLE.toString()); } else if (MetaStoreUtils.isMaterializedViewTable(wrapperApiTable)) { wrapperApiTable.setTableType(TableType.MATERIALIZED_VIEW.toString()); } else if ((wrapperApiTable.getSd() == null) || (wrapperApiTable.getSd().getLocation() == null)) { wrapperApiTable.setTableType(TableType.VIRTUAL_VIEW.toString()); } else { wrapperApiTable.setTableType(TableType.MANAGED_TABLE.toString()); } } initialize(wrapperApiTable); } }
private List<FieldSchema> getColsInternal(boolean forMs) { String serializationLib = getSerializationLib(); try { // Do the lightweight check for general case. if (hasMetastoreBasedSchema(SessionState.getSessionConf(), serializationLib)) { return tTable.getSd().getCols(); } else if (forMs && !shouldStoreFieldsInMetastore( SessionState.getSessionConf(), serializationLib, tTable.getParameters())) { return Hive.getFieldsFromDeserializerForMsStorage(this, getDeserializer()); } else { return MetaStoreUtils.getFieldsFromDeserializer(getTableName(), getDeserializer()); } } catch (Exception e) { LOG.error("Unable to get field from serde: " + serializationLib, e); } return new ArrayList<FieldSchema>(); }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getIntermediateFileTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getReduceValueTableDesc(List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
public static void addSchemaEvolutionToTableScanOperator(Table table, TableScanOperator tableScanOp) { String colNames = MetaStoreUtils.getColumnNamesFromFieldSchema(table.getSd().getCols()); String colTypes = MetaStoreUtils.getColumnTypesFromFieldSchema(table.getSd().getCols()); tableScanOp.setSchemaEvolution(colNames, colTypes); }
public static Properties getTableMetadata( org.apache.hadoop.hive.metastore.api.Table table) { return MetaStoreUtils.getSchema(table.getSd(), table.getSd(), table .getParameters(), table.getDbName(), table.getTableName(), table.getPartitionKeys()); }
List<FieldSchema> partitionKeys) { Properties schema = new Properties(); String inputFormat = sd.getInputFormat(); if (inputFormat == null || inputFormat.length() == 0) { inputFormat = org.apache.hadoop.mapred.SequenceFileInputFormat.class org.apache.hadoop.hive.metastore.api.Constants.FILE_INPUT_FORMAT, inputFormat); String outputFormat = sd.getOutputFormat(); if (outputFormat == null || outputFormat.length() == 0) { outputFormat = org.apache.hadoop.mapred.SequenceFileOutputFormat.class databaseName + "." + tableName); if (sd.getLocation() != null) { schema.setProperty( org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_LOCATION, schema.setProperty( org.apache.hadoop.hive.serde.Constants.SERIALIZATION_DDL, getDDLFromFieldSchema(tableName, sd.getCols()));
List<FieldSchema> partitionKeys) { Properties schema = new Properties(); String inputFormat = sd.getInputFormat(); if (inputFormat == null || inputFormat.length() == 0) { inputFormat = org.apache.hadoop.mapred.SequenceFileInputFormat.class org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT, inputFormat); String outputFormat = sd.getOutputFormat(); if (outputFormat == null || outputFormat.length() == 0) { outputFormat = org.apache.hadoop.mapred.SequenceFileOutputFormat.class databaseName + "." + tableName); if (sd.getLocation() != null) { schema.setProperty( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION, schema.setProperty( org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_DDL, getDDLFromFieldSchema(tableName, sd.getCols()));
NoSuchObjectException, UnsupportedOperationException { String dbName = table.getDbName().toLowerCase(); String tableName = table.getTableName().toLowerCase(); String pathStr = table.getSd().getLocation(); if (pathStr != null) { try { tablePath = new Path(table.getSd().getLocation()); if (!getWh().isWritable(tablePath.getParent())) { throw new MetaException("Table metadata not deleted since " + tablePath.getParent() + if (deleteData && !MetaStoreUtils.isExternalTable(table)) { try { boolean ifPurge = false;
List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys) throws HiveException { try { if (tbl.getDbName() == null || "".equals(tbl.getDbName().trim())) { tbl.setDbName(SessionState.get().getCurrentDatabase()); if (tbl.getCols().size() == 0 || tbl.getSd().getColsSize() == 0) { tbl.setFields(MetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(), tbl.getDeserializer())); principalPrivs.setGroupPrivileges(grants.getGroupGrants()); principalPrivs.setRolePrivileges(grants.getRoleGrants()); tTbl.setPrivileges(principalPrivs); throw new HiveException(e); throw new HiveException(e);
public static Properties getPartitionMetadata( org.apache.hadoop.hive.metastore.api.Partition partition, org.apache.hadoop.hive.metastore.api.Table table) { return MetaStoreUtils .getSchema(partition.getSd(), partition.getSd(), partition .getParameters(), table.getDbName(), table.getTableName(), table.getPartitionKeys()); }
String tableName = tbl.getTableName(); String tblLoc = null; String inputFormattCls = null; if (tbl.getPath() != null) { tblLoc = tbl.getDataLocation().toString(); inputFormattCls = tbl.getInputFormatClass().getName(); String owner = tbl.getOwner(); List<FieldSchema> cols = tbl.getCols(); String ddlCols = MetaStoreUtils.getDDLFromFieldSchema("columns", cols); boolean isPartitioned = tbl.isPartitioned(); String partitionCols = ""; if (isPartitioned) { partitionCols = MetaStoreUtils.getDDLFromFieldSchema( "partition_columns", tbl.getPartCols()); throw new HiveException(e);
public List<FieldSchema> get_fields(String db, String tableName) throws MetaException, UnknownTableException, UnknownDBException { startFunction("get_fields", ": db=" + db + "tbl=" + tableName); String[] names = tableName.split("\\."); String base_table_name = names[0]; Table tbl; try { try { tbl = get_table(db, base_table_name); } catch (NoSuchObjectException e) { throw new UnknownTableException(e.getMessage()); } boolean getColsFromSerDe = SerDeUtils.shouldGetColsFromSerDe( tbl.getSd().getSerdeInfo().getSerializationLib()); if (!getColsFromSerDe) { return tbl.getSd().getCols(); } else { try { Deserializer s = MetaStoreUtils.getDeserializer(hiveConf, tbl); return MetaStoreUtils.getFieldsFromDeserializer(tableName, s); } catch (SerDeException e) { StringUtils.stringifyException(e); throw new MetaException(e.getMessage()); } } } finally { endFunction("get_fields"); } }
private List<FieldSchema> getColsInternal(boolean forMs) { try { String serializationLib = tPartition.getSd().getSerdeInfo().getSerializationLib(); // Do the lightweight check for general case. if (Table.hasMetastoreBasedSchema(SessionState.getSessionConf(), serializationLib)) { return tPartition.getSd().getCols(); } else if (forMs && !Table.shouldStoreFieldsInMetastore( SessionState.getSessionConf(), serializationLib, table.getParameters())) { return Hive.getFieldsFromDeserializerForMsStorage(table, getDeserializer()); } return MetaStoreUtils.getFieldsFromDeserializer(table.getTableName(), getDeserializer()); } catch (Exception e) { LOG.error("Unable to get cols from serde: " + tPartition.getSd().getSerdeInfo().getSerializationLib(), e); } return new ArrayList<FieldSchema>(); }
public List<FieldSchema> getCols() { String serializationLib = getSerializationLib(); try { if (hasMetastoreBasedSchema(SessionState.getSessionConf(), serializationLib)) { return tTable.getSd().getCols(); } else { return MetaStoreUtils.getFieldsFromDeserializer(getTableName(), getDeserializer()); } } catch (Exception e) { LOG.error("Unable to get field from serde: " + serializationLib, e); } return new ArrayList<FieldSchema>(); }
public List<FieldSchema> getCols() { try { if (Table.hasMetastoreBasedSchema(SessionState.getSessionConf(), tPartition.getSd())) { return tPartition.getSd().getCols(); } return MetaStoreUtils.getFieldsFromDeserializer(table.getTableName(), getDeserializer()); } catch (Exception e) { LOG.error("Unable to get cols from serde: " + tPartition.getSd().getSerdeInfo().getSerializationLib(), e); } return new ArrayList<FieldSchema>(); }
org.apache.hadoop.hive.metastore.api.Partition part, org.apache.hadoop.hive.metastore.api.Table table) throws MetaException { String lib = part.getSd().getSerdeInfo().getSerializationLib(); try { Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(lib). asSubclass(Deserializer.class), conf); SerDeUtils.initializeSerDe(deserializer, conf, MetaStoreUtils.getTableMetadata(table), MetaStoreUtils.getPartitionMetadata(part, table)); return deserializer; } catch (RuntimeException e) {
/** * Wrapper around {@link MetaStoreUtils#getPartitionMetadata(Partition, Table)} which also adds parameters from table * to properties returned by {@link MetaStoreUtils#getPartitionMetadata(Partition, Table)}. * * @param partition the source of partition level parameters * @param table the source of table level parameters * @return properties */ public static Properties getPartitionMetadata(final Partition partition, final Table table) { final Properties properties = MetaStoreUtils.getPartitionMetadata(partition, table); // SerDe expects properties from Table, but above call doesn't add Table properties. // Include Table properties in final list in order to not to break SerDes that depend on // Table properties. For example AvroSerDe gets the schema from properties (passed as second argument) for (Map.Entry<String, String> entry : table.getParameters().entrySet()) { if (entry.getKey() != null && entry.getKey() != null) { properties.put(entry.getKey(), entry.getValue()); } } return properties; }
org.apache.hadoop.hive.metastore.api.Partition part, org.apache.hadoop.hive.metastore.api.Table table) throws MetaException { String lib = part.getSd().getSerdeInfo().getSerializationLib(); try { Deserializer deserializer = SerDeUtils.lookupDeserializer(lib); deserializer.initialize(conf, MetaStoreUtils.getPartitionMetadata(part, table)); return deserializer; } catch (RuntimeException e) { LOG.error("error in initSerDe: " + e.getClass().getName() + " " + e.getMessage()); MetaStoreUtils.printStackTrace(e); throw new MetaException(e.getClass().getName() + " " + e.getMessage());