/** * Gets the bucket columns. * * @return the bucket columns */ public List<String> getBucketCols() { return this.sd.getBucketCols(); }
/** * Gets the bucket columns. * * @return the bucket columns */ public List<String> getBucketCols() { return this.sd.getBucketCols(); }
public String getBucketingDimensionId() { List<String> bcols = tTable.getSd().getBucketCols(); if (bcols == null || bcols.size() == 0) { return null; } if (bcols.size() > 1) { LOG.warn(this + " table has more than one dimensions which aren't supported yet"); } return bcols.get(0); }
public List<String> getBucketCols() { return tTable.getSd().getBucketCols(); }
public List<String> getBucketCols() { return tPartition.getSd().getBucketCols(); }
public String getBucketingDimensionId() { List<String> bcols = tTable.getSd().getBucketCols(); if (bcols == null || bcols.size() == 0) { return null; } if (bcols.size() > 1) { LOG.warn(this + " table has more than one dimensions which aren't supported yet"); } return bcols.get(0); }
public List<String> getBucketCols() { return tTable.getSd().getBucketCols(); }
public List<String> getBucketCols() { return tPartition.getSd().getBucketCols(); }
public static Optional<HiveBucketProperty> fromStorageDescriptor(StorageDescriptor storageDescriptor, String tablePartitionName) { boolean bucketColsSet = storageDescriptor.isSetBucketCols() && !storageDescriptor.getBucketCols().isEmpty(); boolean numBucketsSet = storageDescriptor.isSetNumBuckets() && storageDescriptor.getNumBuckets() > 0; if (!numBucketsSet) { // In Hive, a table is considered as not bucketed when its bucketCols is set but its numBucket is not set. return Optional.empty(); } if (!bucketColsSet) { throw new PrestoException(HIVE_INVALID_METADATA, "Table/partition metadata has 'numBuckets' set, but 'bucketCols' is not set: " + tablePartitionName); } List<SortingColumn> sortedBy = ImmutableList.of(); if (storageDescriptor.isSetSortCols()) { sortedBy = storageDescriptor.getSortCols().stream() .map(order -> SortingColumn.fromMetastoreApiOrder(order, tablePartitionName)) .collect(toImmutableList()); } return Optional.of(new HiveBucketProperty(storageDescriptor.getBucketCols(), storageDescriptor.getNumBuckets(), sortedBy)); }
protected void prepareBucketingFields() { this.isBucketed = table.getSd().getNumBuckets() > 0; // For unbucketed tables we have exactly 1 RecordUpdater (until HIVE-19208) for each AbstractRecordWriter which // ends up writing to a file bucket_000000. // See also {@link #getBucket(Object)} this.totalBuckets = isBucketed ? table.getSd().getNumBuckets() : 1; if (isBucketed) { this.bucketIds = getBucketColIDs(table.getSd().getBucketCols(), table.getSd().getCols()); this.bucketFieldData = new Object[bucketIds.size()]; this.bucketObjInspectors = getObjectInspectorsForBucketedCols(bucketIds, inputRowObjectInspector); this.bucketStructFields = new StructField[bucketIds.size()]; List<? extends StructField> allFields = inputRowObjectInspector.getAllStructFieldRefs(); for (int i = 0; i < bucketIds.size(); i++) { bucketStructFields[i] = allFields.get(bucketIds.get(i)); } } }
static Table assemble(TableWrapper wrapper, SharedCache sharedCache) { Table t = wrapper.getTable().deepCopy(); if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); if (sdCopy.getBucketCols() == null) { sdCopy.setBucketCols(Collections.emptyList()); } if (sdCopy.getSortCols() == null) { sdCopy.setSortCols(Collections.emptyList()); } if (sdCopy.getSkewedInfo() == null) { sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); t.setSd(sdCopy); } return t; }
static Partition assemble(PartitionWrapper wrapper, SharedCache sharedCache) { Partition p = wrapper.getPartition().deepCopy(); if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); if (sdCopy.getBucketCols() == null) { sdCopy.setBucketCols(Collections.emptyList()); } if (sdCopy.getSortCols() == null) { sdCopy.setSortCols(Collections.emptyList()); } if (sdCopy.getSkewedInfo() == null) { sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); p.setSd(sdCopy); } return p; }
public Object getFieldValue(_Fields field) { switch (field) { case COLS: return getCols(); case LOCATION: return getLocation(); case INPUT_FORMAT: return getInputFormat(); case OUTPUT_FORMAT: return getOutputFormat(); case COMPRESSED: return isCompressed(); case NUM_BUCKETS: return getNumBuckets(); case SERDE_INFO: return getSerdeInfo(); case BUCKET_COLS: return getBucketCols(); case SORT_COLS: return getSortCols(); case PARAMETERS: return getParameters(); case SKEWED_INFO: return getSkewedInfo(); case STORED_AS_SUB_DIRECTORIES: return isStoredAsSubDirectories(); } throw new IllegalStateException(); }
private static State getStorageProps(StorageDescriptor sd) { State storageProps = new State(); for (Map.Entry<String, String> entry : sd.getParameters().entrySet()) { storageProps.setProp(entry.getKey(), entry.getValue()); } if (sd.isSetLocation()) { storageProps.setProp(HiveConstants.LOCATION, sd.getLocation()); } if (sd.isSetInputFormat()) { storageProps.setProp(HiveConstants.INPUT_FORMAT, sd.getInputFormat()); } if (sd.isSetOutputFormat()) { storageProps.setProp(HiveConstants.OUTPUT_FORMAT, sd.getOutputFormat()); } if (sd.isSetCompressed()) { storageProps.setProp(HiveConstants.COMPRESSED, sd.isCompressed()); } if (sd.isSetNumBuckets()) { storageProps.setProp(HiveConstants.NUM_BUCKETS, sd.getNumBuckets()); } if (sd.isSetBucketCols()) { for (String bucketColumn : sd.getBucketCols()) { storageProps.appendToListProp(HiveConstants.BUCKET_COLUMNS, bucketColumn); } } if (sd.isSetStoredAsSubDirectories()) { storageProps.setProp(HiveConstants.STORED_AS_SUB_DIRS, sd.isStoredAsSubDirectories()); } return storageProps; }
/** * Convert a {@link Partition} into a {@link HivePartition}. */ public static HivePartition getHivePartition(Partition partition) { State partitionProps = getPartitionProps(partition); State storageProps = getStorageProps(partition.getSd()); State serDeProps = getSerDeProps(partition.getSd().getSerdeInfo()); HivePartition hivePartition = new HivePartition.Builder().withDbName(partition.getDbName()).withTableName(partition.getTableName()) .withPartitionValues(partition.getValues()).withProps(partitionProps).withStorageProps(storageProps) .withSerdeProps(serDeProps).build(); if (partition.getCreateTime() > 0) { hivePartition.setCreateTime(partition.getCreateTime()); } if (partition.getSd().getCols() != null) { hivePartition.setColumns(getColumns(partition.getSd().getCols())); } if (partition.getSd().getBucketCols() != null) { hivePartition.setBucketColumns(partition.getSd().getBucketCols()); } return hivePartition; }
/** * Convert a {@link Table} into a {@link HiveTable}. */ public static HiveTable getHiveTable(Table table) { State tableProps = getTableProps(table); State storageProps = getStorageProps(table.getSd()); State serDeProps = getSerDeProps(table.getSd().getSerdeInfo()); HiveTable hiveTable = new HiveTable.Builder().withDbName(table.getDbName()).withTableName(table.getTableName()) .withPartitionKeys(getColumns(table.getPartitionKeys())).withProps(tableProps).withStorageProps(storageProps) .withSerdeProps(serDeProps).build(); if (table.getCreateTime() > 0) { hiveTable.setCreateTime(table.getCreateTime()); } if (table.getSd().getCols() != null) { hiveTable.setColumns(getColumns(table.getSd().getCols())); } if (table.getSd().getBucketCols() != null) { hiveTable.setBucketColumns(table.getSd().getBucketCols()); } return hiveTable; }
private static AddPartitionDesc getBaseAddPartitionDescFromPartition( Path fromPath, String dbName, ImportTableDesc tblDesc, Partition partition, ReplicationSpec replicationSpec, HiveConf conf) throws MetaException, SemanticException { AddPartitionDesc partsDesc = new AddPartitionDesc(dbName, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0); partDesc.setInputFormat(partition.getSd().getInputFormat()); partDesc.setOutputFormat(partition.getSd().getOutputFormat()); partDesc.setNumBuckets(partition.getSd().getNumBuckets()); partDesc.setCols(partition.getSd().getCols()); partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib()); partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters()); partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); if (replicationSpec.isInReplicationScope() && tblDesc.isExternal() && !replicationSpec.isMigratingToExternalTable()) { String newLocation = ReplExternalTables .externalTableLocation(conf, partition.getSd().getLocation()); LOG.debug("partition {} has data location: {}", partition, newLocation); partDesc.setLocation(newLocation); } else { partDesc.setLocation(new Path(fromPath, Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString()); } return partsDesc; }
private static AddPartitionDesc getBaseAddPartitionDescFromPartition( Path fromPath, String dbname, ImportTableDesc tblDesc, Partition partition) throws MetaException, SemanticException { AddPartitionDesc partsDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0); partDesc.setInputFormat(partition.getSd().getInputFormat()); partDesc.setOutputFormat(partition.getSd().getOutputFormat()); partDesc.setNumBuckets(partition.getSd().getNumBuckets()); partDesc.setCols(partition.getSd().getCols()); partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib()); partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters()); partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); partDesc.setLocation(new Path(fromPath, Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString()); return partsDesc; }
/** * Converts a storage descriptor to a db-backed storage descriptor. It points the * storage descriptor's column descriptor to the one passed as an argument, * so it does not create a new mcolumn descriptor object. * @param sd the storage descriptor to wrap in a db-backed object * @param mcd the db-backed column descriptor * @return the db-backed storage descriptor object * @throws MetaException */ private MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd, MColumnDescriptor mcd) throws MetaException { if (sd == null) { return null; } return new MStorageDescriptor(mcd, sd .getLocation(), sd.getInputFormat(), sd.getOutputFormat(), sd .isCompressed(), sd.getNumBuckets(), convertToMSerDeInfo(sd .getSerdeInfo()), sd.getBucketCols(), convertToMOrders(sd.getSortCols()), sd.getParameters(), (null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo().getSkewedColNames(), convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValues()), covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories()); }
private Partition createPartition(List<String> vals, Table table) throws MetaException { return new PartitionBuilder() .inTable(table) .setValues(vals) .addPartParam("key1", "S1") .addPartParam("key2", "S2") .addPartParam(EXCLUDE_KEY_PREFIX + "key1", "e1") .addPartParam(EXCLUDE_KEY_PREFIX + "key2", "e2") .setBucketCols(table.getSd().getBucketCols()) .setSortCols(table.getSd().getSortCols()) .setSerdeName(table.getSd().getSerdeInfo().getName()) .setSerdeLib(table.getSd().getSerdeInfo().getSerializationLib()) .setSerdeParams(table.getSd().getSerdeInfo().getParameters()) .build(conf); }