md.update(sd.getOutputFormat().getBytes(ENCODING)); md.update(sd.isCompressed() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING)); md.update(Integer.toString(sd.getNumBuckets()).getBytes(ENCODING)); if (sd.getSerdeInfo() != null) {
formatOutput("InputFormat:", storageDesc.getInputFormat(), tableInfo); formatOutput("OutputFormat:", storageDesc.getOutputFormat(), tableInfo); formatOutput("Compressed:", storageDesc.isCompressed() ? "Yes" : "No", tableInfo); formatOutput("Num Buckets:", String.valueOf(storageDesc.getNumBuckets()), tableInfo); formatOutput("Bucket Columns:", storageDesc.getBucketCols().toString(), tableInfo);
formatOutput("InputFormat:", storageDesc.getInputFormat(), tableInfo); formatOutput("OutputFormat:", storageDesc.getOutputFormat(), tableInfo); formatOutput("Compressed:", storageDesc.isCompressed() ? "Yes" : "No", tableInfo); formatOutput("Num Buckets:", String.valueOf(storageDesc.getNumBuckets()), tableInfo); formatOutput("Bucket Columns:", storageDesc.getBucketCols().toString(), tableInfo);
job.set(INPUT_FORMAT_CLASS_NAME, sd.getInputFormat()); job.set(OUTPUT_FORMAT_CLASS_NAME, sd.getOutputFormat()); job.setBoolean(IS_COMPRESSED, sd.isCompressed()); job.set(TABLE_PROPS, new StringableMap(t.getParameters()).toString()); job.setInt(NUM_BUCKETS, sd.getNumBuckets());
job.set(INPUT_FORMAT_CLASS_NAME, sd.getInputFormat()); job.set(OUTPUT_FORMAT_CLASS_NAME, sd.getOutputFormat()); job.setBoolean(IS_COMPRESSED, sd.isCompressed()); job.set(TABLE_PROPS, new StringableMap(t.getParameters()).toString()); job.setInt(NUM_BUCKETS, sd.getNumBuckets());
if (sd.isCompressed()) { throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a compressed partition from Pig/Mapreduce is not supported");
public Object getFieldValue(_Fields field) { switch (field) { case COLS: return getCols(); case LOCATION: return getLocation(); case INPUT_FORMAT: return getInputFormat(); case OUTPUT_FORMAT: return getOutputFormat(); case COMPRESSED: return isCompressed(); case NUM_BUCKETS: return getNumBuckets(); case SERDE_INFO: return getSerdeInfo(); case BUCKET_COLS: return getBucketCols(); case SORT_COLS: return getSortCols(); case PARAMETERS: return getParameters(); case SKEWED_INFO: return getSkewedInfo(); case STORED_AS_SUB_DIRECTORIES: return isStoredAsSubDirectories(); } throw new IllegalStateException(); }
public StorageDescriptorWrapper(StorageDescriptor storageDescriptor) { sd = storageDescriptor; location = storageDescriptor.getLocation(); inputFormat = storageDescriptor.getInputFormat(); outputFormat = storageDescriptor.getOutputFormat(); compressed = storageDescriptor.isCompressed(); numBuckets = storageDescriptor.getNumBuckets(); serDeInfo = new SerDeInfoWrapper(storageDescriptor.getSerdeInfo()); if (sd.getSortCols() != null) { sortCols = Lists.newArrayList(); for (Order order : sd.getSortCols()) { sortCols.add(new OrderWrapper(order)); } } parameters = storageDescriptor.getParameters(); if (sd.getCols() != null) { this.columns = Lists.newArrayList(); for (FieldSchema fieldSchema : sd.getCols()) { this.columns.add(new FieldSchemaWrapper(fieldSchema)); } } }
private static State getStorageProps(StorageDescriptor sd) { State storageProps = new State(); for (Map.Entry<String, String> entry : sd.getParameters().entrySet()) { storageProps.setProp(entry.getKey(), entry.getValue()); } if (sd.isSetLocation()) { storageProps.setProp(HiveConstants.LOCATION, sd.getLocation()); } if (sd.isSetInputFormat()) { storageProps.setProp(HiveConstants.INPUT_FORMAT, sd.getInputFormat()); } if (sd.isSetOutputFormat()) { storageProps.setProp(HiveConstants.OUTPUT_FORMAT, sd.getOutputFormat()); } if (sd.isSetCompressed()) { storageProps.setProp(HiveConstants.COMPRESSED, sd.isCompressed()); } if (sd.isSetNumBuckets()) { storageProps.setProp(HiveConstants.NUM_BUCKETS, sd.getNumBuckets()); } if (sd.isSetBucketCols()) { for (String bucketColumn : sd.getBucketCols()) { storageProps.appendToListProp(HiveConstants.BUCKET_COLUMNS, bucketColumn); } } if (sd.isSetStoredAsSubDirectories()) { storageProps.setProp(HiveConstants.STORED_AS_SUB_DIRS, sd.isStoredAsSubDirectories()); } return storageProps; }
"org.apache.hadoop.hive.ql.io.HiveOutputFormat", sd.getOutputFormat()); Assert.assertFalse("The compressed attribute doesn't have the default value.", sd.isCompressed()); Assert.assertFalse("The storedAsSubDirectories attribute doesn't have the default value.", sd.isStoredAsSubDirectories());
/** * Converts a storage descriptor to a db-backed storage descriptor. It points the * storage descriptor's column descriptor to the one passed as an argument, * so it does not create a new mcolumn descriptor object. * @param sd the storage descriptor to wrap in a db-backed object * @param mcd the db-backed column descriptor * @return the db-backed storage descriptor object * @throws MetaException */ private MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd, MColumnDescriptor mcd) throws MetaException { if (sd == null) { return null; } return new MStorageDescriptor(mcd, sd .getLocation(), sd.getInputFormat(), sd.getOutputFormat(), sd .isCompressed(), sd.getNumBuckets(), convertToMSerDeInfo(sd .getSerdeInfo()), sd.getBucketCols(), convertToMOrders(sd.getSortCols()), sd.getParameters(), (null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo().getSkewedColNames(), convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValues()), covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories()); }
assertEquals(tbl2.getTableName(), tblName); assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size()); assertFalse(tbl2.getSd().isCompressed()); assertFalse(tbl2.getSd().isStoredAsSubDirectories()); assertEquals(tbl2.getSd().getNumBuckets(), 1);
assertEquals(tbl2.getTableName(), tblName); assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size()); assertEquals(tbl2.getSd().isCompressed(), false); assertEquals(tbl2.getSd().getNumBuckets(), 1); assertEquals(tbl2.getSd().getLocation(), tbl.getSd().getLocation()); assertEquals(tbl3.getTableName(), tblName2); assertEquals(tbl3.getSd().getCols().size(), typ1.getFields().size()); assertEquals(tbl3.getSd().isCompressed(), false); assertEquals(tbl3.getSd().getNumBuckets(), 1); assertEquals(tbl3.getSd().getLocation(), tbl2.getSd().getLocation()); assertEquals(t.getSd().isCompressed(), false); assertEquals(foundTables.get(0).getSd().getNumBuckets(), 1); assertNotNull(t.getSd().getSerdeInfo());
Assert.assertFalse("Storage descriptor compressed", createdSd.isCompressed()); Assert.assertEquals("Storage descriptor num buckets", 0, createdSd.getNumBuckets()); Assert.assertEquals("Storage descriptor bucket cols", 0, createdSd.getBucketCols().size());
/** * Create a CatalogPartition from the given PartitionSpec and Hive partition. */ public static CatalogPartition createCatalogPartition(CatalogPartition.PartitionSpec spec, Partition partition) { Map<String, String> prop = new HashMap<>(partition.getParameters()); StorageDescriptor sd = partition.getSd(); prop.put(HIVE_TABLE_LOCATION, sd.getLocation()); prop.put(HIVE_TABLE_SERDE_LIBRARY, sd.getSerdeInfo().getSerializationLib()); prop.put(HIVE_TABLE_INPUT_FORMAT, sd.getInputFormat()); prop.put(HIVE_TABLE_OUTPUT_FORMAT, sd.getOutputFormat()); prop.put(HIVE_TABLE_COMPRESSED, String.valueOf(sd.isCompressed())); prop.put(HIVE_TABLE_NUM_BUCKETS, String.valueOf(sd.getNumBuckets())); prop.putAll(partition.getParameters()); return new CatalogPartition(spec, prop); }
/** * Check table is not misconfigured. * @param table Table to check * @param outputDesc HiveOutputDescription to use */ private static void sanityCheck(Table table, HiveOutputDescription outputDesc) { StorageDescriptor sd = table.getSd(); Preconditions.checkArgument(!sd.isCompressed()); Preconditions.checkArgument(nullOrEmpty(sd.getBucketCols())); Preconditions.checkArgument(nullOrEmpty(sd.getSortCols())); Preconditions.checkArgument(table.getPartitionKeysSize() == outputDesc.numPartitionValues()); }
/** * Check table is not misconfigured. * @param table Table to check * @param outputDesc HiveOutputDescription to use */ private static void sanityCheck(Table table, HiveOutputDescription outputDesc) { StorageDescriptor sd = table.getSd(); Preconditions.checkArgument(!sd.isCompressed()); Preconditions.checkArgument(nullOrEmpty(sd.getBucketCols())); Preconditions.checkArgument(nullOrEmpty(sd.getSortCols())); Preconditions.checkArgument(table.getPartitionKeysSize() == outputDesc.numPartitionValues()); }
public Object getFieldValue(_Fields field) { switch (field) { case COLS: return getCols(); case LOCATION: return getLocation(); case INPUT_FORMAT: return getInputFormat(); case OUTPUT_FORMAT: return getOutputFormat(); case COMPRESSED: return new Boolean(isCompressed()); case NUM_BUCKETS: return new Integer(getNumBuckets()); case SERDE_INFO: return getSerdeInfo(); case BUCKET_COLS: return getBucketCols(); case SORT_COLS: return getSortCols(); case PARAMETERS: return getParameters(); } throw new IllegalStateException(); }
private static void getStorageDescriptorInfo(StringBuilder tableInfo, StorageDescriptor storageDesc) { formatOutput("SerDe Library:", storageDesc.getSerdeInfo().getSerializationLib(), tableInfo); formatOutput("InputFormat:", storageDesc.getInputFormat(), tableInfo); formatOutput("OutputFormat:", storageDesc.getOutputFormat(), tableInfo); formatOutput("Compressed:", storageDesc.isCompressed() ? "Yes" : "No", tableInfo); formatOutput("Num Buckets:", String.valueOf(storageDesc.getNumBuckets()), tableInfo); formatOutput("Bucket Columns:", storageDesc.getBucketCols().toString(), tableInfo); formatOutput("Sort Columns:", storageDesc.getSortCols().toString(), tableInfo); if (storageDesc.getSerdeInfo().getParametersSize() > 0) { tableInfo.append("Storage Desc Params:").append(LINE_DELIM); displayAllParameters(storageDesc.getSerdeInfo().getParameters(), tableInfo); } }
private MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd) throws MetaException { if (sd == null) { return null; } return new MStorageDescriptor(convertToMFieldSchemas(sd.getCols()), sd .getLocation(), sd.getInputFormat(), sd.getOutputFormat(), sd .isCompressed(), sd.getNumBuckets(), converToMSerDeInfo(sd .getSerdeInfo()), sd.getBucketCols(), convertToMOrders(sd.getSortCols()), sd.getParameters()); }