private static void printInfo(Logger console, Dataset<?> dataset) { DatasetDescriptor desc = dataset.getDescriptor(); String schema = ColumnMappingParser.removeEmbeddedMapping( PartitionStrategyParser.removeEmbeddedStrategy(desc.getSchema())) .toString(true); Collection<String> properties = desc.listProperties(); console.info("\nDataset \"{}\":", dataset.getName()); console.info("\tURI: \"{}\"", dataset.getUri()); console.info("\tSchema: {}", indent(schema)); if (desc.isPartitioned()) { console.info("\tPartition strategy: {}", indent(desc.getPartitionStrategy().toString(true))); } else { console.info("\tNot partitioned"); } if (desc.isColumnMapped()) { console.info("\tColumn mapping: {}", indent(desc.getColumnMapping().toString(true))); } if (!properties.isEmpty()) { StringBuilder sb = new StringBuilder(); for (String prop : properties) { sb.append("\n\t\t").append(prop).append("=") .append(desc.getProperty(prop)); } console.info("\tProperties:{}", sb.toString()); } }
private static Schema getEmbeddedSchema(DatasetDescriptor descriptor) { // the SchemaManager stores schemas, so this embeds the column mapping and // partition strategy in the schema. the result is parsed by // AvroKeyEntitySchemaParser Schema schema = descriptor.getSchema(); if (descriptor.isColumnMapped()) { schema = ColumnMappingParser .embedColumnMapping(schema, descriptor.getColumnMapping()); } if (descriptor.isPartitioned()) { schema = PartitionStrategyParser .embedPartitionStrategy(schema, descriptor.getPartitionStrategy()); } return schema; }
private static Schema getEmbeddedSchema(DatasetDescriptor descriptor) { // the SchemaManager stores schemas, so this embeds the column mapping and // partition strategy in the schema. the result is parsed by // AvroKeyEntitySchemaParser Schema schema = descriptor.getSchema(); if (descriptor.isColumnMapped()) { schema = ColumnMappingParser .embedColumnMapping(schema, descriptor.getColumnMapping()); } if (descriptor.isPartitioned()) { schema = PartitionStrategyParser .embedPartitionStrategy(schema, descriptor.getPartitionStrategy()); } return schema; }
HBaseMetadataProvider.getTableName(name), descriptor.getSchema()); Preconditions.checkArgument(descriptor.isColumnMapped(), "Cannot create dataset %s: missing column mapping", name);
HBaseMetadataProvider.getTableName(name), descriptor.getSchema()); Preconditions.checkArgument(descriptor.isColumnMapped(), "Cannot create dataset %s: missing column mapping", name);
@Override public String toString() { Objects.ToStringHelper helper = Objects.toStringHelper(this) .add("format", format) .add("schema", schema) .add("location", location) .add("properties", properties) .add("partitionStrategy", partitionStrategy) .add("compressionType", compressionType); if (isColumnMapped()) { helper.add("columnMapping", columnMappings); } return helper.toString(); }
@Override public DatasetDescriptor update(String namespace, String name, DatasetDescriptor descriptor) { Preconditions.checkArgument(DEFAULT_NAMESPACE.equals(namespace), "Non-default namespaces are not supported"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); Preconditions.checkNotNull(descriptor, "Descriptor cannot be null"); Compatibility.checkAndWarn( namespace, HBaseMetadataProvider.getTableName(name), descriptor.getSchema()); Preconditions.checkArgument(descriptor.isColumnMapped(), "Cannot update dataset %s: missing column mapping", name); String tableName = getTableName(name); String entityName = getEntityName(name); schemaManager.refreshManagedSchemaCache(tableName, entityName); Schema newSchema = getEmbeddedSchema(descriptor); String schemaString = newSchema.toString(true); EntitySchema entitySchema = new AvroEntitySchema( newSchema, schemaString, descriptor.getColumnMapping()); if (!schemaManager.hasSchemaVersion(tableName, entityName, entitySchema)) { schemaManager.migrateSchema(tableName, entityName, schemaString); } else { LOG.info("Schema hasn't changed, not migrating: (" + name + ")"); } return getDatasetDescriptor(newSchema, descriptor.getLocation()); }
@Override public DatasetDescriptor update(String namespace, String name, DatasetDescriptor descriptor) { Preconditions.checkArgument(DEFAULT_NAMESPACE.equals(namespace), "Non-default namespaces are not supported"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); Preconditions.checkNotNull(descriptor, "Descriptor cannot be null"); Compatibility.checkAndWarn( namespace, HBaseMetadataProvider.getTableName(name), descriptor.getSchema()); Preconditions.checkArgument(descriptor.isColumnMapped(), "Cannot update dataset %s: missing column mapping", name); String tableName = getTableName(name); String entityName = getEntityName(name); schemaManager.refreshManagedSchemaCache(tableName, entityName); Schema newSchema = getEmbeddedSchema(descriptor); String schemaString = newSchema.toString(true); EntitySchema entitySchema = new AvroEntitySchema( newSchema, schemaString, descriptor.getColumnMapping()); if (!schemaManager.hasSchemaVersion(tableName, entityName, entitySchema)) { schemaManager.migrateSchema(tableName, entityName, schemaString); } else { LOG.info("Schema hasn't changed, not migrating: (" + name + ")"); } return getDatasetDescriptor(newSchema, descriptor.getLocation()); }
return false; Preconditions.checkState(descriptor.isColumnMapped(), "[BUG] Existing descriptor has no column mapping");
return false; Preconditions.checkState(descriptor.isColumnMapped(), "[BUG] Existing descriptor has no column mapping");