String name = spec.getName(); if (defaultSettings != null) { settings = Maps.newHashMap(defaultSettings); if (spec.getSettings() != null) { settings.putAll(spec.getSettings()); settings = spec.getSettings(); if (defaultSchema != null && spec.getSchema() == null) { schema = defaultSchema; } else { schema = spec.getSchema(); FormatSpecification actualSpec = new FormatSpecification(name, schema, settings); format.initialize(actualSpec); return format;
@Override public List<?> makeRow(ViewDetail object) { return Lists.newArrayList(object.getId(), object.getFormat().getName(), object.getTableName(), GSON.toJson(object.getFormat().getSchema()), GSON.toJson(object.getFormat().getSettings())); } }).build();
String formatName = formatSpec.getName(); if (formatName == null) { throw new BadRequestException("A format name must be specified."); formatSpec = new FormatSpecification(formatSpec.getName(), format.getSchema(), formatSpec.getSettings()); } catch (UnsupportedTypeException e) { throw new BadRequestException("Format " + formatName + " does not support the requested schema.");
@Nullable @Override public String getSchemaToAdd() { Schema schema = config.getFormat().getSchema(); return schema == null ? null : schema.toString(); } }
private FormatSpecification getFormatSpec() { FormatSpecification formatSpec = null; if (!Strings.isNullOrEmpty(format)) { // try to parse the schema if there is one Schema schemaObj = parseSchema(); // strip format.settings. from any properties and use them in the format spec ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); for (Map.Entry<String, String> entry : getProperties().getProperties().entrySet()) { if (entry.getKey().startsWith(FORMAT_SETTING_PREFIX)) { String key = entry.getKey(); builder.put(key.substring(FORMAT_SETTING_PREFIX.length(), key.length()), entry.getValue()); } } formatSpec = new FormatSpecification(format, schemaObj, builder.build()); } return formatSpec; }
/** * Initialize the format with the given desired schema and properties. * Guaranteed to be called once before any other method is called. * * @param formatSpecification the specification for the format, containing the desired schema and settings * @throws UnsupportedTypeException if the desired schema and properties are not supported */ public void initialize(@Nullable FormatSpecification formatSpecification) throws UnsupportedTypeException { Schema desiredSchema = null; Map<String, String> settings = Collections.emptyMap(); if (formatSpecification != null) { desiredSchema = formatSpecification.getSchema(); settings = formatSpecification.getSettings(); } desiredSchema = desiredSchema == null ? getDefaultSchema() : desiredSchema; if (desiredSchema == null) { String msg = "A schema must be provided to the format: "; if (formatSpecification != null) { msg += formatSpecification.getName(); } throw new UnsupportedTypeException(msg); } validateIsRecord(desiredSchema); validateSchema(desiredSchema); this.schema = desiredSchema; configure(settings); }
@Nullable @Override public String getSchemaToAdd() { Schema schema = viewSpec.getFormat().getSchema(); if (schema == null) { FormatSpecification format = viewSpec.getFormat(); RecordFormat<Object, Object> initializedFormat; try { initializedFormat = RecordFormats.createInitializedFormat(format); schema = initializedFormat.getSchema(); } catch (IllegalAccessException | InstantiationException | UnsupportedTypeException | ClassNotFoundException e) { LOG.debug("Exception: ", e); LOG.warn("Exception while determining schema for view {}. View {} will not contain schema as metadata.", viewId, viewId); } } return schema == null ? null : schema.toString(); } }
@Override public void perform(Arguments arguments, PrintStream output) throws Exception { StreamId streamId = cliConfig.getCurrentNamespace().stream(arguments.get(ArgumentName.STREAM.toString())); StreamViewId viewId = streamId.view(arguments.get(ArgumentName.VIEW.toString())); String formatName = arguments.get(ArgumentName.FORMAT.toString()); Schema schema = getSchema(arguments); Map<String, String> settings = Collections.emptyMap(); if (arguments.hasArgument(ArgumentName.SETTINGS.toString())) { settings = ArgumentParser.parseMap(arguments.get(ArgumentName.SETTINGS.toString()), ArgumentName.SETTINGS.toString()); } FormatSpecification formatSpecification = new FormatSpecification(formatName, schema, settings); ViewSpecification viewSpecification = new ViewSpecification(formatSpecification); boolean created = client.createOrUpdate(viewId, viewSpecification); if (created) { output.printf("Successfully created stream-view '%s'\n", viewId.getEntityName()); } else { output.printf("Successfully updated stream-view '%s'\n", viewId.getEntityName()); } }
String name = spec.getName(); if (defaultSettings != null) { settings = Maps.newHashMap(defaultSettings); if (spec.getSettings() != null) { settings.putAll(spec.getSettings()); settings = spec.getSettings(); if (defaultSchema != null && spec.getSchema() == null) { schema = defaultSchema; } else { schema = spec.getSchema(); FormatSpecification actualSpec = new FormatSpecification(name, schema, settings); format.initialize(actualSpec); return format;
/** * Initialize the format with the given desired schema and properties. * Guaranteed to be called once before any other method is called. * * @param formatSpecification the specification for the format, containing the desired schema and settings * @throws UnsupportedTypeException if the desired schema and properties are not supported */ public void initialize(@Nullable FormatSpecification formatSpecification) throws UnsupportedTypeException { Schema desiredSchema = null; Map<String, String> settings = Collections.emptyMap(); if (formatSpecification != null) { desiredSchema = formatSpecification.getSchema(); settings = formatSpecification.getSettings(); } desiredSchema = desiredSchema == null ? getDefaultSchema() : desiredSchema; if (desiredSchema == null) { String msg = "A schema must be provided to the format: "; if (formatSpecification != null) { msg += formatSpecification.getName(); } throw new UnsupportedTypeException(msg); } validateIsRecord(desiredSchema); validateSchema(desiredSchema); this.schema = desiredSchema; configure(settings); }
@Override public void configurePipeline(PipelineConfigurer pipelineConfigurer) { super.configurePipeline(pipelineConfigurer); streamBatchConfig.validate(); if (!streamBatchConfig.containsMacro(Properties.Stream.NAME)) { pipelineConfigurer.addStream(new Stream(streamBatchConfig.name)); } // if no format is specified then default schema is used, if otherwise its based on format spec. if (streamBatchConfig.format == null) { pipelineConfigurer.getStageConfigurer().setOutputSchema(DEFAULT_SCHEMA); } else if (streamBatchConfig.getFormatSpec() != null && streamBatchConfig.getFormatSpec().getSchema() != null) { List<Schema.Field> fields = Lists.newArrayList(); fields.add(Schema.Field.of("ts", Schema.of(Schema.Type.LONG))); fields.add(Schema.Field.of("headers", Schema.mapOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.STRING)))); fields.addAll(streamBatchConfig.getFormatSpec().getSchema().getFields()); pipelineConfigurer.getStageConfigurer().setOutputSchema(Schema.recordOf("event", fields)); } }
@Override public void perform(Arguments arguments, PrintStream output) throws Exception { StreamId streamId = cliConfig.getCurrentNamespace().stream(arguments.get(ArgumentName.STREAM.toString())); StreamProperties currentProperties = streamClient.getConfig(streamId); String formatName = arguments.get(ArgumentName.FORMAT.toString()); Schema schema = getSchema(arguments); Map<String, String> settings = Collections.emptyMap(); if (arguments.hasArgument(ArgumentName.SETTINGS.toString())) { settings = ArgumentParser.parseMap(arguments.get(ArgumentName.SETTINGS.toString()), ArgumentName.SETTINGS.toString()); } FormatSpecification formatSpecification = new FormatSpecification(formatName, schema, settings); StreamProperties streamProperties = new StreamProperties(currentProperties.getTTL(), formatSpecification, currentProperties.getNotificationThresholdMB(), currentProperties.getDescription()); streamClient.setStreamProperties(streamId, streamProperties); output.printf("Successfully set format of stream '%s'\n", streamId.getEntityName()); }
schema = formatSpec.getSchema();