protected Schema getSchema() throws IOException { if (schema == null) { schema = liferayConnectionResourceBaseProperties.getSchema(); if (AvroUtils.isIncludeAllFields(schema)) { String resourceURL = null; if (liferayConnectionResourceBaseProperties instanceof TLiferayInputProperties) { resourceURL = liferayConnectionResourceBaseProperties.resource. resourceProperty.getResourceURL(); } BoundedSource boundedSource = getCurrentSource(); schema = boundedSource.getEndpointSchema( runtimeContainer, resourceURL); } } return schema; }
private void setSchema(Schema designSchema) { if (designSchema != null && AvroUtils.isIncludeAllFields(designSchema)) { this.defaultSchema = initDefaultSchema(designSchema); } else { this.defaultSchema = designSchema; } }
private void setSchema(Schema designSchema) { if (designSchema != null && AvroUtils.isIncludeAllFields(designSchema)) { this.defaultSchema = initDefaultSchema(designSchema); } else { this.defaultSchema = designSchema; } }
protected void setupDynamicSchema() throws IOException { if (schema != null) { if (AvroUtils.isIncludeAllFields(schema)) { schema = FileSourceOrSink.getDynamicSchema(values, "dynamicSchema", schema); } } else { throw new IOException("Schema is not setup!"); } }
@Override public Map<String, CustomFieldDesc> getCustomFields(RecordTypeInfo recordTypeInfo) { if (!AvroUtils.isIncludeAllFields(schema)) { if (customFieldDescMap == null) { customFieldDescMap = loadCustomFieldDescMap(); } return customFieldDescMap; } return defaultSource.getCustomFields(recordTypeInfo); }
protected Schema getRuntimeSchema(SchemaResolver resolver, TableAction.TableActionEnum tableAction) throws IOException { SnowflakeConnectionTableProperties connectionTableProperties = ((SnowflakeConnectionTableProperties) properties); Schema schema = connectionTableProperties.getSchema(); // Don't retrieve schema from database if there is a table action that will create the table if (AvroUtils.isIncludeAllFields(schema) && tableAction == TableAction.TableActionEnum.NONE) { schema = resolver.getSchema(); } return schema; }
@Override public IndexedRecord convertToAvro(DataPrepField[] dataPrepDataSetRecord) { if (AvroUtils.isIncludeAllFields(schema)) { AvroRegistry avroRegistry = DataPrepAvroRegistry.getDataPrepInstance(); schema = avroRegistry.inferSchema(dataPrepDataSetRecord); LOGGER.debug("Source schema is: {}", schema.toString(true)); } return new DataPrepIndexedRecord(dataPrepDataSetRecord); }
protected Schema getRuntimeSchema(SchemaResolver resolver, TableAction.TableActionEnum tableAction) throws IOException { SnowflakeConnectionTableProperties connectionTableProperties = ((SnowflakeConnectionTableProperties) properties); Schema schema = connectionTableProperties.getSchema(); // Don't retrieve schema from database if there is a table action that will create the table if (AvroUtils.isIncludeAllFields(schema) && tableAction == TableAction.TableActionEnum.NONE) { schema = resolver.getSchema(); } return schema; }
protected Schema getRuntimeSchema(RuntimeContainer container) throws IOException { SnowflakeConnectionTableProperties connectionTableProperties = ((SnowflakeConnectionTableProperties) properties); Schema schema = connectionTableProperties.getSchema(); if (AvroUtils.isIncludeAllFields(schema)) { schema = getEndpointSchema(container, connectionTableProperties.getTableName()); } return schema; }
protected Schema getSchema() throws IOException { if (querySchema == null) { querySchema = properties.module.main.schema.getValue(); if (AvroUtils.isIncludeAllFields(querySchema)) { String moduleName = null; if (properties instanceof SalesforceConnectionModuleProperties) { moduleName = properties.module.moduleName.getStringValue(); } querySchema = getCurrentSource().getEndpointSchema(container, moduleName); } } return querySchema; }
protected Schema getSchema() throws IOException { if (querySchema == null) { querySchema = properties.table.main.schema.getValue(); if (AvroUtils.isIncludeAllFields(querySchema)) { String tableName = null; if (properties instanceof SnowflakeConnectionTableProperties) { tableName = properties.table.tableName.getStringValue(); } querySchema = getCurrentSource().getEndpointSchema(container, tableName); } } return querySchema; }
protected Schema getSchema() throws IOException { if (querySchema == null) { querySchema = properties.module.main.schema.getValue(); if (AvroUtils.isIncludeAllFields(querySchema)) { String moduleName = null; if (properties instanceof SalesforceConnectionModuleProperties) { moduleName = properties.module.moduleName.getStringValue(); } querySchema = getCurrentSource().getEndpointSchema(container, moduleName); } } return querySchema; }
protected Schema getSchema() throws IOException { if (querySchema == null) { querySchema = properties.module.main.schema.getValue(); if (AvroUtils.isIncludeAllFields(querySchema)) { String moduleName = null; if (properties instanceof SalesforceConnectionModuleProperties) { moduleName = properties.module.moduleName.getStringValue(); } querySchema = getCurrentSource().getEndpointSchema(container, moduleName); } } return querySchema; }
private Schema getSchema() throws IOException { Schema designSchema = properties.schema.schema.getValue(); if (designSchema != null) { if (AvroUtils.isIncludeAllFields(designSchema)) { querySchema = AzureStorageAvroRegistry.get().inferSchema(current); } else { querySchema = designSchema; } } else { querySchema = AzureStorageAvroRegistry.get().inferSchema(current); } return querySchema; }
private Schema getSchema() throws IOException { Schema designSchema = properties.schema.schema.getValue(); if (designSchema != null) { if (AvroUtils.isIncludeAllFields(designSchema)) { querySchema = AzureStorageAvroRegistry.get().inferSchema(current); } else { querySchema = designSchema; } } else { querySchema = AzureStorageAvroRegistry.get().inferSchema(current); } return querySchema; }
@Override public void open(String uId) throws IOException { super.open(uId); properties = (TMarketoInputProperties) sink.getProperties(); flowSchema = properties.schemaFlow.schema.getValue(); dieOnError = properties.dieOnError.getValue(); isDynamic = AvroUtils.isIncludeAllFields(this.properties.schemaInput.schema.getValue()); leadKeyColumn = properties.leadKeyValues.getValue(); }
@Override public void open(String uId) throws IOException { super.open(uId); properties = (TMarketoInputProperties) sink.getProperties(); flowSchema = properties.schemaFlow.schema.getValue(); dieOnError = properties.dieOnError.getValue(); isDynamic = AvroUtils.isIncludeAllFields(this.properties.schemaInput.schema.getValue()); leadKeyColumn = properties.leadKeyValues.getValue(); }
@Override protected Schema getSchema() throws IOException { if (querySchema == null && AvroUtils.isIncludeAllFields(properties.module.main.schema.getValue())) { // This for bulk manual query dynamic to generate schema based on soql if (properties instanceof TSalesforceInputProperties) { TSalesforceInputProperties inProperties = (TSalesforceInputProperties) properties; if (inProperties.manualQuery.getValue()) { querySchema = ((SalesforceSource) getCurrentSource()).guessSchema(inProperties.query.getValue()); return querySchema; } } } return super.getSchema(); } }
@Override protected Schema getSchema() throws IOException { if (querySchema == null && AvroUtils.isIncludeAllFields(properties.module.main.schema.getValue())) { // This for bulk manual query dynamic to generate schema based on soql if (properties instanceof TSalesforceInputProperties) { TSalesforceInputProperties inProperties = (TSalesforceInputProperties) properties; if (inProperties.manualQuery.getValue()) { querySchema = ((SalesforceSource) getCurrentSource()).guessSchema(inProperties.query.getValue()); return querySchema; } } } return super.getSchema(); } }
@Override public ValidationResult initialize(RuntimeContainer container, JDBCInputProperties properties) { this.properties = properties; // In Beam, JdbcIO always has a repartition event, so we are obligated to fetch the schema before any processing // occurs in the nodes. Schema schema = properties.getDatasetProperties().main.schema.getValue(); if (schema == null || AvroUtils.isSchemaEmpty(schema) || AvroUtils.isIncludeAllFields(schema)) { JDBCDatasetRuntime schemaFetcher = new JDBCDatasetRuntime(); schemaFetcher.initialize(container, properties.getDatasetProperties()); schema = schemaFetcher.getSchema(); } this.defaultOutputCoder = AvroCoder.of(schema); return ValidationResult.OK; }