public static View fieldTypesToView(String name, String sql, List<ViewFieldType> fieldTypes, List<String> context) { if (fieldTypes == null) { throw new NullPointerException(); } List<FieldType> fields = new ArrayList<>(); for (ViewFieldType sqlField : fieldTypes) { FieldType fieldType = new View.FieldType( sqlField.getName(), en(SqlTypeName.class, sqlField.getType()), sqlField.getPrecision(), sqlField.getScale(), en(TimeUnit.class, sqlField.getStartUnit()), en(TimeUnit.class, sqlField.getEndUnit()), sqlField.getFractionalSecondPrecision(), sqlField.getIsNullable() ); fields.add(fieldType); } return new View(name, sql, fields, context); }
public static List<ViewFieldType> viewToFieldTypes(List<FieldType> fields) { List<ViewFieldType> sqlFields = new ArrayList<>(); for (FieldType fieldType : fields) { ViewFieldType sqlField = new ViewFieldType(fieldType.getName(), fieldType.getType().getName()); sqlField.setPrecision(fieldType.getPrecision()); sqlField.setScale(fieldType.getScale()); sqlField.setStartUnit(name(fieldType.getStartUnit())); sqlField.setEndUnit(name(fieldType.getEndUnit())); sqlField.setFractionalSecondPrecision(fieldType.getFractionalSecondPrecision()); sqlField.setIsNullable(fieldType.getIsNullable()); sqlField.setTypeFamily(fieldType.getType().getFamily().toString()); sqlFields.add(sqlField); } return sqlFields; }
public void writeTo(Output output, RowType message) throws IOException { if(message.field != null) { for(com.dremio.service.namespace.dataset.proto.ViewFieldType field : message.field) { if(field != null) output.writeObject(1, field, com.dremio.service.namespace.dataset.proto.ViewFieldType.getSchema(), true); } } }
@Override public String apply(final ViewFieldType input) { return input.getName(); } });
public static List<ViewFieldType> getBatchSchemaFields(final BatchSchema batchSchema) { final List<ViewFieldType> fields = Lists.newArrayList(); for (int i = 0; i < batchSchema.getFieldCount(); i++) { final org.apache.arrow.vector.types.pojo.Field field = batchSchema.getColumn(i); final ViewFieldType viewField = new ViewFieldType(field.getName(), TypeInferenceUtils.getCalciteTypeFromMinorType(MajorTypeHelper.getMajorTypeForField(field).getMinorType()).toString()); final CompleteType completeType = CompleteType.fromField(field); final SqlTypeName sqlTypeName = TypeInferenceUtils.getCalciteTypeFromMinorType(getMajorTypeForField(field).getMinorType()); viewField.setPrecision(completeType.getPrecision()); viewField.setScale(completeType.getScale()); viewField.setIsNullable(true); viewField.setTypeFamily(sqlTypeName.getFamily().toString()); // TODO (AH) //viewField.setStartUnit(); //viewField.setEndUnit(); //viewField.setFractionalSecondPrecision(); fields.add(viewField); } return fields; } }
public ViewFieldType newMessage() { return new ViewFieldType(); }
@Override public QlikFieldType apply(ViewFieldType input) { SqlTypeName typeName = SqlTypeName.valueOf(input.getType()); switch (typeName) { case CHAR:
private DataType getColType(String columnName) throws DatasetVersionNotFoundException { for(ViewFieldType type : getDatasetConfig ().getSqlFieldsList()) { if (type.getName().equals(columnName)) { return DataTypeUtil.getDataType(SqlTypeName.get(type.getType())); } } throw new ClientErrorException("Given column '" + columnName + "' doesn't exist in dataset"); }
@Override public String apply(ViewFieldType input) { String name = input.getName(); // Qlik needs field names that contain spaces to be surrounded by " so we surround all field names. // If the field name contains a ", we escape it with "" (per http://help.qlik.com/en-US/sense/3.1/Subsystems/Hub/Content/Scripting/use-quotes-in-script.htm) return quoteString(name); } };
@Test public void testGeneratedOutput() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList( new ViewFieldType("testdimension", "VARCHAR"), new ViewFieldType("testdimension2", "CHAR"), new ViewFieldType("testmeasure", "INTEGER"), new ViewFieldType("testmeasure2", "REAL"), new ViewFieldType("testdetail", "MAP"), new ViewFieldType("testdetail2", "STRUCTURED"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("UNTITLED") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(DatasetTool.TMP_DATASET_PATH.toPathList()) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); assertTrue(script.contains(" DIMENSION \"testdimension\", \"testdimension2\"")); assertTrue(script.contains(" MEASURE \"testmeasure\", \"testmeasure2\"")); assertTrue(script.contains(" DETAIL \"testdetail\", \"testdetail2\"")); assertTrue(script.contains(" FROM \"tmp\".\"UNTITLED\"")); }
public DatasetSearchUI(DatasetConfig datasetConfig, CollaborationTag collaborationTag) { this.fullPath = datasetConfig.getFullPathList(); this.datasetType = datasetConfig.getType(); if (collaborationTag != null) { this.tags = collaborationTag.getTagsList(); } if (datasetType == DatasetType.VIRTUAL_DATASET) { final VirtualDataset virtualDataset = datasetConfig.getVirtualDataset(); this.parents = virtualDataset.getParentsList(); this.displayFullPath = fullPath; this.context = virtualDataset.getContextList(); this.fields = Lists.newArrayList(); for (ViewFieldType field: virtualDataset.getSqlFieldsList()) { fields.add(new DatasetFieldSearchUI(field.getName(), field.getType())); } this.datasetVersion = virtualDataset.getVersion(); } else { this.displayFullPath = fullPath; this.context = null; this.parents = null; this.fields = null; this.datasetVersion = null; } }
@Override public String apply(ViewFieldType field) { return field.getName(); } });
public void writeTo(Output output, RowType message) throws IOException { if(message.field != null) { for(com.dremio.service.namespace.dataset.proto.ViewFieldType field : message.field) { if(field != null) output.writeObject(1, field, com.dremio.service.namespace.dataset.proto.ViewFieldType.getSchema(), true); } } }
@Test public void testQuoteInPathAndDatasetName() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("test dimension", "VARCHAR"), new ViewFieldType("test \" measure", "INTEGER"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("UNTITLED") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(Arrays.asList("@dremio", "fol\"der.ext", "foo", "bar")) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); // make sure everything is escaped correctly assertTrue(script.contains(" DIMENSION \"test dimension\"")); assertTrue(script.contains(" MEASURE \"test \"\" measure\"")); assertTrue(script.contains(" FROM \"@dremio\".\"fol\"\"der.ext\".\"foo\".\"bar\"")); }
@Override public boolean apply(ViewFieldType input) { return !UPDATE_COLUMN.equals(input.getName()); } }).toList();
public void mergeFrom(Input input, RowType message) throws IOException { for(int number = input.readFieldNumber(this);; number = input.readFieldNumber(this)) { switch(number) { case 0: return; case 1: if(message.field == null) message.field = new ArrayList<com.dremio.service.namespace.dataset.proto.ViewFieldType>(); message.field.add(input.mergeObject(null, com.dremio.service.namespace.dataset.proto.ViewFieldType.getSchema())); break; default: input.handleUnknownField(number, this); } } }
@Test public void testQuoting() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("testdimension", "VARCHAR"), new ViewFieldType("testmeasure", "INTEGER"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("UNTITLED") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(Arrays.asList("space", "folder.ext", "UNTITLED")) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); assertTrue(script.contains(" DIMENSION \"testdimension\"")); assertTrue(script.contains(" MEASURE \"testmeasure\"")); assertTrue(script.contains(" FROM \"space\".\"folder.ext\".\"UNTITLED\"")); }
public static String[] getColumnsLowerCase(DatasetConfig datasetConfig) { final ByteString schemaBytes = DatasetHelper.getSchemaBytes(datasetConfig); if (schemaBytes != null) { Schema schema = Schema.getRootAsSchema(schemaBytes.asReadOnlyByteBuffer()); org.apache.arrow.vector.types.pojo.Schema s = org.apache.arrow.vector.types.pojo.Schema.convertSchema(schema); return s.getFields().stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new); } else { // If virtual dataset was created with view fields if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) { final List<ViewFieldType> viewFieldTypes = datasetConfig.getVirtualDataset().getSqlFieldsList(); if (notEmpty(viewFieldTypes)) { return viewFieldTypes.stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new); } } } return new String[0]; }