@Override public String apply(Field field) { return field.getName(); } });
@Nullable @Override public String apply(@Nullable final Field field) { return field.getName(); } })
private CompositeReaderConfig(List<SchemaPath> innerColumns, List<FieldValuePair> partitionFields) { super(); this.innerColumns = ImmutableList.copyOf(innerColumns); this.partitionFieldMap = FluentIterable.from(partitionFields).uniqueIndex(input -> input.field.getName()); }
@Override public String visit(List type) { StringBuilder sb = new StringBuilder(); if(includeName){ sb.append(field.getName()); sb.append("::"); } sb.append("list<"); sb.append(describe(field.getChildren().get(0), false)); sb.append(">"); return sb.toString(); }
private static Map<String, Integer> getColumnIndicesFromSchema(BatchSchema schema){ ImmutableMap.Builder<String, Integer> columns = ImmutableMap.builder(); for (int i = 0; i < schema.getFieldCount(); ++i) { final Field column = schema.getColumn(i); final String name = column.getName(); columns.put(name, i); } return columns.build(); } }
private List<NamedExpression> getExpressionList() { List<NamedExpression> exprs = Lists.newArrayList(); for (Field field : incoming.getSchema()) { if (field.getName().equalsIgnoreCase(config.getColumn().getAsUnescapedPath())) { continue; } exprs.add(new NamedExpression(SchemaPath.getSimplePath(field.getName()), new FieldReference(field.getName()))); } return exprs; }
public static ImmutableMap<String, Column> getColumnsFromSchema(BatchSchema schema){ ImmutableMap.Builder<String, Column> columns = ImmutableMap.builder(); for (int i = 0; i < schema.getFieldCount(); ++i) { final Field column = schema.getColumn(i); final String name = column.getName(); final MajorType type = getMajorTypeForField(column); DataType dataType = DataTypeUtil.getDataType(type); columns.put(name, new Column(name, dataType, i)); } return columns.build(); }
@Override public SchemaPath apply(Field input) { return SchemaPath.getSimplePath(input.getName()); }}).toList(); return new SystemScanPrel(cluster, traits, Mockito.mock(RelOptTable.class), metadata, columns, 1.0d, rowType);
public void materialize(Field field) { List<Field> children = field.getChildren(); for (Field child : children) { FieldVector v = TypeHelper.getNewVector(child, structVector.allocator, structVector.callBack); ComplexTypeHelper.materialize(v, child); structVector.putChild(child.getName(), v); } }
private static Set<String> extractSchema(RecordBatchData batch) { Set<String> schema = Sets.newHashSet(); for (ValueVector vv : batch.getVectors()) { schema.add(vv.getField().getName()); } return schema; }
@Override public Field visitGeneric(ArrowType type) { if(field.getName().equals(column.getAsUnescapedPath())){ throw UserException.validationError().message("You're trying to flatten a field that is not a list. The offending field is %s.", Describer.describe(field)).build(logger); } return super.visitGeneric(type); }
private static void assertContainsFields(Set<String> fields, List<ValueVector> original) { assertEquals(original.size(), fields.size()); for (ValueVector vv : original) { final String fieldName = vv.getField().getName(); assertTrue("original field '" + fieldName + "' is not part of the new schema", fields.contains(fieldName)); } }
public void removeField(Field field) throws SchemaChangeException { ValueVector vector = fieldVectorMap.remove(field.getName().toLowerCase()); if (vector == null) { throw new SchemaChangeException("Failure attempting to remove an unknown field."); } container.remove(vector); vector.close(); }
@SuppressWarnings("unchecked") @Override public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) { try{ TransferPair tp = vector.getTransferPair(vector.getField().getName(), allocator, callback); tp.transfer(); return new SimpleVectorWrapper<T>((T) tp.getTo()); }catch(RuntimeException ex){ throw ex; } }
public void addIfMissing(Field field, boolean isHyper) { final TypedFieldId id = FieldIdUtil2.getFieldId(getPartialSchema(), SchemaPath.getSimplePath(field.getName())); if(id == null){ if(isHyper){ addEmptyHyper(field); }else{ ValueVector vector = TypeHelper.getNewVector(field, allocator, resetter); add(vector); } } }
public SerializedField getMetadata() { SerializedField.Builder b = SerializedField.newBuilder() .setNamePart(NamePart.newBuilder().setName(structVector.getField().getName())) .setMajorType(Types.optional(MinorType.STRUCT)) .setBufferLength(structVector.getBufferSize()) .setValueCount(structVector.valueCount); for(ValueVector v : structVector.getChildren()) { b.addChild(TypeHelper.getMetadata(v)); } return b.build(); } }
private Map<SchemaPath, MajorType> getTypeMapFromBatch(QueryDataBatch batch) { Map<SchemaPath, MajorType> typeMap = new TreeMap<>(); for (int i = 0; i < batch.getHeader().getDef().getFieldCount(); i++) { typeMap.put(SchemaPath.getSimplePath(SerializedFieldHelper.create(batch.getHeader().getDef().getField(i)).getName()), batch.getHeader().getDef().getField(i).getMajorType()); } return typeMap; }
private Reflection createReflection() { // create a reflection List<ReflectionField> displayFields = new ArrayList<>(); DremioTable table = newCatalogService().getCatalog(SchemaConfig.newBuilder(SystemUser.SYSTEM_USERNAME).build()).getTable(datasetId); for (int i = 0; i < table.getSchema().getFieldCount(); i++) { Field field = table.getSchema().getColumn(i); displayFields.add(new ReflectionField(field.getName())); } return Reflection.newRawReflection(null, "My Raw", null, null, null, table.getDatasetConfig().getId().getId(), null, null, true, null, displayFields, null, null, null, PartitionDistributionStrategy.CONSOLIDATED); }
public SerializedField.Builder getMetadataBuilder() { return SerializedField.newBuilder() .setMajorType(MajorType.newBuilder().setMinorType(MinorType.LIST).setMode(DataMode.OPTIONAL).build()) .setNamePart(NamePart.newBuilder().setName(listVector.getField().getName())) .setValueCount(listVector.getValueCount()) .setBufferLength(listVector.getBufferSize()) .addChild(buildOffsetMetadata()) .addChild(buildValidityMetadata()) .addChild(TypeHelper.getMetadata(listVector.vector)); }