Refine search
@Test public void testEmptyColumns() { IonSqlQueryBuilder queryBuilder = new IonSqlQueryBuilder(new TypeRegistry()); assertEquals("SELECT ' ' FROM S3Object s", queryBuilder.buildSql(ImmutableList.of(), TupleDomain.all())); }
@Test public void testNonexistentType() { try { TypeManager typeManager = new TypeRegistry(); typeManager.getType(parseTypeSignature("not a real type")); fail("Expect to throw IllegalArgumentException"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().matches("Unknown type.*")); } catch (Throwable t) { fail("Expect to throw IllegalArgumentException, got " + t.getClass()); } }
private PartitionedOutputOperator createPartitionedOutputOperator() { PartitionFunction partitionFunction = new LocalPartitionGenerator(new InterpretedHashGenerator(ImmutableList.of(BIGINT), new int[] {0}), PARTITION_COUNT); PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new BlockEncodingManager(new TypeRegistry()), false); OutputBuffers buffers = createInitialEmptyOutputBuffers(PARTITIONED); for (int partition = 0; partition < PARTITION_COUNT; partition++) { buffers = buffers.withBuffer(new OutputBuffers.OutputBufferId(partition), partition); } PartitionedOutputBuffer buffer = createPartitionedBuffer( buffers.withNoMoreBufferIds(), new DataSize(Long.MAX_VALUE, BYTE)); // don't let output buffer block PartitionedOutputFactory operatorFactory = new PartitionedOutputFactory( partitionFunction, ImmutableList.of(0), ImmutableList.of(Optional.empty()), false, OptionalInt.empty(), buffer, new DataSize(1, GIGABYTE)); return (PartitionedOutputOperator) operatorFactory .createOutputOperator(0, new PlanNodeId("plan-node-0"), TYPES, Function.identity(), serdeFactory) .createOperator(createDriverContext()); }
@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "No free space available for spill") public void throwsIfNoDiskSpace() { List<Type> types = ImmutableList.of(BIGINT); BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(new TypeRegistry()); List<Path> spillPaths = ImmutableList.of(spillPath1.toPath(), spillPath2.toPath()); FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory( executor, // executor won't be closed, because we don't call destroy() on the spiller factory blockEncodingSerde, new SpillerStats(), spillPaths, 0.0); spillerFactory.create(types, bytes -> {}, newSimpleAggregatedMemoryContext().newLocalMemoryContext("test")); }
public void testSimpleExplicitSpecializedAggregationParse() { Signature expectedSignature = new Signature( "explicit_specialized_aggregate", FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), parseTypeSignature("T"), ImmutableList.of(new TypeSignature(ARRAY, TypeSignatureParameter.of(parseTypeSignature("T")))), false); ParametricAggregation aggregation = parseFunctionDefinition(ExplicitSpecializedAggregationFunction.class); assertEquals(aggregation.getDescription(), "Simple explicit specialized aggregate"); assertTrue(aggregation.isDeterministic()); assertEquals(aggregation.getSignature(), expectedSignature); ParametricImplementationsGroup<AggregationImplementation> implementations = aggregation.getImplementations(); assertImplementationCount(implementations, 0, 1, 1); AggregationImplementation implementation1 = implementations.getSpecializedImplementations().get(0); assertTrue(implementation1.hasSpecializedTypeParameters()); assertFalse(implementation1.hasSpecializedTypeParameters()); List<AggregationMetadata.ParameterMetadata.ParameterType> expectedMetadataTypes = ImmutableList.of(AggregationMetadata.ParameterMetadata.ParameterType.STATE, AggregationMetadata.ParameterMetadata.ParameterType.INPUT_CHANNEL); assertTrue(implementation1.getInputParameterMetadataTypes().equals(expectedMetadataTypes)); AggregationImplementation implementation2 = implementations.getSpecializedImplementations().get(1); assertTrue(implementation2.hasSpecializedTypeParameters()); assertFalse(implementation2.hasSpecializedTypeParameters()); assertTrue(implementation2.getInputParameterMetadataTypes().equals(expectedMetadataTypes)); InternalAggregationFunction specialized = aggregation.specialize(BoundVariables.builder().setTypeVariable("T", DoubleType.DOUBLE).build(), 1, new TypeRegistry(), null); assertEquals(specialized.getFinalType(), DoubleType.DOUBLE); assertTrue(specialized.isDecomposable()); assertEquals(specialized.name(), "implicit_specialized_aggregate"); }
@Test public void testDisplayName() { TypeManager typeManager = new TypeRegistry(); Type function = typeManager.getType(TypeSignature.parseTypeSignature("function<row(field double),bigint>")); assertEquals(function.getDisplayName(), "function<row(field double),bigint>"); } }
@Test public void testCleanupOldSpillFiles() throws Exception BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(new TypeRegistry()); List<Path> spillPaths = ImmutableList.of(spillPath1.toPath(), spillPath2.toPath()); spillPath1.mkdirs(); spillPath2.mkdirs();
public void testSimpleImplicitSpecializedAggregationParse() { Signature expectedSignature = new Signature( "implicit_specialized_aggregate", FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), parseTypeSignature("T"), ImmutableList.of(new TypeSignature(ARRAY, TypeSignatureParameter.of(parseTypeSignature("T"))), parseTypeSignature("T")), false); ParametricAggregation aggregation = parseFunctionDefinition(ImplicitSpecializedAggregationFunction.class); assertEquals(aggregation.getDescription(), "Simple implicit specialized aggregate"); assertTrue(aggregation.isDeterministic()); assertEquals(aggregation.getSignature(), expectedSignature); ParametricImplementationsGroup<AggregationImplementation> implementations = aggregation.getImplementations(); assertImplementationCount(implementations, 0, 0, 2); AggregationImplementation implementation1 = implementations.getSpecializedImplementations().get(0); assertTrue(implementation1.hasSpecializedTypeParameters()); assertFalse(implementation1.hasSpecializedTypeParameters()); List<AggregationMetadata.ParameterMetadata.ParameterType> expectedMetadataTypes = ImmutableList.of(AggregationMetadata.ParameterMetadata.ParameterType.STATE, AggregationMetadata.ParameterMetadata.ParameterType.INPUT_CHANNEL, AggregationMetadata.ParameterMetadata.ParameterType.INPUT_CHANNEL); assertTrue(implementation1.getInputParameterMetadataTypes().equals(expectedMetadataTypes)); AggregationImplementation implementation2 = implementations.getSpecializedImplementations().get(1); assertTrue(implementation2.hasSpecializedTypeParameters()); assertFalse(implementation2.hasSpecializedTypeParameters()); assertTrue(implementation2.getInputParameterMetadataTypes().equals(expectedMetadataTypes)); InternalAggregationFunction specialized = aggregation.specialize(BoundVariables.builder().setTypeVariable("T", DoubleType.DOUBLE).build(), 1, new TypeRegistry(), null); assertEquals(specialized.getFinalType(), DoubleType.DOUBLE); assertTrue(specialized.isDecomposable()); assertEquals(specialized.name(), "implicit_specialized_aggregate"); }
@Test public void testListingHiddenFunctions() { TypeRegistry typeManager = new TypeRegistry(); FunctionRegistry registry = new FunctionRegistry(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); List<SqlFunction> functions = registry.list(); List<String> names = transform(functions, input -> input.getSignature().getName()); assertTrue(names.contains("length"), "Expected function names " + names + " to contain 'length'"); assertTrue(names.contains("stddev"), "Expected function names " + names + " to contain 'stddev'"); assertTrue(names.contains("rank"), "Expected function names " + names + " to contain 'rank'"); assertFalse(names.contains("like"), "Expected function names " + names + " not to contain 'like'"); assertFalse(names.contains("$internal$sum_data_size_for_stats"), "Expected function names " + names + " not to contain '$internal$sum_data_size_for_stats'"); assertFalse(names.contains("$internal$max_data_size_for_stats"), "Expected function names " + names + " not to contain '$internal$max_data_size_for_stats'"); }
@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "No spill paths configured") public void throwIfNoSpillPaths() { List<Path> spillPaths = emptyList(); List<Type> types = ImmutableList.of(BIGINT); FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory( executor, // executor won't be closed, because we don't call destroy() on the spiller factory new BlockEncodingManager(new TypeRegistry()), new SpillerStats(), spillPaths, 1.0); spillerFactory.create(types, bytes -> {}, newSimpleAggregatedMemoryContext().newLocalMemoryContext("test")); }
public TestInformationSchemaMetadata() MockConnectorFactory mockConnectorFactory = builder.withListSchemaNames(connectorSession -> ImmutableList.of("test_schema")) .withListTables((connectorSession, schemaNameOrNull) -> ImmutableList.of( new SchemaTableName("test_schema", "test_view"), new SchemaTableName("test_schema", "another_table"))) .withGetViews((connectorSession, prefix) -> { String viewJson = VIEW_DEFINITION_JSON_CODEC.toJson(new ViewDefinition("select 1", Optional.of("test_catalog"), Optional.of("test_schema"), ImmutableList.of(), Optional.empty())); SchemaTableName viewName = new SchemaTableName("test_schema", "test_view"); return ImmutableMap.of(viewName, new ConnectorViewDefinition(viewName, Optional.empty(), viewJson)); metadata = new MetadataManager( new FeaturesConfig(), new TypeRegistry(), new BlockEncodingManager(new TypeRegistry()), new SessionPropertyManager(), new SchemaPropertyManager(),
@Test public void testCustomStateSerializerAggregationParse() { ParametricAggregation aggregation = parseFunctionDefinition(CustomStateSerializerAggregationFunction.class); AggregationImplementation implementation = getOnlyElement(aggregation.getImplementations().getExactImplementations().values()); assertTrue(implementation.getStateSerializerFactory().isPresent()); InternalAggregationFunction specialized = aggregation.specialize(BoundVariables.builder().build(), 1, new TypeRegistry(), null); AccumulatorStateSerializer<?> createdSerializer = getOnlyElement(((LazyAccumulatorFactoryBinder) specialized.getAccumulatorFactoryBinder()) .getGenericAccumulatorFactoryBinder().getStateDescriptors()).getSerializer(); Class<?> serializerFactory = implementation.getStateSerializerFactory().get().type().returnType(); assertTrue(serializerFactory.isInstance(createdSerializer)); }
@Test public void testNotAnnotatedAggregateStateAggregationParse() { ParametricAggregation aggregation = parseFunctionDefinition(NotAnnotatedAggregateStateAggregationFunction.class); AggregationImplementation implementation = getOnlyElement(aggregation.getImplementations().getExactImplementations().values()); List<AggregationMetadata.ParameterMetadata.ParameterType> expectedMetadataTypes = ImmutableList.of(AggregationMetadata.ParameterMetadata.ParameterType.STATE, AggregationMetadata.ParameterMetadata.ParameterType.INPUT_CHANNEL); assertTrue(implementation.getInputParameterMetadataTypes().equals(expectedMetadataTypes)); InternalAggregationFunction specialized = aggregation.specialize(BoundVariables.builder().build(), 1, new TypeRegistry(), null); assertEquals(specialized.getFinalType(), DoubleType.DOUBLE); assertTrue(specialized.isDecomposable()); assertEquals(specialized.name(), "no_aggregation_state_aggregate"); }
public TestResetSessionTask() { CatalogManager catalogManager = new CatalogManager(); transactionManager = createTestTransactionManager(catalogManager); accessControl = new AllowAllAccessControl(); metadata = new MetadataManager( new FeaturesConfig(), new TypeRegistry(), new BlockEncodingManager(new TypeRegistry()), new SessionPropertyManager(), new SchemaPropertyManager(), new TablePropertyManager(), new ColumnPropertyManager(), transactionManager); metadata.getSessionPropertyManager().addSystemSessionProperty(stringProperty( "foo", "test property", null, false)); Catalog bogusTestingCatalog = createBogusTestingCatalog(CATALOG_NAME); metadata.getSessionPropertyManager().addConnectorSessionProperties(bogusTestingCatalog.getConnectorId(), ImmutableList.of(stringProperty( "baz", "test property", null, false))); catalogManager.registerCatalog(bogusTestingCatalog); }
@Test public void testSpill() throws Exception PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new BlockEncodingManager(new TypeRegistry()), false); PagesSerde serde = serdeFactory.createPagesSerde(); SpillerStats spillerStats = new SpillerStats();
@Test public void testDateColumn() { IonSqlQueryBuilder queryBuilder = new IonSqlQueryBuilder(new TypeRegistry()); List<HiveColumnHandle> columns = ImmutableList.of( new HiveColumnHandle("t1", HIVE_TIMESTAMP, parseTypeSignature(TIMESTAMP), 0, REGULAR, Optional.empty()), new HiveColumnHandle("t2", HIVE_DATE, parseTypeSignature(StandardTypes.DATE), 1, REGULAR, Optional.empty())); TupleDomain<HiveColumnHandle> tupleDomain = withColumnDomains(ImmutableMap.of( columns.get(1), Domain.create(SortedRangeSet.copyOf(DATE, ImmutableList.of(Range.equal(DATE, (long) DateTimeUtils.parseDate("2001-08-22")))), false))); assertEquals("SELECT s._1, s._2 FROM S3Object s WHERE (case s._2 when '' then null else CAST(s._2 AS TIMESTAMP) end = `2001-08-22`)", queryBuilder.buildSql(columns, tupleDomain)); }
public TestRealHistogramAggregation() { TypeRegistry typeRegistry = new TypeRegistry(); FunctionRegistry functionRegistry = new FunctionRegistry(typeRegistry, new BlockEncodingManager(typeRegistry), new FeaturesConfig()); InternalAggregationFunction function = functionRegistry.getAggregateFunctionImplementation( new Signature("numeric_histogram", AGGREGATE, parseTypeSignature("map(real, real)"), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.REAL), parseTypeSignature(StandardTypes.DOUBLE))); factory = function.bind(ImmutableList.of(0, 1, 2), Optional.empty()); input = makeInput(10); }
@Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "\\QFunction already registered: custom_add(bigint,bigint):bigint\\E") public void testDuplicateFunctions() { List<SqlFunction> functions = new FunctionListBuilder() .scalars(CustomFunctions.class) .getFunctions() .stream() .filter(input -> input.getSignature().getName().equals("custom_add")) .collect(toImmutableList()); TypeRegistry typeManager = new TypeRegistry(); FunctionRegistry registry = new FunctionRegistry(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); registry.addFunctions(functions); registry.addFunctions(functions); }
@Test public void testDistributesSpillOverPaths() throws Exception { List<Type> types = ImmutableList.of(BIGINT); BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(new TypeRegistry()); List<Path> spillPaths = ImmutableList.of(spillPath1.toPath(), spillPath2.toPath()); FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory( executor, // executor won't be closed, because we don't call destroy() on the spiller factory blockEncodingSerde, new SpillerStats(), spillPaths, 1.0); assertEquals(listFiles(spillPath1.toPath()).size(), 0); assertEquals(listFiles(spillPath2.toPath()).size(), 0); Page page = buildPage(); List<SingleStreamSpiller> spillers = new ArrayList<>(); for (int i = 0; i < 10; ++i) { SingleStreamSpiller singleStreamSpiller = spillerFactory.create(types, bytes -> {}, newSimpleAggregatedMemoryContext().newLocalMemoryContext("test")); getUnchecked(singleStreamSpiller.spill(page)); spillers.add(singleStreamSpiller); } assertEquals(listFiles(spillPath1.toPath()).size(), 5); assertEquals(listFiles(spillPath2.toPath()).size(), 5); spillers.forEach(SingleStreamSpiller::close); assertEquals(listFiles(spillPath1.toPath()).size(), 0); assertEquals(listFiles(spillPath2.toPath()).size(), 0); }
public TestDoubleHistogramAggregation() { TypeRegistry typeRegistry = new TypeRegistry(); FunctionRegistry functionRegistry = new FunctionRegistry(typeRegistry, new BlockEncodingManager(typeRegistry), new FeaturesConfig()); InternalAggregationFunction function = functionRegistry.getAggregateFunctionImplementation( new Signature("numeric_histogram", AGGREGATE, parseTypeSignature("map(double,double)"), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.DOUBLE), parseTypeSignature(StandardTypes.DOUBLE))); factory = function.bind(ImmutableList.of(0, 1, 2), Optional.empty()); input = makeInput(10); }