unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION, "Directory explorers " + dirExplorers + " functions can not be used " + "when " + PlannerSettings.CONSTANT_FOLDING.getOptionName() + " option is set to false"); throw new UnsupportedOperationException();
@Test @Ignore("interval") public void checkReadWriteExtended() throws Exception { final String originalFile = "${WORKING_PATH}/src/test/resources/vector/complex/extended.json".replaceAll( Pattern.quote("${WORKING_PATH}"), Matcher.quoteReplacement(TestTools.getWorkingPath())); final String newTable = "TestExtendedTypes/newjson"; try { testNoResult(String.format("ALTER SESSION SET \"%s\" = 'json'", ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName())); testNoResult(String.format("ALTER SESSION SET \"%s\" = true", ExecConstants.JSON_EXTENDED_TYPES.getOptionName())); // create table test("create table dfs_test.tmp.\"%s\" as select * from dfs.\"%s\"", newTable, originalFile); // check query of table. test("select * from dfs_test.tmp.\"%s\"", newTable); // check that original file and new file match. final byte[] originalData = Files.readAllBytes(Paths.get(originalFile)); final byte[] newData = Files.readAllBytes(Paths.get(BaseTestQuery.getDfsTestTmpSchemaLocation() + '/' + newTable + "/0_0_0.json")); assertEquals(new String(originalData), new String(newData)); } finally { testNoResult(String.format("ALTER SESSION SET \"%s\" = '%s'", ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName(), ExecConstants.OUTPUT_FORMAT_VALIDATOR.getDefault().getValue())); testNoResult(String.format("ALTER SESSION SET \"%s\" = %s", ExecConstants.JSON_EXTENDED_TYPES.getOptionName(), ExecConstants.JSON_EXTENDED_TYPES.getDefault().getValue())); } }
OptionValue.createBoolean(OptionValue.OptionType.SYSTEM, ExecConstants.USE_NEW_MEMORY_BOUNDED_BEHAVIOR.getOptionName(), false) ); final QueryContext queryContext = new QueryContext(session(), context, UserBitShared.QueryId.getDefaultInstance());
} else { logger.info("Old metadata data may be bad, but preserving descendants of source [{}] because '{}' is enabled", config.getName(), CatalogOptions.STORAGE_PLUGIN_KEEP_METADATA_ON_REPLACE.getOptionName());
session.getOptions().setOption(OptionValue.createBoolean(OptionType.SESSION, ExecConstants.USE_LEGACY_CATALOG_NAME.getOptionName(), true));
OptionValue.OptionType.SYSTEM, ExecConstants.SPLIT_ENABLED.getOptionName(), true)); ExpressionEvaluationOptions options = new ExpressionEvaluationOptions(optionManager); options.setCodeGenOption(SupportedEngines.CodeGenOption.Gandiva.toString());
try { if (edgeProjectEnabled) { test("ALTER SYSTEM SET " + ExecConstants.ELASTIC_RULES_EDGE_PROJECT.getOptionName() + " = true"); test("ALTER SYSTEM SET " + ExecConstants.ELASTIC_RULES_EDGE_PROJECT.getOptionName() + " = false");
SystemOptionManager.STORE_NAME, SystemOptionManager.OptionStoreCreator.class, new JacksonSerializer<>(context.getLpPersistence().getMapper(), OptionValue.class)); options.put(ReflectionOptions.ENABLE_SUBHOUR_POLICIES.getOptionName(), OptionValue.createBoolean( OptionValue.OptionType.SYSTEM, ReflectionOptions.ENABLE_SUBHOUR_POLICIES.getOptionName(), true)); options.close(); } catch (Exception e) {
testContext.getOptions().setOption(OptionValue.createBoolean(OptionType.SYSTEM, ExecConstants.FAST_OR_ENABLE.getOptionName(), false)); testFunction(expr, ObjectArrays.concat(objects, result)); } catch(Throwable t) { throw t; } finally { testContext.getOptions().deleteOption(ExecConstants.FAST_OR_ENABLE.getOptionName(), OptionType.SYSTEM);
@Test public void changeTablesInAliasAndNoticeReattempt() throws Exception { test("ALTER SYSTEM SET \"" + ExecConstants.ENABLE_REATTEMPTS.getOptionName() + "\" = true"); try { final String movieType = TestNameGenerator.tableName(); .go(); } finally { test("ALTER SYSTEM RESET \"" + ExecConstants.ENABLE_REATTEMPTS.getOptionName() + "\"");
@Test public void keepLearningSchemaAcrossFiles() throws Exception { test("ALTER SYSTEM SET \"" + ExecConstants.ENABLE_REATTEMPTS.getOptionName() + "\" = true"); try { final String query = String.format("select * from dfs_root.\"%s/schemachange/differentschemas/\"", TEST_RES_PATH); try { testBuilder() .sqlQuery(query) .unOrdered() .jsonBaselineFile("results/differentschemas.json") .build() .run(); } catch (Exception e) { // first attempt may fail as a batch may have been sent to the user, so if it does, this must be the message .. assertTrue(e.getMessage() .contains("New schema found and recorded. Please reattempt the query.")); } // .. but the second attempt must not fail, full schema must have been learnt at this point testBuilder() .sqlQuery(query) .unOrdered() .jsonBaselineFile("results/differentschemas.json") .build() .run(); } finally { test("ALTER SYSTEM RESET \"" + ExecConstants.ENABLE_REATTEMPTS.getOptionName() + "\""); } } }
@Test public void testMongoExtendedTypes() throws Exception { final String originalFile = "${WORKING_PATH}/src/test/resources/vector/complex/mongo_extended.json".replaceAll( Pattern.quote("${WORKING_PATH}"), Matcher.quoteReplacement(TestTools.getWorkingPath())); try { testNoResult(String.format("ALTER SESSION SET \"%s\" = 'json'", ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName())); testNoResult(String.format("ALTER SESSION SET \"%s\" = true", ExecConstants.JSON_EXTENDED_TYPES.getOptionName())); int actualRecordCount = testSql(String.format("select * from dfs.\"%s\"", originalFile)); assertEquals( String.format( "Received unexpected number of rows in output: expected=%d, received=%s", 1, actualRecordCount), 1, actualRecordCount); List<QueryDataBatch> resultList = testSqlWithResults(String.format("select * from dfs.\"%s\"", originalFile)); String actual = getResultString(resultList, ","); String expected = "dremio_timestamp_millies,bin,bin1\n2015-07-07T03:59:43.488,dremio,dremio\n"; Assert.assertEquals(expected, actual); } finally { testNoResult(String.format("ALTER SESSION SET \"%s\" = '%s'", ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName(), ExecConstants.OUTPUT_FORMAT_VALIDATOR.getDefault().getValue())); testNoResult(String.format("ALTER SESSION SET \"%s\" = %s", ExecConstants.JSON_EXTENDED_TYPES.getOptionName(), ExecConstants.JSON_EXTENDED_TYPES.getDefault().getValue())); } } }
@Test public void simpleSelectWithLimitWithSoftScanWithLeafLimitsEnabled() { when(optionManager.getOption(eq(PlannerSettings.ENABLE_LEAF_LIMITS.getOptionName()))) .thenReturn(OptionValue.createBoolean(OptionType.QUERY, PlannerSettings.ENABLE_LEAF_LIMITS.getOptionName(), true)); try { Prel input = newScreen( newLimit(0, 10, newProject(exprs(), rowType(), newUnionExchange( newLimit(0, 10, newProject(exprs(), rowType(), newSoftScan(rowType()) ) ) ) ) ) ); Prel output = SimpleLimitExchangeRemover.apply(plannerSettings, input); verifyOutput(output, "Screen", "Limit", "Project", "UnionExchange", "Limit", "Project", "SystemScan"); } finally { when(optionManager.getOption(eq(PlannerSettings.ENABLE_LEAF_LIMITS.getOptionName()))) .thenReturn(PlannerSettings.ENABLE_LEAF_LIMITS.getDefault()); } }
@Override public void applyOptions(OptionManager manager) { if (enableLeafLimits) { manager.setOption(createBoolean(QUERY, PlannerSettings.ENABLE_LEAF_LIMITS.getOptionName(), true)); manager.setOption(createLong(QUERY, ExecConstants.MAX_WIDTH_GLOBAL_KEY, manager.getOption(PlannerSettings.LEAF_LIMIT_MAX_WIDTH))); manager.setOption(createLong(QUERY, ExecConstants.SLICE_TARGET, 1)); } if (enableOutputLimits) { manager.setOption(createBoolean(QUERY, PlannerSettings.ENABLE_OUTPUT_LIMITS.getOptionName(), true)); } // always store results manager.setOption(createString(QUERY, PlannerSettings.STORE_QUERY_RESULTS.getOptionName(), StoreQueryResultsPolicy.PATH_AND_ATTEMPT_ID.name())); manager.setOption(createString(QUERY, PlannerSettings.QUERY_RESULTS_STORE_TABLE.getOptionName(), queryResultsStorePath)); if (!allowPartitionPruning) { manager.setOption(createBoolean(QUERY, PlannerSettings.ENABLE_PARTITION_PRUNING.getOptionName(), false)); } if (internalSingleThreaded) { manager.setOption(createBoolean(QUERY, ExecConstants.SORT_FILE_BLOCKS.getOptionName(), true)); manager.setOption(createBoolean(QUERY, PlannerSettings.EXCHANGE.getOptionName(), true)); } }
@Test public void testVJoinDecimal() throws Exception { try { test(String.format("alter system set \"%s\" = true", PlannerSettings .ENABLE_DECIMAL_DATA_TYPE_KEY)); final String query = "select count(*) from " + "(select EXPR$0 as d1 from cp.\"join/decimal_join/decimals-500.parquet\")t1" + " join " + "(select EXPR$0 as d2 from cp.\"join/decimal_join/decimals-a.parquet\")t2" + " on t1.d1 = t2.d2"; testBuilder().sqlQuery(query) .unOrdered() .baselineColumns("EXPR$0") .baselineValues(500l) .go(); } finally { Boolean decimalDefault = PlannerSettings.ENABLE_DECIMAL_DATA_TYPE.getDefault().getBoolVal(); test(String.format("alter system set \"%s\" = %s", PlannerSettings .ENABLE_DECIMAL_DATA_TYPE_KEY, decimalDefault.toString())); } } }
@Test @Ignore // TODO file JIRA to fix this public void testFix2967() throws Exception { setSessionOption(PlannerSettings.BROADCAST.getOptionName(), "false"); setSessionOption(PlannerSettings.HASHJOIN.getOptionName(), "false"); setSessionOption(ExecConstants.SLICE_TARGET, "1"); setSessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, "23"); final String TEST_RES_PATH = TestTools.getWorkingPath() + "/src/test/resources"; try { test("select * from dfs.\"%s/join/j1\" j1 left outer join dfs.\"%s/join/j2\" j2 on (j1.c_varchar = j2.c_varchar)", TEST_RES_PATH, TEST_RES_PATH); } finally { setSessionOption(PlannerSettings.BROADCAST.getOptionName(), String.valueOf(PlannerSettings.BROADCAST.getDefault ().getBoolVal())); setSessionOption(PlannerSettings.HASHJOIN.getOptionName(), String.valueOf(PlannerSettings.HASHJOIN.getDefault() .getBoolVal())); setSessionOption(ExecConstants.SLICE_TARGET, String.valueOf(ExecConstants.SLICE_TARGET_DEFAULT)); setSessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, String.valueOf(ExecConstants.MAX_WIDTH_PER_NODE .getDefault().getNumVal())); } }
public void compareParquetReadersHyperVector(String selection, String table) throws Exception { String query = "select " + selection + " from " + table; try { testBuilder() .ordered() .highPerformanceComparison() .sqlQuery(query) .optionSettingQueriesForTestQuery( "alter system set \"store.parquet.use_new_reader\" = false") .sqlBaselineQuery(query) .optionSettingQueriesForBaseline( "alter system set \"store.parquet.use_new_reader\" = true") .build().run(); } finally { test("alter system set \"%s\" = %b", ExecConstants.PARQUET_NEW_RECORD_READER, ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR .getDefault().getBoolVal()); } }
public void compareParquetReadersColumnar(String selection, String table) throws Exception { String query = "select " + selection + " from " + table; try { testBuilder() .ordered() .sqlQuery(query) .optionSettingQueriesForTestQuery( "alter system set \"store.parquet.use_new_reader\" = false") .sqlBaselineQuery(query) .optionSettingQueriesForBaseline( "alter system set \"store.parquet.use_new_reader\" = true") .build().run(); } finally { test("alter system set \"%s\" = %b", ExecConstants.PARQUET_NEW_RECORD_READER, ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR .getDefault().getBoolVal()); } }
public void compareParquetReadersColumnar(String selection, String table) throws Exception { String query = "select " + selection + " from " + table; try(AutoCloseable o = withSystemOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP_VALIDATOR, false)) { testBuilder() .ordered() .sqlQuery(query) .optionSettingQueriesForTestQuery( "alter system set \"store.parquet.use_new_reader\" = false") .sqlBaselineQuery(query) .optionSettingQueriesForBaseline( "alter system set \"store.parquet.use_new_reader\" = true") .build().run(); } finally { test("alter system set \"%s\" = %b", ExecConstants.PARQUET_NEW_RECORD_READER, ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR .getDefault().getBoolVal()); } }
@PUT @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @Path("/acceleration/settings") public void saveSystemSettings(final SystemSettingsApiDescriptor descriptor) { Preconditions.checkArgument(descriptor.getLimit() != null, "limit is required"); Preconditions.checkArgument(descriptor.getLimit() > 0, "limit must be positive"); Preconditions.checkArgument(descriptor.getAccelerateAggregation() != null, "accelerateAggregation is required"); Preconditions.checkArgument(descriptor.getAccelerateRaw() != null, "accelerateRaw is required"); SystemOptionManager optionManager = context.getOptionManager(); optionManager.setOption(OptionValue.createLong(OptionValue.OptionType.SYSTEM, ReflectionOptions.MAX_AUTOMATIC_REFLECTIONS.getOptionName(), descriptor.getLimit())); optionManager.setOption(OptionValue.createBoolean(OptionValue.OptionType.SYSTEM, ReflectionOptions.ENABLE_AUTOMATIC_AGG_REFLECTIONS.getOptionName(), descriptor.getAccelerateAggregation())); optionManager.setOption(OptionValue.createBoolean(OptionValue.OptionType.SYSTEM, ReflectionOptions.ENABLE_AUTOMATIC_RAW_REFLECTIONS.getOptionName(), descriptor.getAccelerateRaw())); if (descriptor.getLayoutRefreshMaxAttempts() != null) { optionManager.setOption(OptionValue.createLong(OptionValue.OptionType.SYSTEM, ExecConstants.LAYOUT_REFRESH_MAX_ATTEMPTS.getOptionName(), descriptor.getLayoutRefreshMaxAttempts())); } } }