private IIInstance getII(String iiName) { IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); IIInstance ii = mgr.getII(iiName); if (ii == null) throw new IllegalArgumentException("No Inverted Index found by name " + iiName); return ii; }
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<String>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) { for (IISegment seg : ii.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } return result; } }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_II_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME); String input = getOptionValue(OPTION_INPUT_PATH); String iiname = getOptionValue(OPTION_II_NAME); FileSystem fs = FileSystem.get(getConf()); FsPermission permission = new FsPermission((short) 0777); fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission); int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName }); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); IIInstance ii = mgr.getII(iiname); IISegment seg = ii.getFirstSegment(); seg.setStorageLocationIdentifier(tableName); seg.setStatus(SegmentStatusEnum.READY); mgr.updateII(ii); return hbaseExitCode; } catch (Exception e) { printUsage(options); throw e; } }
private void saveResource(IIInstance ii) throws IOException { ResourceStore store = getStore(); store.putResource(ii.getResourcePath(), ii, II_SERIALIZER); this.afterIIUpdated(ii); }
@Test public void testCreateIIInstance() throws IOException { IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_desc"); IIInstance ii = IIInstance.create("new ii", "default", iiDesc); IIManager iiMgr = IIManager.getInstance(getTestConfig()); List<IIInstance> allIIList = iiMgr.listAllIIs(); iiMgr.createII(ii); Assert.assertNotNull(iiMgr.getII("new ii")); }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_II_NAME); options.addOption(OPTION_INPUT_PATH); parseOptions(options, args); String iiname = getOptionValue(OPTION_II_NAME); String factColumnsInputPath = getOptionValue(OPTION_INPUT_PATH); KylinConfig config = KylinConfig.getInstanceFromEnv(); IIManager mgr = IIManager.getInstance(config); IIInstance ii = mgr.getII(iiname); mgr.buildInvertedIndexDictionary(ii.getFirstSegment(), factColumnsInputPath); return 0; } catch (Exception e) { printUsage(options); throw e; } }
private String buildSegment(String iiName, long startDate, long endDate) throws Exception { IIInstance iiInstance = iiManager.getII(iiName); IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate); iiInstance.getSegments().add(segment); iiManager.updateII(iiInstance); IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig); IIJob job = iiJobBuilder.buildJob(segment); jobService.addJob(job); waitForJob(job.getId()); return job.getId(); }
@Test public void testGetIIsByDesc() throws IOException { IIManager mgr = IIManager.getInstance(getTestConfig()); List<IIInstance> iiInstances = mgr.getIIsByDesc("test_kylin_ii_desc"); Assert.assertTrue(iiInstances.size() > 0); IIInstance instance = iiInstances.get(0); Dictionary dict = mgr.getDictionary(instance.getFirstSegment(), instance.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID")); Assert.assertNotNull(dict); }
@Override public IRealization getRealization(String name) { return getII(name); } }
public IIInstance dropII(String iiName, boolean deleteDesc) throws IOException { logger.info("Dropping II '" + iiName + "'"); IIInstance ii = getII(iiName); if (deleteDesc && ii.getDescriptor() != null) { IIDescManager.getInstance(config).removeIIDesc(ii.getDescriptor()); } removeII(ii); ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.INVERTED_INDEX, iiName); return ii; }
public List<IIInstance> getIIsByDesc(String descName) { List<IIInstance> list = listAllIIs(); List<IIInstance> result = new ArrayList<IIInstance>(); Iterator<IIInstance> it = list.iterator(); while (it.hasNext()) { IIInstance ci = it.next(); if (descName.equalsIgnoreCase(ci.getDescName())) { result.add(ci); } } return result; }
public static IIManager getInstance(KylinConfig config) { IIManager r = CACHE.get(config); if (r != null) { return r; } synchronized (IIManager.class) { r = CACHE.get(config); if (r != null) { return r; } try { r = new IIManager(config); CACHE.put(config, r); if (CACHE.size() > 1) { logger.warn("More than one singleton exist"); } return r; } catch (IOException e) { throw new IllegalStateException("Failed to init IIManager from " + config, e); } } }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_II_NAME); options.addOption(OPTION_INPUT_PATH); parseOptions(options, args); String iiname = getOptionValue(OPTION_II_NAME); String factColumnsInputPath = getOptionValue(OPTION_INPUT_PATH); KylinConfig config = KylinConfig.getInstanceFromEnv(); IIManager mgr = IIManager.getInstance(config); IIInstance ii = mgr.getII(iiname); mgr.buildInvertedIndexDictionary(ii.getFirstSegment(), factColumnsInputPath); return 0; } catch (Exception e) { printUsage(options); throw e; } }
@Override public IRealization getRealization(String name) { return getII(name); } }
public List<IIInstance> getIIsByDesc(String descName) { List<IIInstance> list = listAllIIs(); List<IIInstance> result = new ArrayList<IIInstance>(); Iterator<IIInstance> it = list.iterator(); while (it.hasNext()) { IIInstance ci = it.next(); if (descName.equalsIgnoreCase(ci.getDescName())) { result.add(ci); } } return result; }
public static IIManager getInstance(KylinConfig config) { IIManager r = CACHE.get(config); if (r != null) { return r; } synchronized (IIManager.class) { r = CACHE.get(config); if (r != null) { return r; } try { r = new IIManager(config); CACHE.put(config, r); if (CACHE.size() > 1) { logger.warn("More than one singleton exist"); } return r; } catch (IOException e) { throw new IllegalStateException("Failed to init IIManager from " + config, e); } } }
@Override public void setConf(Configuration conf) { this.conf = conf; try { KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf); IIManager mgr = IIManager.getInstance(config); IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME)); IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW); this.info = new TableRecordInfo(seg); this.rec = this.info.createTableRecord(); } catch (IOException e) { throw new RuntimeException("", e); } }
@Before public void before() throws Exception { HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); DeployUtil.initCliWorkDir(); // DeployUtil.deployMetadata(); DeployUtil.overrideJobJarLocations(); final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); jobService = ExecutableManager.getInstance(kylinConfig); scheduler = DefaultScheduler.getInstance(); scheduler.init(new JobEngineConfig(kylinConfig)); if (!scheduler.hasStarted()) { throw new RuntimeException("scheduler has not been started"); } iiManager = IIManager.getInstance(kylinConfig); jobEngineConfig = new JobEngineConfig(kylinConfig); for (String jobId : jobService.getAllJobIds()) { if(jobService.getJob(jobId) instanceof IIJob){ jobService.deleteJob(jobId); } } IIInstance ii = iiManager.getII(TEST_II_NAME); if (ii.getStatus() != RealizationStatusEnum.DISABLED) { ii.setStatus(RealizationStatusEnum.DISABLED); iiManager.updateII(ii); } }