public IIInstance createII(IIInstance ii) throws IOException { if (this.getII(ii.getName()) != null) throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists."); // other logic is the same as update. return updateII(ii); }
private void clearSegment(String iiName) throws Exception { IIInstance ii = iiManager.getII(iiName); ii.getSegments().clear(); iiManager.updateII(ii); }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_II_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME); String input = getOptionValue(OPTION_INPUT_PATH); String iiname = getOptionValue(OPTION_II_NAME); FileSystem fs = FileSystem.get(getConf()); FsPermission permission = new FsPermission((short) 0777); fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission); int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName }); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); IIInstance ii = mgr.getII(iiname); IISegment seg = ii.getFirstSegment(); seg.setStorageLocationIdentifier(tableName); seg.setStatus(SegmentStatusEnum.READY); mgr.updateII(ii); return hbaseExitCode; } catch (Exception e) { printUsage(options); throw e; } }
@After public void after() throws Exception { IIInstance ii = iiManager.getII(TEST_II_NAME); if (ii.getStatus() != RealizationStatusEnum.READY) { ii.setStatus(RealizationStatusEnum.READY); iiManager.updateII(ii); } backup(); }
private String buildSegment(String iiName, long startDate, long endDate) throws Exception { IIInstance iiInstance = iiManager.getII(iiName); IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate); iiInstance.getSegments().add(segment); iiManager.updateII(iiInstance); IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig); IIJob job = iiJobBuilder.buildJob(segment); jobService.addJob(job); waitForJob(job.getId()); return job.getId(); }
@Before public void before() throws Exception { HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); DeployUtil.initCliWorkDir(); // DeployUtil.deployMetadata(); DeployUtil.overrideJobJarLocations(); final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); jobService = ExecutableManager.getInstance(kylinConfig); scheduler = DefaultScheduler.getInstance(); scheduler.init(new JobEngineConfig(kylinConfig)); if (!scheduler.hasStarted()) { throw new RuntimeException("scheduler has not been started"); } iiManager = IIManager.getInstance(kylinConfig); jobEngineConfig = new JobEngineConfig(kylinConfig); for (String jobId : jobService.getAllJobIds()) { if(jobService.getJob(jobId) instanceof IIJob){ jobService.deleteJob(jobId); } } IIInstance ii = iiManager.getII(TEST_II_NAME); if (ii.getStatus() != RealizationStatusEnum.DISABLED) { ii.setStatus(RealizationStatusEnum.DISABLED); iiManager.updateII(ii); } }
public IIInstance createII(IIInstance ii) throws IOException { if (this.getII(ii.getName()) != null) throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists."); this.updateII(ii); // FIXME need to pass in project name String projectName = ProjectInstance.DEFAULT_PROJECT_NAME; ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner()); return ii; }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_II_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME); String input = getOptionValue(OPTION_INPUT_PATH); String iiname = getOptionValue(OPTION_II_NAME); FileSystem fs = FileSystem.get(getConf()); FsPermission permission = new FsPermission((short) 0777); fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission); int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName }); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); IIInstance ii = mgr.getII(iiname); IISegment seg = ii.getFirstSegment(); seg.setStorageLocationIdentifier(tableName); seg.setStatus(SegmentStatusEnum.READY); mgr.updateII(ii); return hbaseExitCode; } catch (Exception e) { printUsage(options); throw e; } }