public InvertedIndexStorageEngine(IIInstance ii) { this.seg = ii.getFirstSegment(); }
private void attachMetadata(IIInstance ii) throws IOException { Configuration conf = job.getConfiguration(); attachKylinPropsAndMetadata(ii, conf); IISegment seg = ii.getFirstSegment(); conf.set(BatchConstants.CFG_II_NAME, ii.getName()); conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName()); }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_II_NAME); options.addOption(OPTION_INPUT_PATH); parseOptions(options, args); String iiname = getOptionValue(OPTION_II_NAME); String factColumnsInputPath = getOptionValue(OPTION_INPUT_PATH); KylinConfig config = KylinConfig.getInstanceFromEnv(); IIManager mgr = IIManager.getInstance(config); IIInstance ii = mgr.getII(iiname); mgr.buildInvertedIndexDictionary(ii.getFirstSegment(), factColumnsInputPath); return 0; } catch (Exception e) { printUsage(options); throw e; } }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_II_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME); String input = getOptionValue(OPTION_INPUT_PATH); String iiname = getOptionValue(OPTION_II_NAME); FileSystem fs = FileSystem.get(getConf()); FsPermission permission = new FsPermission((short) 0777); fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission); int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName }); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); IIInstance ii = mgr.getII(iiname); IISegment seg = ii.getFirstSegment(); seg.setStorageLocationIdentifier(tableName); seg.setStatus(SegmentStatusEnum.READY); mgr.updateII(ii); return hbaseExitCode; } catch (Exception e) { printUsage(options); throw e; } }
public static void main(String[] args) throws IOException { Configuration hconf = HadoopUtil.getCurrentConfiguration(); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); String iiName = args[0]; IIInstance ii = mgr.getII(iiName); String path = args[1]; System.out.println("Reading from " + path + " ..."); TableRecordInfo info = new TableRecordInfo(ii.getFirstSegment()); IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest()); int count = 0; for (Slice slice : codec.decodeKeyValue(readSequenceKVs(hconf, path))) { for (RawTableRecord rec : slice) { System.out.printf(new TableRecord(rec, info).toString()); count++; } } System.out.println("Total " + count + " records"); }
@Test public void testGetIIsByDesc() throws IOException { IIManager mgr = IIManager.getInstance(getTestConfig()); List<IIInstance> iiInstances = mgr.getIIsByDesc("test_kylin_ii_desc"); Assert.assertTrue(iiInstances.size() > 0); IIInstance instance = iiInstances.get(0); Dictionary dict = mgr.getDictionary(instance.getFirstSegment(), instance.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID")); Assert.assertNotNull(dict); }
@Before public void setUp() throws Exception { this.createTestMetadata(); this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii"); this.info = new TableRecordInfo(ii.getFirstSegment()); }
@Before public void setup() throws IOException { this.createTestMetadata(); this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii"); this.tableRecordInfo = new TableRecordInfo(ii.getFirstSegment()); }
@Before public void setup() throws Exception { this.createTestMetadata(); this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii"); this.seg = ii.getFirstSegment(); String hbaseUrl = KylinConfig.getInstanceFromEnv().getStorageUrl(); Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl); hconn = HConnectionManager.createConnection(hconf); this.info = new TableRecordInfo(seg); }
@Before public void setup() throws IOException { this.createTestMetadata(); this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii"); this.tableRecordInfo = new TableRecordInfo(ii.getFirstSegment()); factTableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.TEST_KYLIN_FACT"); TblColRef formatName = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_FORMAT_NAME"); TblColRef siteId = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID"); Collection<TblColRef> dims = new HashSet<>(); dims.add(formatName); projector = CoprocessorProjector.makeForEndpoint(tableRecordInfo, dims); aggregators = EndpointAggregators.fromFunctions(tableRecordInfo, buildAggregations()); CompareTupleFilter rawFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ); rawFilter.addChild(new ColumnTupleFilter(siteId)); rawFilter.addChild(new ConstantTupleFilter("0")); filter = CoprocessorFilter.fromFilter(this.ii.getFirstSegment(), rawFilter); aggCache = new EndpointAggregationCache(aggregators); tableData = mockTable(); }
private void attachMetadata(IIInstance ii) throws IOException { Configuration conf = job.getConfiguration(); attachKylinPropsAndMetadata(ii, conf); IISegment seg = ii.getFirstSegment(); conf.set(BatchConstants.CFG_II_NAME, ii.getName()); conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName()); }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_II_NAME); options.addOption(OPTION_INPUT_PATH); parseOptions(options, args); String iiname = getOptionValue(OPTION_II_NAME); String factColumnsInputPath = getOptionValue(OPTION_INPUT_PATH); KylinConfig config = KylinConfig.getInstanceFromEnv(); IIManager mgr = IIManager.getInstance(config); IIInstance ii = mgr.getII(iiname); mgr.buildInvertedIndexDictionary(ii.getFirstSegment(), factColumnsInputPath); return 0; } catch (Exception e) { printUsage(options); throw e; } }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_II_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME); String input = getOptionValue(OPTION_INPUT_PATH); String iiname = getOptionValue(OPTION_II_NAME); FileSystem fs = FileSystem.get(getConf()); FsPermission permission = new FsPermission((short) 0777); fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission); int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName }); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); IIInstance ii = mgr.getII(iiname); IISegment seg = ii.getFirstSegment(); seg.setStorageLocationIdentifier(tableName); seg.setStatus(SegmentStatusEnum.READY); mgr.updateII(ii); return hbaseExitCode; } catch (Exception e) { printUsage(options); throw e; } }