@Override public String getFactTable() { return getDescriptor().getFactTableName(); }
public InvertedIndexStorageEngine(IIInstance ii) { this.seg = ii.getFirstSegment(); }
protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException { File tmp = File.createTempFile("kylin_job_meta", ""); tmp.delete(); // we need a directory, so delete the file first File metaDir = new File(tmp, "meta"); metaDir.mkdirs(); metaDir.getParentFile().deleteOnExit(); // write kylin.properties KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); File kylinPropsFile = new File(metaDir, "kylin.properties"); kylinConfig.writeProperties(kylinPropsFile); // write II / model_desc / II_desc / dict / table ArrayList<String> dumpList = new ArrayList<String>(); dumpList.add(ii.getResourcePath()); dumpList.add(ii.getDescriptor().getModel().getResourcePath()); dumpList.add(ii.getDescriptor().getResourcePath()); for (String tableName : ii.getDescriptor().getModel().getAllTables()) { TableDesc table = MetadataManager.getInstance(kylinConfig).getTableDesc(tableName); dumpList.add(table.getResourcePath()); } for (IISegment segment : ii.getSegments()) { dumpList.addAll(segment.getDictionaryPaths()); } dumpResources(kylinConfig, metaDir, dumpList); // hadoop distributed cache conf.set("tmpfiles", "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath())); }
public static IIInstance create(String iiName, String projectName, IIDesc iiDesc) { IIInstance iii = new IIInstance(); iii.setConfig(iiDesc.getConfig()); iii.setName(iiName); iii.setDescName(iiDesc.getName()); iii.setCreateTimeUTC(System.currentTimeMillis()); iii.setStatus(RealizationStatusEnum.DISABLED); iii.updateRandomUuid(); return iii; }
public IIInstance updateII(IIInstance ii) throws IOException { logger.info("Updating II instance '" + ii.getName()); // save resource saveResource(ii); logger.info("II with " + ii.getSegments().size() + " segments is saved"); return ii; }
private void attachMetadata(IIInstance ii) throws IOException { Configuration conf = job.getConfiguration(); attachKylinPropsAndMetadata(ii, conf); IISegment seg = ii.getFirstSegment(); conf.set(BatchConstants.CFG_II_NAME, ii.getName()); conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName()); }
@Test public void testGetIIsByDesc() throws IOException { IIManager mgr = IIManager.getInstance(getTestConfig()); List<IIInstance> iiInstances = mgr.getIIsByDesc("test_kylin_ii_desc"); Assert.assertTrue(iiInstances.size() > 0); IIInstance instance = iiInstances.get(0); Dictionary dict = mgr.getDictionary(instance.getFirstSegment(), instance.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID")); Assert.assertNotNull(dict); }
for (IISegment seg : ii.getSegments()) { String tablename = seg.getStorageLocationIdentifier(); allTablesNeedToBeDropped.remove(tablename); log.info("Remove table " + tablename + " from drop list, as the table belongs to ii " + ii.getName() + " with status " + ii.getStatus());
@JsonProperty("input_records_count") public long getInputRecordCount() { long sizeRecordCount = 0L; for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { sizeRecordCount += cubeSegment.getInputRecords(); } return sizeRecordCount; }
private synchronized IIInstance loadIIInstance(String path) throws IOException { ResourceStore store = getStore(); logger.debug("Loading IIInstance " + store.getReadableResourcePath(path)); IIInstance IIInstance = null; try { IIInstance = store.getResource(path, IIInstance.class, II_SERIALIZER); IIInstance.setConfig(config); if (StringUtils.isBlank(IIInstance.getName())) throw new IllegalStateException("IIInstance name must not be blank"); iiMap.putLocal(IIInstance.getName(), IIInstance); for (IISegment segment : IIInstance.getSegments()) { usedStorageLocation.add(segment.getName()); } return IIInstance; } catch (Exception e) { logger.error("Error during load ii instance " + path, e); return null; } }
private static boolean isMatchedWithJoins(Collection<JoinDesc> joins, IIInstance iiInstance) { IIDesc iiDesc = iiInstance.getDescriptor(); List<TableDesc> tables = iiDesc.listTables(); logger.info("Fact Table" + factTable + " not matched in join: " + j + " on ii " + iiInstance.getName()); return false; logger.info("Query joins don't match on ii " + iiInstance.getName()); return false;
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<String>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (!StringUtils.isBlank(tableName)) { result.add(tableName); System.out.println("added new table: " + tableName); } } } for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) { if (ii.getStatus() == RealizationStatusEnum.READY) { for (IISegment seg : ii.getSegments()) {//streaming segment is never "READY" String tableName = seg.getStorageLocationIdentifier(); if (!StringUtils.isBlank(tableName)) { result.add(tableName); System.out.println("added new table: " + tableName); } } } } return result; }
private void removeII(IIInstance ii) throws IOException { getStore().deleteResource(ii.getResourcePath()); iiMap.remove(ii.getName()); }
public boolean isReady() { return getStatus() == RealizationStatusEnum.READY; }
public List<IIInstance> getIIsByDesc(String descName) { List<IIInstance> list = listAllIIs(); List<IIInstance> result = new ArrayList<IIInstance>(); Iterator<IIInstance> it = list.iterator(); while (it.hasNext()) { IIInstance ci = it.next(); if (descName.equalsIgnoreCase(ci.getDescName())) { result.add(ci); } } return result; }
public IIInstance createII(IIInstance ii) throws IOException { if (this.getII(ii.getName()) != null) throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists."); this.updateII(ii); // FIXME need to pass in project name String projectName = ProjectInstance.DEFAULT_PROJECT_NAME; ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner()); return ii; }
if (!digest.factTable.equalsIgnoreCase(ii.getFactTable())) { logger.info("Exclude II " + ii.getName() + " because fact table unmatch"); return result; logger.info("Exclude II " + ii.getName() + " because unmatched joins"); return result; logger.info("Exclude ii " + ii.getName() + " because unmatched dimensions"); return result; logger.info("Exclude ii " + ii.getName() + " because unmatched aggregations"); return result;
private void saveResource(IIInstance ii) throws IOException { ResourceStore store = getStore(); store.putResource(ii.getResourcePath(), ii, II_SERIALIZER); this.afterIIUpdated(ii); }