@JsonProperty("input_records_count") public long getInputRecordCount() { long sizeRecordCount = 0L; for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { sizeRecordCount += cubeSegment.getInputRecords(); } return sizeRecordCount; }
@JsonProperty("input_records_size") public long getInputRecordSize() { long sizeRecordSize = 0L; for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { sizeRecordSize += cubeSegment.getInputRecordsSize(); } return sizeRecordSize; }
@JsonProperty("size_kb") public long getSizeKB() { long sizeKb = 0L; for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { sizeKb += cubeSegment.getSizeKB(); } return sizeKb; }
public IIInstance updateII(IIInstance ii) throws IOException { logger.info("Updating II instance '" + ii.getName()); // save resource saveResource(ii); logger.info("II with " + ii.getSegments().size() + " segments is saved"); return ii; }
public void removeIICache(IIInstance ii) { iiMap.remove(ii.getName()); for (IISegment segment : ii.getSegments()) { usedStorageLocation.remove(segment.getName()); } }
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<String>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) { for (IISegment seg : ii.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } return result; } }
private synchronized IIInstance loadIIInstance(String path) throws IOException { ResourceStore store = getStore(); logger.debug("Loading IIInstance " + store.getReadableResourcePath(path)); IIInstance IIInstance = null; try { IIInstance = store.getResource(path, IIInstance.class, II_SERIALIZER); IIInstance.setConfig(config); if (StringUtils.isBlank(IIInstance.getName())) throw new IllegalStateException("IIInstance name must not be blank"); iiMap.putLocal(IIInstance.getName(), IIInstance); for (IISegment segment : IIInstance.getSegments()) { usedStorageLocation.add(segment.getName()); } return IIInstance; } catch (Exception e) { logger.error("Error during load ii instance " + path, e); return null; } }
private void clearSegment(String iiName) throws Exception { IIInstance ii = iiManager.getII(iiName); ii.getSegments().clear(); iiManager.updateII(ii); }
protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException { File tmp = File.createTempFile("kylin_job_meta", ""); tmp.delete(); // we need a directory, so delete the file first File metaDir = new File(tmp, "meta"); metaDir.mkdirs(); metaDir.getParentFile().deleteOnExit(); // write kylin.properties KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); File kylinPropsFile = new File(metaDir, "kylin.properties"); kylinConfig.writeProperties(kylinPropsFile); // write II / model_desc / II_desc / dict / table ArrayList<String> dumpList = new ArrayList<String>(); dumpList.add(ii.getResourcePath()); dumpList.add(ii.getDescriptor().getModel().getResourcePath()); dumpList.add(ii.getDescriptor().getResourcePath()); for (String tableName : ii.getDescriptor().getModel().getAllTables()) { TableDesc table = MetadataManager.getInstance(kylinConfig).getTableDesc(tableName); dumpList.add(table.getResourcePath()); } for (IISegment segment : ii.getSegments()) { dumpList.addAll(segment.getDictionaryPaths()); } dumpResources(kylinConfig, metaDir, dumpList); // hadoop distributed cache conf.set("tmpfiles", "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath())); }
for (IISegment seg : ii.getSegments()) { String tablename = seg.getStorageLocationIdentifier(); allTablesNeedToBeDropped.remove(tablename);
private String buildSegment(String iiName, long startDate, long endDate) throws Exception { IIInstance iiInstance = iiManager.getII(iiName); IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate); iiInstance.getSegments().add(segment); iiManager.updateII(iiInstance); IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig); IIJob job = iiJobBuilder.buildJob(segment); jobService.addJob(job); waitForJob(job.getId()); return job.getId(); }
@JsonProperty("input_records_size") public long getInputRecordSize() { long sizeRecordSize = 0L; for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { sizeRecordSize += cubeSegment.getInputRecordsSize(); } return sizeRecordSize; }
@JsonProperty("size_kb") public long getSizeKB() { long sizeKb = 0L; for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { sizeKb += cubeSegment.getSizeKB(); } return sizeKb; }
@JsonProperty("input_records_count") public long getInputRecordCount() { long sizeRecordCount = 0L; for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { sizeRecordCount += cubeSegment.getInputRecords(); } return sizeRecordCount; }
@Override public long getDateRangeStart() { List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY); long startTime = Long.MAX_VALUE; for (IISegment seg : readySegs) { if (seg.getDateRangeStart() < startTime) startTime = seg.getDateRangeStart(); } return startTime; }
@Override public long getDateRangeEnd() { List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY); long endTime = 0; for (IISegment seg : readySegs) { if (seg.getDateRangeEnd() > endTime) endTime = seg.getDateRangeEnd(); } return endTime; }
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<String>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) { for (IISegment seg : ii.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } return result; } }
private synchronized IIInstance reloadIILocalAt(String path) throws IOException { ResourceStore store = getStore(); logger.debug("Loading IIInstance " + store.getReadableResourcePath(path)); IIInstance ii = null; try { ii = store.getResource(path, IIInstance.class, II_SERIALIZER); ii.setConfig(config); if (StringUtils.isBlank(ii.getName())) throw new IllegalStateException("IIInstance name must not be blank"); iiMap.putLocal(ii.getName(), ii); for (IISegment segment : ii.getSegments()) { usedStorageLocation.put(ii.getName().toUpperCase(), segment.getStorageLocationIdentifier()); } return ii; } catch (Exception e) { logger.error("Error during load ii instance " + path, e); return null; } }
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<String>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (!StringUtils.isBlank(tableName)) { result.add(tableName); System.out.println("added new table: " + tableName); } } } for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) { if (ii.getStatus() == RealizationStatusEnum.READY) { for (IISegment seg : ii.getSegments()) {//streaming segment is never "READY" String tableName = seg.getStorageLocationIdentifier(); if (!StringUtils.isBlank(tableName)) { result.add(tableName); System.out.println("added new table: " + tableName); } } } } return result; }
protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException { File tmp = File.createTempFile("kylin_job_meta", ""); FileUtils.forceDelete(tmp); File metaDir = new File(tmp, "meta"); metaDir.mkdirs(); // write kylin.properties KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); File kylinPropsFile = new File(metaDir, "kylin.properties"); kylinConfig.writeProperties(kylinPropsFile); // write II / model_desc / II_desc / dict / table ArrayList<String> dumpList = new ArrayList<String>(); dumpList.add(ii.getResourcePath()); dumpList.add(ii.getDescriptor().getModel().getResourcePath()); dumpList.add(ii.getDescriptor().getResourcePath()); for (String tableName : ii.getDescriptor().getModel().getAllTables()) { TableDesc table = MetadataManager.getInstance(kylinConfig).getTableDesc(tableName); dumpList.add(table.getResourcePath()); } for (IISegment segment : ii.getSegments()) { dumpList.addAll(segment.getDictionaryPaths()); } dumpResources(kylinConfig, metaDir, dumpList); addToHadoopDistCache(conf, metaDir); }