@Override public int getEngineType() { return getDescriptor().getEngineType(); }
@Override public IMRBatchCubingOutputSide2 getBatchCubingOutputSide(final CubeSegment seg) { boolean useSpark = seg.getCubeDesc().getEngineType() == IEngineAware.ID_SPARK; // TODO need refactor final HBaseJobSteps steps = useSpark ? new HBaseSparkSteps(seg) : new HBaseMRSteps(seg); return new IMRBatchCubingOutputSide2() { @Override public void addStepPhase2_BuildDictionary(DefaultChainedExecutable jobFlow) { jobFlow.addTask(steps.createCreateHTableStep(jobFlow.getId())); } @Override public void addStepPhase3_BuildCube(DefaultChainedExecutable jobFlow) { jobFlow.addTask(steps.createConvertCuboidToHfileStep(jobFlow.getId())); jobFlow.addTask(steps.createBulkLoadStep(jobFlow.getId())); } @Override public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) { steps.addCubingGarbageCollectionSteps(jobFlow); } @Override public IMROutputFormat getOuputFormat() { return new HBaseMROutputFormat(); } }; }
if (cubeDesc.getEngineType()== IEngineAware.ID_SPARK){ exportHBaseConfiguration(cubeSegment.getStorageLocationIdentifier());
newCubeDesc.setVolatileRange(cubeDesc.getVolatileRange()); newCubeDesc.setRetentionRange(cubeDesc.getRetentionRange()); newCubeDesc.setEngineType(cubeDesc.getEngineType()); newCubeDesc.setStorageType(cubeDesc.getStorageType()); newCubeDesc.setAggregationGroups(cubeDesc.getAggregationGroups());
@Override public int getEngineType() { return getDescriptor().getEngineType(); }
@Override public IMRBatchCubingOutputSide2 getBatchCubingOutputSide(final CubeSegment seg) { boolean useSpark = seg.getCubeDesc().getEngineType() == IEngineAware.ID_SPARK; // TODO need refactor final HBaseJobSteps steps = useSpark ? new HBaseSparkSteps(seg) : new HBaseMRSteps(seg); return new IMRBatchCubingOutputSide2() { @Override public void addStepPhase2_BuildDictionary(DefaultChainedExecutable jobFlow) { jobFlow.addTask(steps.createCreateHTableStep(jobFlow.getId())); } @Override public void addStepPhase3_BuildCube(DefaultChainedExecutable jobFlow) { jobFlow.addTask(steps.createConvertCuboidToHfileStep(jobFlow.getId())); jobFlow.addTask(steps.createBulkLoadStep(jobFlow.getId())); } @Override public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) { steps.addCubingGarbageCollectionSteps(jobFlow); } @Override public IMROutputFormat getOuputFormat() { return new HBaseMROutputFormat(); } }; }
if (cubeDesc.getEngineType()== IEngineAware.ID_SPARK){ exportHBaseConfiguration(cubeSegment.getStorageLocationIdentifier());
newCubeDesc.setVolatileRange(cubeDesc.getVolatileRange()); newCubeDesc.setRetentionRange(cubeDesc.getRetentionRange()); newCubeDesc.setEngineType(cubeDesc.getEngineType()); newCubeDesc.setStorageType(cubeDesc.getStorageType()); newCubeDesc.setAggregationGroups(cubeDesc.getAggregationGroups());