/** * Deserializes the plan. * * @param in The stream to read from. * @param planClass class of plan * @return The plan, such as QueryPlan, MapredWork, etc. */ public static <T> T deserializePlan(InputStream in, Class<T> planClass) { return deserializePlan(in, planClass, false); }
public static <T> T deserializePlan(Kryo kryo, InputStream in, Class<T> planClass) { return deserializePlan(kryo, in, planClass, false); }
/** * Deserializes the plan. * * @param in The stream to read from. * @param planClass class of plan * @return The plan, such as QueryPlan, MapredWork, etc. */ public static <T> T deserializePlan(InputStream in, Class<T> planClass) { return deserializePlan(in, planClass, false); }
public static <T> T deserializePlan(Kryo kryo, InputStream in, Class<T> planClass) { return deserializePlan(kryo, in, planClass, false); }
private static <T> T deserializePlan(InputStream in, Class<T> planClass, boolean cloningPlan) { Kryo kryo = borrowKryo(); T result = null; try { result = deserializePlan(kryo, in, planClass, cloningPlan); } finally { releaseKryo(kryo); } return result; }
private static <T> T deserializePlan(InputStream in, Class<T> planClass, boolean cloningPlan) { Kryo kryo = borrowKryo(); T result = null; try { result = deserializePlan(kryo, in, planClass, cloningPlan); } finally { releaseKryo(kryo); } return result; }
/** * Clones using the powers of XML. Do not use unless necessary. * @param roots The roots. * @return The clone. */ public static List<Operator<?>> cloneOperatorTree(List<Operator<?>> roots) { if (roots.isEmpty()) { return new ArrayList<>(); } ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); CompilationOpContext ctx = roots.get(0).getCompilationOpContext(); serializePlan(roots, baos, true); @SuppressWarnings("unchecked") List<Operator<?>> result = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), roots.getClass(), true); // Restore the context. LinkedList<Operator<?>> newOps = new LinkedList<>(result); while (!newOps.isEmpty()) { Operator<?> newOp = newOps.poll(); newOp.setCompilationOpContext(ctx); List<Operator<?>> children = newOp.getChildOperators(); if (children != null) { newOps.addAll(children); } } return result; }
/** * Clones using the powers of XML. Do not use unless necessary. * @param roots The roots. * @return The clone. */ public static List<Operator<?>> cloneOperatorTree(List<Operator<?>> roots) { ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); CompilationOpContext ctx = roots.isEmpty() ? null : roots.get(0).getCompilationOpContext(); serializePlan(roots, baos, true); @SuppressWarnings("unchecked") List<Operator<?>> result = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), roots.getClass(), true); // Restore the context. LinkedList<Operator<?>> newOps = new LinkedList<>(result); while (!newOps.isEmpty()) { Operator<?> newOp = newOps.poll(); newOp.setCompilationOpContext(ctx); List<Operator<?>> children = newOp.getChildOperators(); if (children != null) { newOps.addAll(children); } } return result; }
public static List<Operator<?>> cloneOperatorTree(List<Operator<?>> roots, int indexForTezUnion) { ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); CompilationOpContext ctx = roots.isEmpty() ? null : roots.get(0).getCompilationOpContext(); serializePlan(roots, baos, true); @SuppressWarnings("unchecked") List<Operator<?>> result = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), roots.getClass(), true); // Restore the context. LinkedList<Operator<?>> newOps = new LinkedList<>(result); while (!newOps.isEmpty()) { Operator<?> newOp = newOps.poll(); newOp.setIndexForTezUnion(indexForTezUnion); newOp.setCompilationOpContext(ctx); List<Operator<?>> children = newOp.getChildOperators(); if (children != null) { newOps.addAll(children); } } return result; }
gWork = SerializationUtilities.deserializePlan(kryo, in, MapWork.class); } else if(MergeFileMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS))) { gWork = SerializationUtilities.deserializePlan(kryo, in, MergeFileWork.class); } else if(ColumnTruncateMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS))) { gWork = SerializationUtilities.deserializePlan(kryo, in, ColumnTruncateWork.class); } else { throw new RuntimeException("unable to determine work from configuration ." gWork = SerializationUtilities.deserializePlan(kryo, in, ReduceWork.class); } else { throw new RuntimeException("unable to determine work from configuration ." gWork = SerializationUtilities.deserializePlan(kryo, in, MapWork.class); } else if (name.startsWith(REDUCENAME)) { gWork = SerializationUtilities.deserializePlan(kryo, in, ReduceWork.class); } else { throw new RuntimeException("Unknown work type: " + name);
gWork = SerializationUtilities.deserializePlan(kryo, in, MapWork.class); } else if(MergeFileMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS))) { gWork = SerializationUtilities.deserializePlan(kryo, in, MergeFileWork.class); } else if(ColumnTruncateMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS))) { gWork = SerializationUtilities.deserializePlan(kryo, in, ColumnTruncateWork.class); } else if(PartialScanMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS))) { gWork = SerializationUtilities.deserializePlan(kryo, in, PartialScanWork.class); } else { throw new RuntimeException("unable to determine work from configuration ." gWork = SerializationUtilities.deserializePlan(kryo, in, ReduceWork.class); } else { throw new RuntimeException("unable to determine work from configuration ." gWork = SerializationUtilities.deserializePlan(kryo, in, MapWork.class); } else if (name.startsWith(REDUCENAME)) { gWork = SerializationUtilities.deserializePlan(kryo, in, ReduceWork.class); } else { throw new RuntimeException("Unknown work type: " + name);
/** * Clones using the powers of XML. Do not use unless necessary. * @param plan The plan. * @return The clone. */ public static BaseWork cloneBaseWork(BaseWork plan) { PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); Operator<?> op = plan.getAnyRootOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); BaseWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), plan.getClass(), true); // Restore the context. for (Operator<?> newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; }
/** * Clones using the powers of XML. Do not use unless necessary. * @param plan The plan. * @return The clone. */ public static BaseWork cloneBaseWork(BaseWork plan) { PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); Operator<?> op = plan.getAnyRootOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); BaseWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), plan.getClass(), true); // Restore the context. for (Operator<?> newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; }
/** * Clones using the powers of XML. Do not use unless necessary. * @param plan The plan. * @return The clone. */ public static MapredWork clonePlan(MapredWork plan) { // TODO: need proper clone. Meanwhile, let's at least keep this horror in one place PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); Operator<?> op = plan.getAnyOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); MapredWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), MapredWork.class, true); // Restore the context. for (Operator<?> newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; }
/** * Clones using the powers of XML. Do not use unless necessary. * @param plan The plan. * @return The clone. */ public static MapredWork clonePlan(MapredWork plan) { // TODO: need proper clone. Meanwhile, let's at least keep this horror in one place PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); Operator<?> op = plan.getAnyOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); MapredWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), MapredWork.class, true); // Restore the context. for (Operator<?> newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; }
if (localtask) { memoryMXBean = ManagementFactory.getMemoryMXBean(); MapredLocalWork plan = SerializationUtilities.deserializePlan(pathData, MapredLocalWork.class); MapredLocalTask ed = new MapredLocalTask(plan, conf, isSilent); ret = ed.executeInProcess(new DriverContext()); MapredWork plan = SerializationUtilities.deserializePlan(pathData, MapredWork.class); ExecDriver ed = new ExecDriver(plan, conf, isSilent); ret = ed.execute(new DriverContext());
if (localtask) { memoryMXBean = ManagementFactory.getMemoryMXBean(); MapredLocalWork plan = SerializationUtilities.deserializePlan(pathData, MapredLocalWork.class); MapredLocalTask ed = new MapredLocalTask(plan, conf, isSilent); ret = ed.executeInProcess(new DriverContext()); MapredWork plan = SerializationUtilities.deserializePlan(pathData, MapredWork.class); ExecDriver ed = new ExecDriver(plan, conf, isSilent); ret = ed.execute(new DriverContext());