/** * Serializes the plan. * * @param plan The plan, such as QueryPlan, MapredWork, etc. * @param out The stream to write to. */ public static void serializePlan(Object plan, OutputStream out) { serializePlan(plan, out, false); }
public static void serializePlan(Kryo kryo, Object plan, OutputStream out) { serializePlan(kryo, plan, out, false); }
/** * Serializes the plan. * * @param plan The plan, such as QueryPlan, MapredWork, etc. * @param out The stream to write to. */ public static void serializePlan(Object plan, OutputStream out) { serializePlan(plan, out, false); }
public static void serializePlan(Kryo kryo, Object plan, OutputStream out) { serializePlan(kryo, plan, out, false); }
private static void serializePlan(Object plan, OutputStream out, boolean cloningPlan) { Kryo kryo = borrowKryo(); try { serializePlan(kryo, plan, out, cloningPlan); } finally { releaseKryo(kryo); } }
private static void serializePlan(Object plan, OutputStream out, boolean cloningPlan) { Kryo kryo = borrowKryo(); try { serializePlan(kryo, plan, out, cloningPlan); } finally { releaseKryo(kryo); } }
/** * Clones using the powers of XML. Do not use unless necessary. * @param roots The roots. * @return The clone. */ public static List<Operator<?>> cloneOperatorTree(List<Operator<?>> roots) { if (roots.isEmpty()) { return new ArrayList<>(); } ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); CompilationOpContext ctx = roots.get(0).getCompilationOpContext(); serializePlan(roots, baos, true); @SuppressWarnings("unchecked") List<Operator<?>> result = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), roots.getClass(), true); // Restore the context. LinkedList<Operator<?>> newOps = new LinkedList<>(result); while (!newOps.isEmpty()) { Operator<?> newOp = newOps.poll(); newOp.setCompilationOpContext(ctx); List<Operator<?>> children = newOp.getChildOperators(); if (children != null) { newOps.addAll(children); } } return result; }
/** * Clones using the powers of XML. Do not use unless necessary. * @param roots The roots. * @return The clone. */ public static List<Operator<?>> cloneOperatorTree(List<Operator<?>> roots) { ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); CompilationOpContext ctx = roots.isEmpty() ? null : roots.get(0).getCompilationOpContext(); serializePlan(roots, baos, true); @SuppressWarnings("unchecked") List<Operator<?>> result = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), roots.getClass(), true); // Restore the context. LinkedList<Operator<?>> newOps = new LinkedList<>(result); while (!newOps.isEmpty()) { Operator<?> newOp = newOps.poll(); newOp.setCompilationOpContext(ctx); List<Operator<?>> children = newOp.getChildOperators(); if (children != null) { newOps.addAll(children); } } return result; }
public static List<Operator<?>> cloneOperatorTree(List<Operator<?>> roots, int indexForTezUnion) { ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); CompilationOpContext ctx = roots.isEmpty() ? null : roots.get(0).getCompilationOpContext(); serializePlan(roots, baos, true); @SuppressWarnings("unchecked") List<Operator<?>> result = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), roots.getClass(), true); // Restore the context. LinkedList<Operator<?>> newOps = new LinkedList<>(result); while (!newOps.isEmpty()) { Operator<?> newOp = newOps.poll(); newOp.setIndexForTezUnion(indexForTezUnion); newOp.setCompilationOpContext(ctx); List<Operator<?>> children = newOp.getChildOperators(); if (children != null) { newOps.addAll(children); } } return result; }
/** * Clones using the powers of XML. Do not use unless necessary. * @param plan The plan. * @return The clone. */ public static BaseWork cloneBaseWork(BaseWork plan) { PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); Operator<?> op = plan.getAnyRootOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); BaseWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), plan.getClass(), true); // Restore the context. for (Operator<?> newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; }
try { out = new DeflaterOutputStream(byteOut, new Deflater(Deflater.BEST_SPEED)); SerializationUtilities.serializePlan(kryo, w, out); out.close(); out = null; try { out = fs.create(planPath); SerializationUtilities.serializePlan(kryo, w, out); out.close(); out = null;
/** * Clones using the powers of XML. Do not use unless necessary. * @param plan The plan. * @return The clone. */ public static BaseWork cloneBaseWork(BaseWork plan) { PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); Operator<?> op = plan.getAnyRootOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); BaseWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), plan.getClass(), true); // Restore the context. for (Operator<?> newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; }
/** * Clones using the powers of XML. Do not use unless necessary. * @param plan The plan. * @return The clone. */ public static MapredWork clonePlan(MapredWork plan) { // TODO: need proper clone. Meanwhile, let's at least keep this horror in one place PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); Operator<?> op = plan.getAnyOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); MapredWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), MapredWork.class, true); // Restore the context. for (Operator<?> newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; }
try { out = new DeflaterOutputStream(byteOut, new Deflater(Deflater.BEST_SPEED)); SerializationUtilities.serializePlan(kryo, w, out); out.close(); out = null; try { out = fs.create(planPath); SerializationUtilities.serializePlan(kryo, w, out); out.close(); out = null;
/** * Clones using the powers of XML. Do not use unless necessary. * @param plan The plan. * @return The clone. */ public static MapredWork clonePlan(MapredWork plan) { // TODO: need proper clone. Meanwhile, let's at least keep this horror in one place PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); Operator<?> op = plan.getAnyOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); MapredWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), MapredWork.class, true); // Restore the context. for (Operator<?> newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; }
try { out = FileSystem.getLocal(conf).create(planPath); SerializationUtilities.serializePlan(plan, out); out.close(); out = null;
try { out = FileSystem.getLocal(conf).create(planPath); SerializationUtilities.serializePlan(plan, out); out.close(); out = null;
localFs.delete(mapXml, true); FSDataOutputStream planStream = localFs.create(mapXml); SerializationUtilities.serializePlan(mapWork, planStream); conf.setBoolean(Utilities.HAS_MAP_WORK, true); planStream.close();
try { out = FileSystem.getLocal(conf).create(planPath); SerializationUtilities.serializePlan(plan, out); out.close(); out = null;
SerializationUtilities.serializePlan(mrwork, baos); baos.close(); String v1 = baos.toString(); SerializationUtilities.serializePlan(mrwork2, baos); baos.close();