private Task<? extends Serializable> resolveDriverAlias(ConditionalResolverCommonJoinCtx ctx, HiveConf conf) { try { resolveUnknownSizes(ctx, conf); return resolveMapJoinTask(ctx, conf); } catch (Exception e) { LOG.info("Failed to resolve driver alias by exception.. Falling back to common join", e); } return null; }
protected Task<? extends Serializable> resolveMapJoinTask( ConditionalResolverCommonJoinCtx ctx, HiveConf conf) throws Exception { Set<String> participants = getParticipants(ctx);
@Override public List<Task<? extends Serializable>> getTasks(HiveConf conf, Object objCtx) { ConditionalResolverCommonJoinCtx ctx = ((ConditionalResolverCommonJoinCtx) objCtx).clone(); List<Task<? extends Serializable>> resTsks = new ArrayList<Task<? extends Serializable>>(); // get aliasToPath and pass it to the heuristic Task<? extends Serializable> task = resolveDriverAlias(ctx, conf); if (task == null) { // run common join task resTsks.add(ctx.getCommonJoinTask()); } else { // run the map join task, set task tag if (task.getBackupTask() != null) { task.getBackupTask().setTaskTag(Task.BACKUP_COMMON_JOIN); } resTsks.add(task); } return resTsks; }
@Test public void testResolvingDriverAlias() throws Exception { ConditionalResolverCommonJoin resolver = new ConditionalResolverCommonJoin(); Task resolved = resolver.resolveMapJoinTask(ctx, conf); Assert.assertEquals("alias3", resolved.getId()); resolved = resolver.resolveMapJoinTask(ctx, conf); Assert.assertEquals("alias3", resolved.getId()); resolved = resolver.resolveMapJoinTask(ctx, conf); Assert.assertNull(resolved);
cndTsk.setResolver(new ConditionalResolverCommonJoin()); ConditionalResolverCommonJoinCtx resolverCtx = new ConditionalResolverCommonJoinCtx(); resolverCtx.setPathToAliases(pathToAliases);
@Override public List<Task<? extends Serializable>> getTasks(HiveConf conf, Object objCtx) { ConditionalResolverCommonJoinCtx ctx = (ConditionalResolverCommonJoinCtx) objCtx; List<Task<? extends Serializable>> resTsks = new ArrayList<Task<? extends Serializable>>(); // get aliasToPath and pass it to the heuristic HashMap<String, String> aliasToPath = ctx.getAliasToPath(); String bigTableAlias = this.resolveMapJoinTask(aliasToPath, conf); if (bigTableAlias == null) { // run common join task resTsks.add(ctx.getCommonJoinTask()); } else { // run the map join task Task<? extends Serializable> task = ctx.getAliasToTask().get(bigTableAlias); //set task tag if(task.getTaskTag() == Task.CONVERTED_LOCAL_MAPJOIN) { task.getBackupTask().setTaskTag(Task.BACKUP_COMMON_JOIN); } resTsks.add(task); } return resTsks; }
cndTsk.setResolver(new ConditionalResolverCommonJoin()); ConditionalResolverCommonJoinCtx resolverCtx = new ConditionalResolverCommonJoinCtx(); resolverCtx.setPathToAliases(pathToAliases);
private Task<? extends Serializable> resolveDriverAlias(ConditionalResolverCommonJoinCtx ctx, HiveConf conf) { try { resolveUnknownSizes(ctx, conf); return resolveMapJoinTask(ctx, conf); } catch (Exception e) { LOG.info("Failed to resolve driver alias by exception.. Falling back to common join", e); } return null; }
cndTsk.setResolver(new ConditionalResolverCommonJoin()); ConditionalResolverCommonJoinCtx resolverCtx = new ConditionalResolverCommonJoinCtx(); resolverCtx.setPathToAliases(pathToAliases);
protected void resolveUnknownSizes(ConditionalResolverCommonJoinCtx ctx, HiveConf conf) throws Exception { Set<String> aliases = getParticipants(ctx);
@Override public List<Task<? extends Serializable>> getTasks(HiveConf conf, Object objCtx) { ConditionalResolverCommonJoinCtx ctx = ((ConditionalResolverCommonJoinCtx) objCtx).clone(); List<Task<? extends Serializable>> resTsks = new ArrayList<Task<? extends Serializable>>(); // get aliasToPath and pass it to the heuristic Task<? extends Serializable> task = resolveDriverAlias(ctx, conf); if (task == null) { // run common join task resTsks.add(ctx.getCommonJoinTask()); } else { // run the map join task, set task tag if (task.getBackupTask() != null) { task.getBackupTask().setTaskTag(Task.BACKUP_COMMON_JOIN); } resTsks.add(task); } return resTsks; }
private Task<? extends Serializable> resolveDriverAlias(ConditionalResolverCommonJoinCtx ctx, HiveConf conf) { try { resolveUnknownSizes(ctx, conf); return resolveMapJoinTask(ctx, conf); } catch (Exception e) { LOG.info("Failed to resolve driver alias by exception.. Falling back to common join", e); } return null; }
cndTsk.setResolver(new ConditionalResolverCommonJoin()); ConditionalResolverCommonJoinCtx resolverCtx = new ConditionalResolverCommonJoinCtx(); resolverCtx.setPathToAliases(pathToAliases);
protected Task<? extends Serializable> resolveMapJoinTask( ConditionalResolverCommonJoinCtx ctx, HiveConf conf) throws Exception { Set<String> participants = getParticipants(ctx);
@Override public List<Task<? extends Serializable>> getTasks(HiveConf conf, Object objCtx) { ConditionalResolverCommonJoinCtx ctx = ((ConditionalResolverCommonJoinCtx) objCtx).clone(); List<Task<? extends Serializable>> resTsks = new ArrayList<Task<? extends Serializable>>(); // get aliasToPath and pass it to the heuristic Task<? extends Serializable> task = resolveDriverAlias(ctx, conf); if (task == null) { // run common join task resTsks.add(ctx.getCommonJoinTask()); } else { // run the map join task, set task tag if (task.getBackupTask() != null) { task.getBackupTask().setTaskTag(Task.BACKUP_COMMON_JOIN); } resTsks.add(task); } return resTsks; }
cndTsk.setResolver(new ConditionalResolverCommonJoin()); ConditionalResolverCommonJoinCtx resolverCtx = new ConditionalResolverCommonJoinCtx(); resolverCtx.setAliasToPath(aliasToPath);
protected void resolveUnknownSizes(ConditionalResolverCommonJoinCtx ctx, HiveConf conf) throws Exception { Set<String> aliases = getParticipants(ctx);
cndTsk.setResolver(new ConditionalResolverCommonJoin()); ConditionalResolverCommonJoinCtx resolverCtx = new ConditionalResolverCommonJoinCtx(); resolverCtx.setPathToAliases(pathToAliases);
protected void resolveUnknownSizes(ConditionalResolverCommonJoinCtx ctx, HiveConf conf) throws Exception { Set<String> aliases = getParticipants(ctx);
cndTsk.setResolver(new ConditionalResolverCommonJoin()); ConditionalResolverCommonJoinCtx resolverCtx = new ConditionalResolverCommonJoinCtx(); resolverCtx.setPathToAliases(pathToAliases);