@Override protected List<IteratorSetting> contextIterators(TaskAttemptContext context, String tableName) { return getIterators(context); }
@Override protected void setupAccumuloInput(final Job job) throws AccumuloSecurityException { // set up accumulo input if (!hdfsInput) { job.setInputFormatClass(AccumuloInputFormat.class); } else { job.setInputFormatClass(AccumuloHDFSFileInputFormat.class); } AbstractInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd)); InputFormatBase.setInputTableName(job, RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix)); AbstractInputFormat.setScanAuthorizations(job, authorizations); if (!mock) { AbstractInputFormat.setZooKeeperInstance(job, new ClientConfiguration().withInstance(instance).withZkHosts(zk)); } else { AbstractInputFormat.setMockInstance(job, instance); } if (ttl != null) { final IteratorSetting setting = new IteratorSetting(1, "fi", AgeOffFilter.class); AgeOffFilter.setTTL(setting, Long.valueOf(ttl)); InputFormatBase.addIterator(job, setting); } for (final IteratorSetting iteratorSetting : AccumuloRyaUtils.COMMON_REG_EX_FILTER_SETTINGS) { InputFormatBase.addIterator(job, iteratorSetting); } }
accumuloOptions.getUser(), new PasswordToken(accumuloOptions.getPassword())); InputFormatBase.setInputTableName(job, ingestOptions.getQualifiedTableName()); AbstractInputFormat.setZooKeeperInstance( job, InputFormatBase.addIterator(job, is); job.setInputFormatClass(AccumuloInputFormat.class); final Range r = new Range(); InputFormatBase.setRanges(job, Arrays.asList(r));
public void setTableName(String tableName) { InputFormatBase.setInputTableName(job, tableName); }
public void addIterator(IteratorSetting is) { InputFormatBase.addIterator(job, is); }
public void setRanges(Collection<Range> ranges) { InputFormatBase.setRanges(job, ranges); }
public static AuthenticationToken getPassword(JobContext conf) { return InputFormatBase.getAuthenticationToken(conf); } public static String getUsername(JobContext conf) {
public void fetchColumns(Collection<Pair<Text, Text>> columnFamilyColumnQualifierPairs) { InputFormatBase.fetchColumns(job, columnFamilyColumnQualifierPairs); }
/** * Create the desired number of splits, dividing the number of rows between the mappers. */ public List<InputSplit> getSplits(JobContext job) { long totalRows = InputFormatBase.getConfiguration(job).getLong(NUMROWS, 0); int numSplits = InputFormatBase.getConfiguration(job).getInt(NUMSPLITS, 1); long rowsPerSplit = totalRows / numSplits; System.out.println("Generating " + totalRows + " using " + numSplits + " maps with step of " + rowsPerSplit); ArrayList<InputSplit> splits = new ArrayList<InputSplit>(numSplits); long currentRow = 0; for (int split = 0; split < numSplits - 1; ++split) { splits.add(new RangeInputSplit(currentRow, rowsPerSplit)); currentRow += rowsPerSplit; } splits.add(new RangeInputSplit(currentRow, totalRows - currentRow)); System.out.println("Done Generating."); return splits; }
InputFormatBase.setInputTableName(job, RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix)); AbstractInputFormat.setScanAuthorizations(job, authorizations); if (!mock) { final IteratorSetting setting = new IteratorSetting(1, "fi", AgeOffFilter.class); AgeOffFilter.setTTL(setting, Long.valueOf(ttl)); InputFormatBase.addIterator(job, setting); InputFormatBase.addIterator(job, setting); InputFormatBase.addIterator(job, iteratorSetting);
InputFormatBase.setInputTableName(job, table);
public static AuthenticationToken getPassword(JobContext conf) { return InputFormatBase.getAuthenticationToken(conf); } public static String getUsername(JobContext conf) {
InputFormatBase.setInputTableName(job, table);
@Override protected List<IteratorSetting> contextIterators(TaskAttemptContext context, String tableName) { return getIterators(context); } }
RyaInputFormat.setScanAuthorizations(job, authorizations); String tableName = RdfCloudTripleStoreUtils.layoutPrefixToTable(TABLE_LAYOUT.SPO, tablePrefix); InputFormatBase.setInputTableName(job, tableName); return sc.newAPIHadoopRDD(job.getConfiguration(), GraphXEdgeInputFormat.class, Object.class, Edge.class);
@Override protected List<IteratorSetting> contextIterators(TaskAttemptContext context, String tableName) { return getIterators(context); }