Refine search
@Override protected void setup(Context context) throws IOException, InterruptedException { conf = context.getConfiguration(); separator = conf.get("ReportSeparator",":"); }
@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration configuration = context.getConfiguration(); String tableName = configuration.get("index.tablename"); String[] fields = configuration.getStrings("index.fields"); String familyName = configuration.get("index.familyname"); family = Bytes.toBytes(familyName); indexes = new TreeMap<>(Bytes.BYTES_COMPARATOR); for(String field : fields) { // if the table is "people" and the field to index is "email", then the // index table will be called "people-email" indexes.put(Bytes.toBytes(field), new ImmutableBytesWritable(Bytes.toBytes(tableName + "-" + field))); } } }
/** * Handles common parameter initialization that a subclass might want to leverage. * @param context */ protected void doSetup(Context context) { Configuration conf = context.getConfiguration(); // If a custom separator has been used, // decode it back from Base64 encoding. separator = conf.get(ImportTsv.SEPARATOR_CONF_KEY); if (separator == null) { separator = ImportTsv.DEFAULT_SEPARATOR; } else { separator = new String(Base64.getDecoder().decode(separator)); } skipBadLines = context.getConfiguration().getBoolean(ImportTsv.SKIP_LINES_CONF_KEY, true); logBadLines = context.getConfiguration().getBoolean(ImportTsv.LOG_BAD_LINES_CONF_KEY, false); badLineCount = context.getCounter("ImportTsv", "Bad Lines"); }
throws IOException { if (replicatedScanner == null) { Configuration conf = context.getConfiguration(); sleepMsBeforeReCompare = conf.getInt(NAME +".sleepMsBeforeReCompare", 0); delimiter = conf.get(NAME + ".delimiter", ""); verbose = conf.getBoolean(NAME +".verbose", false); batch = conf.getInt(NAME + ".batch", -1); final Scan scan = new Scan(); if (batch > 0) { scan.setCaching(conf.getInt(TableInputFormat.SCAN_CACHEDROWS, 1)); long startTime = conf.getLong(NAME + ".startTime", 0); long endTime = conf.getLong(NAME + ".endTime", Long.MAX_VALUE); String families = conf.get(NAME + ".families", null); if(families != null) { String[] fams = families.split(","); String rowPrefixes = conf.get(NAME + ".rowPrefixes", null); setRowPrefixFilter(scan, rowPrefixes); scan.setTimeRange(startTime, endTime);
@Override public void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); filesGroup = conf.get(CONF_FILES_GROUP); filesUser = conf.get(CONF_FILES_USER); filesMode = (short)conf.getInt(CONF_FILES_MODE, 0); outputRoot = new Path(conf.get(CONF_OUTPUT_ROOT)); inputRoot = new Path(conf.get(CONF_INPUT_ROOT)); bufferSize = conf.getInt(CONF_BUFFER_SIZE, defaultBlockSize); LOG.info("Using bufferSize=" + StringUtils.humanReadableInt(bufferSize)); context.getCounter(c).increment(0); if (context.getConfiguration().getBoolean(Testing.CONF_TEST_FAILURE, false)) { testing.failuresCountToInject = conf.getInt(Testing.CONF_TEST_FAILURE_COUNT, 0);
/** * Handles common parameter initialization that a subclass might want to leverage. * @param context */ protected void doSetup(Context context) { Configuration conf = context.getConfiguration(); // If a custom separator has been used, // decode it back from Base64 encoding. separator = conf.get(ImportTsv.SEPARATOR_CONF_KEY); if (separator == null) { separator = ImportTsv.DEFAULT_SEPARATOR; } else { separator = new String(Base64.getDecoder().decode(separator)); } // Should never get 0 as we are setting this to a valid value in job // configuration. ts = conf.getLong(ImportTsv.TIMESTAMP_CONF_KEY, 0); skipEmptyColumns = context.getConfiguration().getBoolean( ImportTsv.SKIP_EMPTY_COLUMNS, false); skipBadLines = context.getConfiguration().getBoolean( ImportTsv.SKIP_LINES_CONF_KEY, true); badLineCount = context.getCounter("ImportTsv", "Bad Lines"); logBadLines = context.getConfiguration().getBoolean(ImportTsv.LOG_BAD_LINES_CONF_KEY, false); hfileOutPath = conf.get(ImportTsv.BULK_OUTPUT_CONF_KEY); }
@Override public void setup(org.apache.hadoop.mapreduce.Mapper.Context context) { index = context.getConfiguration().getInt(INDEX_KEY, -1); labels = context.getConfiguration().get(LABELS_KEY); split = labels.split(COMMA); super.setup(context); }
@Override public void setup(Context context) throws IOException { conf = context.getConfiguration(); recordsToWrite = conf.getLong(NUM_TO_WRITE_KEY, NUM_TO_WRITE_DEFAULT); String tableName = conf.get(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); numBackReferencesPerRow = conf.getInt(NUM_BACKREFS_KEY, NUM_BACKREFS_DEFAULT); this.connection = ConnectionFactory.createConnection(conf); mutator = connection.getBufferedMutator( new BufferedMutatorParams(TableName.valueOf(tableName)) .writeBufferSize(4 * 1024 * 1024)); String taskId = conf.get("mapreduce.task.attempt.id"); Matcher matcher = Pattern.compile(".+_m_(\\d+_\\d+)").matcher(taskId); if (!matcher.matches()) { throw new RuntimeException("Strange task ID: " + taskId); } shortTaskId = matcher.group(1); rowsWritten = context.getCounter(Counters.ROWS_WRITTEN); refsWritten = context.getCounter(Counters.REFERENCES_WRITTEN); }
int bandwidthMB = context.getConfiguration().getInt(CONF_BANDWIDTH_MB, 100); if (Integer.MAX_VALUE != bandwidthMB) { in = new ThrottledInputStream(new BufferedInputStream(in), bandwidthMB * 1024 * 1024L);
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); keyLength = conf.getInt(KEYLEN_CONF, KEYLEN_DEFAULT); valLength = conf.getInt(VALLEN_CONF, VALLEN_DEFAULT); multiTableMapper = conf.getBoolean(HFileOutputFormat2.MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false); if (multiTableMapper) { tables = TABLE_NAMES; } else { tables = new TableName[]{TABLE_NAMES[0]}; } }
@Override public void setup(Context context) throws IOException { String atTable = context.getConfiguration().get(ACCESS_TRACKER_TABLE_CONF); String atCF = context.getConfiguration().get(ACCESS_TRACKER_CF_CONF); String atName = context.getConfiguration().get(ACCESS_TRACKER_NAME_CONF); HTable table = new HTable(context.getConfiguration(), atTable); long timestamp = context.getConfiguration().getLong(TIMESTAMP_CONF, -1); if(timestamp < 0) { throw new IllegalStateException("Must specify a timestamp that is positive."); } try { tracker = AccessTrackerUtil.INSTANCE.loadAll(AccessTrackerUtil.INSTANCE.loadAll(table, atCF, atName, timestamp)); } catch (Throwable e) { throw new IllegalStateException("Unable to load the accesstrackers from the directory", e); } }
@Override protected void setup(Context context) throws IOException, InterruptedException { id = Bytes.toBytes("Job: "+context.getJobID() + " Task: " + context.getTaskAttemptID()); this.connection = ConnectionFactory.createConnection(context.getConfiguration()); instantiateHTable(); this.width = context.getConfiguration().getInt(GENERATOR_WIDTH_KEY, WIDTH_DEFAULT); current = new byte[this.width][]; int wrapMultiplier = context.getConfiguration().getInt(GENERATOR_WRAP_KEY, WRAP_DEFAULT); this.wrap = (long)wrapMultiplier * width; this.numNodes = context.getConfiguration().getLong( GENERATOR_NUM_ROWS_PER_MAP_KEY, (long)WIDTH_DEFAULT * WRAP_DEFAULT); if (this.numNodes < this.wrap) { this.wrap = this.numNodes; this.multipleUnevenColumnFamilies = isMultiUnevenColumnFamilies(context.getConfiguration()); this.numWalkers = context.getConfiguration().getInt(CONCURRENT_WALKER_KEY, CONCURRENT_WALKER_DEFAULT); this.walkersStop = false; this.conf = context.getConfiguration(); int n = context.getConfiguration().getInt(BIG_FAMILY_VALUE_SIZE_KEY, 256); int limit = context.getConfiguration().getInt( ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, ConnectionConfiguration.MAX_KEYVALUE_SIZE_DEFAULT);
@Override protected void cleanup( Context context ) throws IOException { final String tmpDirLoc = context.getConfiguration().get(TMP_FILE_LOC_KEY); final File tmpDir = Paths.get(tmpDirLoc).toFile(); FileUtils.deleteDirectory(tmpDir); context.progress(); context.setStatus("Clean"); } }
@Override protected void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); sourceHashDir = new Path(conf.get(SOURCE_HASH_DIR_CONF_KEY)); sourceConnection = openConnection(conf, SOURCE_ZK_CLUSTER_CONF_KEY, null); targetConnection = openConnection(conf, TARGET_ZK_CLUSTER_CONF_KEY, TableOutputFormat.OUTPUT_CONF_PREFIX); sourceTable = openTable(sourceConnection, conf, SOURCE_TABLE_CONF_KEY); targetTable = openTable(targetConnection, conf, TARGET_TABLE_CONF_KEY); dryRun = conf.getBoolean(DRY_RUN_CONF_KEY, false); doDeletes = conf.getBoolean(DO_DELETES_CONF_KEY, true); doPuts = conf.getBoolean(DO_PUTS_CONF_KEY, true); sourceTableHash = HashTable.TableHash.read(conf, sourceHashDir); LOG.info("Read source hash manifest: " + sourceTableHash); LOG.info("Read " + sourceTableHash.partitions.size() + " partition keys"); TableSplit split = (TableSplit) context.getInputSplit(); ImmutableBytesWritable splitStartKey = new ImmutableBytesWritable(split.getStartRow()); sourceHashReader = sourceTableHash.newReader(conf, splitStartKey); findNextKeyHashPair(); // create a hasher, but don't start it right away // instead, find the first hash batch at or after the start row // and skip any rows that come before. they will be caught by the previous task targetHasher = new HashTable.ResultHasher(); }
@Override public void setup(org.apache.hadoop.mapreduce.Mapper.Context context) { index = context.getConfiguration().getInt(INDEX_KEY, -1); labels = context.getConfiguration().get(LABELS_KEY); split = labels.split(COMMA); super.setup(context); }
@Override protected void setup(Context context) throws IOException, InterruptedException { this.numberOfPairs = context.getConfiguration().getInt("number.of.pairs", DEFAULT_NUMBER_OF_PAIRS); THE_LOGGER.info("setup() numberOfPairs = " + numberOfPairs); }
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); keyLength = conf.getInt(KEYLEN_CONF, KEYLEN_DEFAULT); valLength = conf.getInt(VALLEN_CONF, VALLEN_DEFAULT); multiTableMapper = conf.getBoolean(HFileOutputFormat2.MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false); if (multiTableMapper) { tables = TABLE_NAMES; } else { tables = new TableName[]{TABLE_NAMES[0]}; } }
@Override protected InputConverterUnit<ByteArray> getInputConverterUnit(Context context) { String updateShard = context.getConfiguration().get(BatchConstants.CFG_UPDATE_SHARD); if (updateShard == null || updateShard.equalsIgnoreCase("false")) { return new InputConverterUnitForBaseCuboid(false); } else { return new InputConverterUnitForBaseCuboid(true); } }
@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); // pass client configuration into driver Properties clientInfos = new Properties(); for (Map.Entry<String, String> entry : conf) { clientInfos.setProperty(entry.getKey(), entry.getValue()); } try { conn = (PhoenixConnection) QueryUtil.getConnectionOnServer(clientInfos, conf); // We are dependent on rolling back before performing commits, so we need to be sure // that auto-commit is not turned on conn.setAutoCommit(false); final String tableNamesConf = conf.get(TABLE_NAMES_CONFKEY); final String logicalNamesConf = conf.get(LOGICAL_NAMES_CONFKEY); tableNames = TargetTableRefFunctions.NAMES_FROM_JSON.apply(tableNamesConf); logicalNames = TargetTableRefFunctions.NAMES_FROM_JSON.apply(logicalNamesConf); initColumnIndexes(); } catch (SQLException | ClassNotFoundException e) { throw new RuntimeException(e); } upsertListener = new MapperUpsertListener<RECORD>( context, conf.getBoolean(IGNORE_INVALID_ROW_CONFKEY, true)); upsertExecutor = buildUpsertExecutor(conf); preUpdateProcessor = PhoenixConfigurationUtil.loadPreUpsertProcessor(conf); }
@Override protected void setup(Context context) throws IOException,InterruptedException { this.N = context.getConfiguration().getInt("word.count.ignored.length", DEFAULT_IGNORED_LENGTH); }