Refine search
FileSystem fs = FileSystem.get(conf); for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) { String cfName = cf.getName(); fs.setPermission(new Path(input + cfName), permission); int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs); log.debug("End to run LoadIncrementalHFiles"); return ret;
Path inputPath = new Path(inputFile); Path outPath = new Path(outFile); Configuration conf = new Configuration(); loadMapRedConfigs(conf); FileInputFormat.addInputPath(job, inputPath); FileSystem fs = FileSystem.get(conf); fs.delete(outPath); FileOutputFormat.setOutputPath(job, outPath); LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf); loader.doBulkLoad(new Path(outFile), hDataTable);
Path stagingDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); hbaseConf.set("hbase.loadincremental.threads.max", "2"); FileSystem fs = jobAttemptPath.getFileSystem(queryContext.getConf()); if (!fs.exists(jobAttemptPath) || fs.listStatus(jobAttemptPath) == null) { LOG.warn("No query attempt file in " + jobAttemptPath); return stagingResultDir; LoadIncrementalHFiles loadIncrementalHFiles = null; try { loadIncrementalHFiles = new LoadIncrementalHFiles(hbaseConf); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new IOException(e.getMessage(), e); loadIncrementalHFiles.doBulkLoad(stagingResultDir, htable);
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); int status = ToolRunner.run(conf, new BulkImportTool(), args); if (status != 0) { System.exit(status); } SchemaMetrics.configureGlobally(conf); status = ToolRunner.run(new LoadIncrementalHFiles(conf), new String[]{conf.get(HFILE_PATH), conf.get(LilyJythonMapper.TABLE_NAME)}); FileSystem.get(conf).delete(new Path(new URI(conf.get(HFILE_PATH))), true); System.exit(status); }
HFileOutputFormat.setOutputPath(job, hfileOutputPath); job.getConfiguration().set("mapred.map.tasks.speculative.execution", "false"); job.getConfiguration().set("mapred.reduce.tasks.speculative.execution", "false"); new LoadIncrementalHFiles(conf).doBulkLoad(hfileOutputPath, destHTable); } catch (Exception e) { throw new IOException("Bulkloader couldn't run", e); FileSystem fs = FileSystem.get(hfileOutputPath.toUri(), conf); FileStatus stat = fs.getFileStatus(hfileOutputPath); FileStatus[] dirListing = fs.listStatus(hfileOutputPath); if (stat.isDir() && dirListing.length <= 3) { List<String> fileNames = new ArrayList<String>(); for (FileStatus dentry : dirListing) { fileNames.add(dentry.getPath().toString());
String workdir = cmd.getOptionValue('w'); String target = cmd.getOptionValue('t'); getConf().setBoolean(SKIP_INVALID_PROPERTY, cmd.hasOption('i')); getConf().setBoolean(VERIFY_DATATYPE_VALUES_PROPERTY, cmd.hasOption('d')); getConf().setBoolean(TRUNCATE_PROPERTY, cmd.hasOption('r')); getConf().setInt(SPLIT_BITS_PROPERTY, Integer.parseInt(cmd.getOptionValue('b', "3"))); if (cmd.hasOption('g')) getConf().set(DEFAULT_CONTEXT_PROPERTY, cmd.getOptionValue('g')); FileInputFormat.setInputDirRecursive(job, true); FileInputFormat.setInputPaths(job, source); FileOutputFormat.setOutputPath(job, new Path(workdir)); TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.initCredentials(job); HalyardTableUtils.truncateTable(hTable).close(); new LoadIncrementalHFiles(getConf()).doBulkLoad(new Path(workdir), hTable); LOG.info("Bulk Load Completed.."); return 0;
FileSystem fs = HBaseTestUtil.getClusterFileSystem(); Path dir = HBaseTestUtil.getMROutputDir(testName); Path familyDir = new Path(dir, Bytes.toString(FAMILY)); Configuration conf = HBaseConfiguration.create(); byte[] from = range[0]; byte[] to = range[1]; HBaseTestUtil.createHFile(conf, fs, new Path(familyDir, "hfile_" + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000); chmod(familyDir.toString()); LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf); loader.doBulkLoad(dir, table); fs.delete(dir, true);
final LoadIncrementalHFiles loader = createHFileLoader(mConf); final String hFileScheme = hfilePath.toUri().getScheme(); Token<DelegationTokenIdentifier> hdfsDelegationToken = null; final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); final DistributedFileSystem fileSystem = (DistributedFileSystem) hfilePath.getFileSystem(mConf); hdfsDelegationToken = fileSystem.getDelegationToken(RENEWER); ugi.addToken(hdfsDelegationToken); hfilePath.getFileSystem(mConf).globStatus(new Path(hfilePath, "*")); for (FileStatus hfile : hfiles) { String partName = hfile.getPath().getName(); loader.doBulkLoad(path, htable); LOG.info("Successfully loaded: " + path.toString());
runIncrementalPELoad(Bytes.toString(TABLE_NAME), testDir.toString()); for (FileStatus f : fs.listStatus(testDir)) { for (byte[] family : FAMILIES) { if (Bytes.toString(family).equals(f.getPath().getName())) { ++dir; chmod(testDir.toString()); new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table); for (KeyValue kv : res.raw()) { assertTrue(KeyValue.COMPARATOR.matchingRows(first, kv)); assertTrue(Bytes.equals(first.getValue(), kv.getValue())); admin.deleteTable(TABLE_NAME); fs.delete(testDir, true);
String queryFiles = cmd.getOptionValue('q'); String workdir = cmd.getOptionValue('w'); getConf().setLong(DEFAULT_TIMESTAMP_PROPERTY, Long.parseLong(cmd.getOptionValue('e', String.valueOf(System.currentTimeMillis())))); if (cmd.hasOption('i')) getConf().set(ELASTIC_INDEX_URL, cmd.getOptionValue('i')); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); getConf().setStrings(TABLE_NAME_PROPERTY, source); getConf().setLong(DEFAULT_TIMESTAMP_PROPERTY, getConf().getLong(DEFAULT_TIMESTAMP_PROPERTY, System.currentTimeMillis())); int stages = 1; HFileOutputFormat2.configureIncrementalLoad(job, hTable.getTableDescriptor(), hTable.getRegionLocator()); QueryInputFormat.setQueriesFromDirRecursive(job.getConfiguration(), queryFiles, true, stage); Path outPath = new Path(workdir, "stage"+stage); FileOutputFormat.setOutputPath(job, outPath); TableMapReduceUtil.addDependencyJars(job); new LoadIncrementalHFiles(getConf()).doBulkLoad(outPath, hTable); LOG.log(Level.INFO, "Stage #{0} of {1} completed..", new Object[]{stage, stages}); } else {
Job job = Job.getInstance(getConf(), "HalyardDelete " + source); if (cmd.hasOption('s')) { job.getConfiguration().set(SUBJECT, cmd.getOptionValue('s')); job.getConfiguration().set(PREDICATE, cmd.getOptionValue('p')); job.getConfiguration().set(OBJECT, cmd.getOptionValue('o')); try (HTable hTable = HalyardTableUtils.getTable(getConf(), source, false, 0)) { HFileOutputFormat2.configureIncrementalLoad(job, hTable.getTableDescriptor(), hTable.getRegionLocator()); FileOutputFormat.setOutputPath(job, new Path(cmd.getOptionValue('f'))); TableMapReduceUtil.addDependencyJars(job); if (job.waitForCompletion(true)) { new LoadIncrementalHFiles(getConf()).doBulkLoad(new Path(cmd.getOptionValue('f')), hTable); LOG.info("Bulk Delete Completed.."); return 0;
@Override public int run(String[] args) throws Exception { if (args.length != 2) { usage(); return -1; } initialize(); String dirPath = args[0]; TableName tableName = TableName.valueOf(args[1]); boolean tableExists = this.doesTableExist(tableName); if (!tableExists) { if ("yes".equalsIgnoreCase(getConf().get(CREATE_TABLE_CONF_KEY, "yes"))) { this.createTable(tableName, dirPath); } else { String errorMsg = format("Table '%s' does not exist.", tableName); LOG.error(errorMsg); throw new TableNotFoundException(errorMsg); } } Path hfofDir = new Path(dirPath); try (Connection connection = ConnectionFactory.createConnection(getConf()); HTable table = (HTable) connection.getTable(tableName);) { doBulkLoad(hfofDir, table); } return 0; }
@Override public int run(String[] args) throws Exception { String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); Job job = createSubmittableJob(otherArgs); if (job == null) return 1; if (!job.waitForCompletion(true)) { LOG.info("Map-reduce job failed!"); if (bulkload) { LOG.info("Files are not bulkloaded!"); } return 1; } int code = 0; if (bulkload) { code = new LoadIncrementalHFiles(this.getConf()).run(new String[]{this.bulkloadDir.toString(), this.dstTableName}); if (code == 0) { // bulkloadDir is deleted only LoadIncrementalHFiles was successful so that one can rerun // LoadIncrementalHFiles. FileSystem fs = FileSystem.get(this.getConf()); if (!fs.delete(this.bulkloadDir, true)) { LOG.error("Deleting folder " + bulkloadDir + " failed!"); code = 1; } } } return code; } }
Bytes.toStringBinary(table.getTableName()) + "is not currently available."); int nrThreads = cfg.getInt("hbase.loadincremental.threads.max", Runtime.getRuntime().availableProcessors()); ThreadFactoryBuilder builder = new ThreadFactoryBuilder(); discoverLoadQueue(queue, hfofDir); int count = 0; "directory " + hfofDir.toUri() + ". Does it contain files in " + "subdirectories that correspond to column family names?"); return; FileSystem fs = FileSystem.get(cfg); userToken = fs.getDelegationToken("renewer"); int maxRetries = cfg.getInt("hbase.bulkload.retries.number", 0); if (maxRetries != 0 && count >= maxRetries) { LOG.error("Retry attempted " + count + " times without completing, bailing out"); Multimap<ByteBuffer, LoadQueueItem> regionGroups = groupOrSplitPhase(table, pool, queue, startEndKeys); bulkLoadPhase(table, conn, pool, queue, regionGroups);
int nrThreads = getConf().getInt("hbase.loadincremental.threads.max", Runtime.getRuntime().availableProcessors()); ThreadFactoryBuilder builder = new ThreadFactoryBuilder(); boolean validateHFile = getConf().getBoolean("hbase.loadincremental.validate.hfile", true); if(!validateHFile) { LOG.warn("You are skipping HFiles validation, it might cause some data loss if files " + "See HBASE-13985"); discoverLoadQueue(queue, hfofDir, validateHFile); if(isSecureBulkLoadEndpointAvailable()) { bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getName()); int maxRetries = getConf().getInt("hbase.bulkload.retries.number", 10); maxRetries = Math.max(maxRetries, startEndKeys.getFirst().length + 1); if (maxRetries != 0 && count >= maxRetries) { Multimap<ByteBuffer, LoadQueueItem> regionGroups = groupOrSplitPhase(table, pool, queue, startEndKeys); if (!checkHFilesCountPerRegionPerFamily(regionGroups)) { bulkLoadPhase(table, admin.getConnection(), pool, queue, regionGroups);
@Override public int run(String[] args) throws Exception { if (args.length != 2) { usage(); return -1; } String dirPath = args[0]; String tableName = args[1]; boolean tableExists = this.doesTableExist(tableName); if (!tableExists) this.createTable(tableName,dirPath); Path hfofDir = new Path(dirPath); HTable table = new HTable(this.cfg, tableName); doBulkLoad(hfofDir, table); return 0; }
HFileOutputFormat2.configureIncrementalLoad(job, table, regionLocator); if (skipDependencyJars) { job.getConfiguration().unset("tmpjars"); LoadIncrementalHFiles loader = new LoadIncrementalHFiles(configuration); loader.doBulkLoad(outputPath, admin, table, regionLocator); FileSystem.get(configuration).delete(outputPath, true);
public void completeImport() throws Exception { LoadIncrementalHFiles loader = new LoadIncrementalHFiles(getConfiguration()); HTable table = new HTable(getConfiguration(), _tableName); loader.doBulkLoad(_hfilePath, table); FileSystem fs = _hfilePath.getFileSystem(getConfiguration()); fs.delete(_hfilePath, true); }
@Override public void commitJob(JobContext jobContext) throws IOException { try { baseOutputCommitter.commitJob(jobContext); Configuration conf = jobContext.getConfiguration(); try { //import hfiles new LoadIncrementalHFiles(conf) .doBulkLoad(HFileOutputFormat.getOutputPath(jobContext), new HTable(conf, conf.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY))); } catch (Exception e) { throw new IOException("BulkLoad failed.", e); } } finally { cleanupScratch(jobContext); } }
int ret = MRUtil.runMRJob(new LoadIncrementalHFiles(conf), newArgs); logger.debug("End to run LoadIncrementalHFiles"); return ret;