/** * Main entry point. * * @param args The command line parameters. * @throws Exception When running the job fails. */ public static void main(String[] args) throws Exception { int ret = ToolRunner.run(HBaseConfiguration.create(), new CopyTable(), args); System.exit(ret); }
@Override public int run(String[] args) throws Exception { Job job = createSubmittableJob(args); if (job == null) return 1; if (!job.waitForCompletion(true)) { LOG.info("command: ./bin/hbase org.apache.hadoop.hbase.tool.LoadIncrementalHFiles {} {}", this.bulkloadDir.toString(), this.dstTableName); code = new LoadIncrementalHFiles(this.getConf()) .run(new String[] { this.bulkloadDir.toString(), this.dstTableName }); if (code == 0) { FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); if (!fs.delete(this.bulkloadDir, true)) { LOG.error("Deleting folder " + bulkloadDir + " failed!");
if (!doCommandLine(args)) { return null; Job job = Job.getInstance(getConf(), getConf().get(JOB_NAME_CONF_KEY, jobName)); job.setJarByClass(CopyTable.class); Scan scan = new Scan(); scan.setCaching(cacheRow); } else { scan.setCaching(getConf().getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, 100)); initCopyTableMapperReducerJob(job, scan); bulkloadDir = generateUniqTempDir(false); LOG.info("HFiles will be stored at " + this.bulkloadDir); HFileOutputFormat2.setOutputPath(job, bulkloadDir); try (Connection conn = ConnectionFactory.createConnection(getConf()); Admin admin = conn.getAdmin()) { HFileOutputFormat2.configureIncrementalLoadMap(job, initCopyTableMapperReducerJob(job, scan); TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress, null, null);
if (!doCommandLine(args)) { return null; Job job = Job.getInstance(getConf(), getConf().get(JOB_NAME_CONF_KEY, NAME + "_" + tableName)); FileSystem fs = FileSystem.get(getConf()); Random rand = new Random(); Path root = new Path(fs.getWorkingDirectory(), "copytable"); try (Connection conn = ConnectionFactory.createConnection(getConf()); Table htable = conn.getTable(TableName.valueOf(dstTableName))) { HFileOutputFormat2.configureIncrementalLoadMap(job, htable);
private boolean doCommandLine(final String[] args) { if (args.length < 1) { printUsage(null); return false; String cmd = args[i]; if (cmd.equals("-h") || cmd.startsWith("--h")) { printUsage(null); return false; printUsage("Invalid argument '" + cmd + "'"); return false; printUsage("At least a new table name or a peer address must be specified"); return false; printUsage("Invalid time range filter: starttime=" + startTime + " > endtime=" + endTime); return false; printUsage("Remote bulkload is not supported!"); return false; printUsage("Loading data from snapshot to remote peer cluster is not supported."); return false; printUsage("The --new.name=<table> for destination table should be " + "provided when copying data from snapshot ."); return false; printUsage("Snapshot shouldn't be null when --snapshot is enabled.");
private Path generateUniqTempDir(boolean withDirCreated) throws IOException { FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); Path dir = new Path(fs.getWorkingDirectory(), NAME); if (!fs.exists(dir)) { fs.mkdirs(dir); } Path newDir = new Path(dir, UUID.randomUUID().toString()); if (withDirCreated) { fs.mkdirs(newDir); } return newDir; }
/** * Main entry point. * * @param args The command line parameters. * @throws Exception When running the job fails. */ public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = createSubmittableJob(conf, otherArgs); if (job != null) { System.exit(job.waitForCompletion(true) ? 0 : 1); } } }
if (!doCommandLine(args)) { return null;
/** * Test main method of CopyTable. */ @Test public void testMainMethod() throws Exception { String[] emptyArgs = { "-h" }; PrintStream oldWriter = System.err; ByteArrayOutputStream data = new ByteArrayOutputStream(); PrintStream writer = new PrintStream(data); System.setErr(writer); SecurityManager SECURITY_MANAGER = System.getSecurityManager(); LauncherSecurityManager newSecurityManager= new LauncherSecurityManager(); System.setSecurityManager(newSecurityManager); try { CopyTable.main(emptyArgs); fail("should be exit"); } catch (SecurityException e) { assertEquals(1, newSecurityManager.getExitCode()); } finally { System.setErr(oldWriter); System.setSecurityManager(SECURITY_MANAGER); } assertTrue(data.toString().contains("rs.class")); // should print usage information assertTrue(data.toString().contains("Usage:")); }
if (!doCommandLine(args)) { return null; Job job = Job.getInstance(getConf(), getConf().get(JOB_NAME_CONF_KEY, NAME + "_" + tableName)); job.setJarByClass(CopyTable.class); Scan scan = new Scan(); scan.setCaching(cacheRow); } else { scan.setCaching(getConf().getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, 100)); FileSystem fs = FileSystem.get(getConf()); Random rand = new Random(); Path root = new Path(fs.getWorkingDirectory(), "copytable"); try (Connection conn = ConnectionFactory.createConnection(getConf()); Admin admin = conn.getAdmin()) { HFileOutputFormat2.configureIncrementalLoadMap(job,
printUsage(null); return false; String cmd = args[i]; if (cmd.equals("-h") || cmd.startsWith("--h")) { printUsage(null); return false; tableName = cmd; } else { printUsage("Invalid argument '" + cmd + "'" ); return false; printUsage("At least a new table name or a " + "peer address must be specified"); return false; printUsage("Invalid time range filter: starttime=" + startTime + " > endtime=" + endTime); return false; printUsage("Can't start because " + e.getMessage()); return false;
/** * Test main method of CopyTable. */ @Test public void testMainMethod() throws Exception { String[] emptyArgs = { "-h" }; PrintStream oldWriter = System.err; ByteArrayOutputStream data = new ByteArrayOutputStream(); PrintStream writer = new PrintStream(data); System.setErr(writer); SecurityManager SECURITY_MANAGER = System.getSecurityManager(); LauncherSecurityManager newSecurityManager= new LauncherSecurityManager(); System.setSecurityManager(newSecurityManager); try { CopyTable.main(emptyArgs); fail("should be exit"); } catch (SecurityException e) { assertEquals(1, newSecurityManager.getExitCode()); } finally { System.setErr(oldWriter); System.setSecurityManager(SECURITY_MANAGER); } assertTrue(data.toString().contains("rs.class")); // should print usage information assertTrue(data.toString().contains("Usage:")); }
private boolean runCopy(String[] args) throws Exception { int status = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), new CopyTable(), args); return status == 0; }
@Override public int run(String[] args) throws Exception { Job job = createSubmittableJob(args); if (job == null) return 1; if (!job.waitForCompletion(true)) { LOG.info("Map-reduce job failed!"); if (bulkload) { LOG.info("Files are not bulkloaded!"); } return 1; } int code = 0; if (bulkload) { code = new LoadIncrementalHFiles(this.getConf()).run(new String[]{this.bulkloadDir.toString(), this.dstTableName}); if (code == 0) { // bulkloadDir is deleted only LoadIncrementalHFiles was successful so that one can rerun // LoadIncrementalHFiles. FileSystem fs = FileSystem.get(this.getConf()); if (!fs.delete(this.bulkloadDir, true)) { LOG.error("Deleting folder " + bulkloadDir + " failed!"); code = 1; } } } return code; } }
if (!doCommandLine(args)) { return null; Job job = Job.getInstance(getConf(), getConf().get(JOB_NAME_CONF_KEY, NAME + "_" + tableName)); job.setJarByClass(CopyTable.class); Scan scan = new Scan(); scan.setCaching(cacheRow); } else { scan.setCaching(getConf().getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, 100)); FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); Random rand = new Random(); Path root = new Path(fs.getWorkingDirectory(), "copytable"); try (Connection conn = ConnectionFactory.createConnection(getConf()); Admin admin = conn.getAdmin()) { HFileOutputFormat2.configureIncrementalLoadMap(job,
printUsage(null); return false; String cmd = args[i]; if (cmd.equals("-h") || cmd.startsWith("--h")) { printUsage(null); return false; tableName = cmd; } else { printUsage("Invalid argument '" + cmd + "'" ); return false; printUsage("At least a new table name or a " + "peer address must be specified"); return false; printUsage("Invalid time range filter: starttime=" + startTime + " > endtime=" + endTime); return false; printUsage("Remote bulkload is not supported!"); return false; printUsage("Can't start because " + e.getMessage()); return false;
/** * Test main method of CopyTable. */ @Test public void testMainMethod() throws Exception { String[] emptyArgs = { "-h" }; PrintStream oldWriter = System.err; ByteArrayOutputStream data = new ByteArrayOutputStream(); PrintStream writer = new PrintStream(data); System.setErr(writer); SecurityManager SECURITY_MANAGER = System.getSecurityManager(); LauncherSecurityManager newSecurityManager= new LauncherSecurityManager(); System.setSecurityManager(newSecurityManager); try { CopyTable.main(emptyArgs); fail("should be exit"); } catch (SecurityException e) { assertEquals(1, newSecurityManager.getExitCode()); } finally { System.setErr(oldWriter); System.setSecurityManager(SECURITY_MANAGER); } assertTrue(data.toString().contains("rs.class")); // should print usage information assertTrue(data.toString().contains("Usage:")); }
private void doCopyTableTest(boolean bulkload) throws Exception { final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); final byte[] FAMILY = Bytes.toBytes("family"); final byte[] COLUMN1 = Bytes.toBytes("c1"); try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) { // put rows into the first table loadData(t1, FAMILY, COLUMN1); CopyTable copy = new CopyTable(); int code; if (bulkload) { code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { "--new.name=" + tableName2.getNameAsString(), "--bulkload", tableName1.getNameAsString() }); } else { code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { "--new.name=" + tableName2.getNameAsString(), tableName1.getNameAsString() }); } assertEquals("copy job failed", 0, code); // verify the data was copied into table 2 verifyRows(t2, FAMILY, COLUMN1); } finally { TEST_UTIL.deleteTable(tableName1); TEST_UTIL.deleteTable(tableName2); } }
@Override public int run(String[] args) throws Exception { Job job = createSubmittableJob(args); if (job == null) return 1; if (!job.waitForCompletion(true)) { LOG.info("Map-reduce job failed!"); if (bulkload) { LOG.info("Files are not bulkloaded!"); } return 1; } int code = 0; if (bulkload) { code = new LoadIncrementalHFiles(this.getConf()) .run(new String[] { this.bulkloadDir.toString(), this.dstTableName }); if (code == 0) { // bulkloadDir is deleted only LoadIncrementalHFiles was successful so that one can rerun // LoadIncrementalHFiles. FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); if (!fs.delete(this.bulkloadDir, true)) { LOG.error("Deleting folder " + bulkloadDir + " failed!"); code = 1; } } } return code; } }
printUsage(null); return false; String cmd = args[i]; if (cmd.equals("-h") || cmd.startsWith("--h")) { printUsage(null); return false; tableName = cmd; } else { printUsage("Invalid argument '" + cmd + "'"); return false; printUsage("At least a new table name or a " + "peer address must be specified"); return false; printUsage("Invalid time range filter: starttime=" + startTime + " > endtime=" + endTime); return false; printUsage("Remote bulkload is not supported!"); return false; printUsage("Can't start because " + e.getMessage()); return false;