protected void runLoadTestOnExistingTable() throws IOException { writerThreads.start(0, numKeys, NUM_THREADS); writerThreads.waitForFinish(); assertEquals(0, writerThreads.getNumWriteFailures()); readerThreads.start(0, numKeys, NUM_THREADS); readerThreads.waitForFinish(); assertEquals(0, readerThreads.getNumReadFailures()); assertEquals(0, readerThreads.getNumReadErrors()); assertEquals(numKeys, readerThreads.getNumKeysVerified()); }
/** Loads data into the table using the multi-threaded writer. */ private void loadData() throws IOException { long startKey = 0; long endKey = 100000; int minColsPerKey = 5; int maxColsPerKey = 15; int minColDataSize = 256; int maxColDataSize = 256 * 3; int numThreads = 10; // print out the arguments System.out.printf("Key range %d .. %d\n", startKey, endKey); System.out.printf("Number of Columns/Key: %d..%d\n", minColsPerKey, maxColsPerKey); System.out.printf("Data Size/Column: %d..%d bytes\n", minColDataSize, maxColDataSize); System.out.printf("Client Threads: %d\n", numThreads); // start the writers LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator( minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, HFileTestUtil.DEFAULT_COLUMN_FAMILY); MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); writer.setMultiPut(true); writer.start(startKey, endKey, numThreads); System.out.printf("Started loading data..."); writer.waitForFinish(); System.out.printf("Finished loading data..."); }
@Override public void start(long startKey, long endKey, int numThreads) throws IOException { super.start(startKey, endKey, numThreads); if (verbose) { LOG.debug("Inserting keys [" + startKey + ", " + endKey + ")"); } createWriterThreads(numThreads); startThreads(writers); }
protected MultiThreadedWriter prepareWriterThreads(LoadTestDataGenerator dataGen, Configuration conf, TableName tableName) throws IOException { MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, tableName); writer.setMultiPut(isMultiPut); return writer; }
writerThreads.setMultiPut(isMultiPut); writerThreads.start(startKey, endKey, numWriterThreads); writerThreads.waitForFinish(); success = success && writerThreads.getNumWriteFailures() == 0;
new MultiThreadedWriter(dataGen, conf, TABLE_NAME); writer.setMultiPut(true); writer.start(1, keysToWrite, WRITER_THREADS); LOG.info("Starting the scan; wrote approximately " + dataGen.getTotalNumberOfKeys() + " keys"); isWriterDone = writer.isDone(); if (isWriterDone) { LOG.info("Scanning full result, writer is done"); boolean isOk = writer.verifyResultAgainstDataGenerator(result, true, true); Assert.assertTrue("Failed to verify [" + Bytes.toString(result.getRow())+ "]", isOk); ++resultCount; Assert.assertEquals("There are write failures", 0, writer.getNumWriteFailures()); Assert.assertTrue("Writer is not done", isWriterDone);
MultiThreadedWriter preloader = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); long time = System.currentTimeMillis(); preloader.start(0, startKey, writeThreads); preloader.waitForFinish(); if (preloader.getNumWriteFailures() > 0) { throw new IOException("Preload failed"); MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); MultiThreadedReader reader = new MultiThreadedReader(dataGen, conf, TABLE_NAME, 100); writer.start(startKey, endKey, writeThreads); reader.start(startKey, endKey, readThreads); writer.waitForFinish(); reader.waitForFinish(); boolean success = writer.getNumWriteFailures() == 0; if (!success) { LOG.error("Write failed");
@Override public void start(long startKey, long endKey, int numThreads) throws IOException { super.start(startKey, endKey, numThreads); }
public void insert(Table table, Put put, long keyBase) { long start = System.currentTimeMillis(); try { put = (Put) dataGenerator.beforeMutate(keyBase, put); table.put(put); totalOpTimeMs.addAndGet(System.currentTimeMillis() - start); } catch (IOException e) { failedKeySet.add(keyBase); String exceptionInfo; if (e instanceof RetriesExhaustedWithDetailsException) { RetriesExhaustedWithDetailsException aggEx = (RetriesExhaustedWithDetailsException)e; exceptionInfo = aggEx.getExhaustiveDescription(); } else { StringWriter stackWriter = new StringWriter(); PrintWriter pw = new PrintWriter(stackWriter); e.printStackTrace(pw); pw.flush(); exceptionInfo = StringUtils.stringifyException(e); } LOG.error("Failed to insert: " + keyBase + " after " + (System.currentTimeMillis() - start) + "ms; region information: " + getRegionDebugInfoSafe(table, put.getRow()) + "; errors: " + exceptionInfo); } } protected void closeHTable() {
writerThreads.setMultiPut(isMultiPut); writerThreads.start(startKey, endKey, numWriterThreads); writerThreads.waitForFinish(); success = success && writerThreads.getNumWriteFailures() == 0;
new MultiThreadedWriter(dataGen, conf, TABLE_NAME); writer.setMultiPut(true); writer.start(1, keysToWrite, WRITER_THREADS); LOG.info("Starting the scan; wrote approximately " + dataGen.getTotalNumberOfKeys() + " keys"); isWriterDone = writer.isDone(); if (isWriterDone) { LOG.info("Scanning full result, writer is done"); boolean isOk = writer.verifyResultAgainstDataGenerator(result, true, true); Assert.assertTrue("Failed to verify [" + Bytes.toString(result.getRow())+ "]", isOk); ++resultCount; Assert.assertEquals("There are write failures", 0, writer.getNumWriteFailures()); Assert.assertTrue("Writer is not done", isWriterDone);
MultiThreadedWriter preloader = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); long time = System.currentTimeMillis(); preloader.start(0, startKey, writeThreads); preloader.waitForFinish(); if (preloader.getNumWriteFailures() > 0) { throw new IOException("Preload failed"); MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); MultiThreadedReader reader = new MultiThreadedReader(dataGen, conf, TABLE_NAME, 100); writer.start(startKey, endKey, writeThreads); reader.start(startKey, endKey, readThreads); writer.waitForFinish(); reader.waitForFinish(); boolean success = writer.getNumWriteFailures() == 0; if (!success) { LOG.error("Write failed");
protected MultiThreadedWriter prepareWriterThreads(LoadTestDataGenerator dataGen, Configuration conf, TableName tableName) throws IOException { MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, tableName); writer.setMultiPut(isMultiPut); return writer; }
@Override public void start(long startKey, long endKey, int numThreads) throws IOException { super.start(startKey, endKey, numThreads); }
public void insert(Table table, Put put, long keyBase) { long start = System.currentTimeMillis(); try { put = (Put) dataGenerator.beforeMutate(keyBase, put); table.put(put); totalOpTimeMs.addAndGet(System.currentTimeMillis() - start); } catch (IOException e) { failedKeySet.add(keyBase); String exceptionInfo; if (e instanceof RetriesExhaustedWithDetailsException) { RetriesExhaustedWithDetailsException aggEx = (RetriesExhaustedWithDetailsException)e; exceptionInfo = aggEx.getExhaustiveDescription(); } else { StringWriter stackWriter = new StringWriter(); PrintWriter pw = new PrintWriter(stackWriter); e.printStackTrace(pw); pw.flush(); exceptionInfo = StringUtils.stringifyException(e); } LOG.error("Failed to insert: " + keyBase + " after " + (System.currentTimeMillis() - start) + "ms; region information: " + getRegionDebugInfoSafe(table, put.getRow()) + "; errors: " + exceptionInfo); } } protected void closeHTable() {
writerThreads.setMultiPut(isMultiPut); writerThreads.start(startKey, endKey, numWriterThreads); writerThreads.waitForFinish(); success = success && writerThreads.getNumWriteFailures() == 0;
@Override @Test public void loadTest() throws Exception { prepareForLoadTest(); readerThreads.linkToWriter(writerThreads); writerThreads.start(0, numKeys, NUM_THREADS); readerThreads.start(0, numKeys, NUM_THREADS); writerThreads.waitForFinish(); readerThreads.waitForFinish(); assertEquals(0, writerThreads.getNumWriteFailures()); assertEquals(0, readerThreads.getNumReadFailures()); assertEquals(0, readerThreads.getNumReadErrors()); assertEquals(numKeys, readerThreads.getNumUniqueKeysVerified()); }
/** Loads data into the table using the multi-threaded writer. */ private void loadData() throws IOException { long startKey = 0; long endKey = 100000; int minColsPerKey = 5; int maxColsPerKey = 15; int minColDataSize = 256; int maxColDataSize = 256 * 3; int numThreads = 10; // print out the arguments System.out.printf("Key range %d .. %d\n", startKey, endKey); System.out.printf("Number of Columns/Key: %d..%d\n", minColsPerKey, maxColsPerKey); System.out.printf("Data Size/Column: %d..%d bytes\n", minColDataSize, maxColDataSize); System.out.printf("Client Threads: %d\n", numThreads); // start the writers LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator( minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, HFileTestUtil.DEFAULT_COLUMN_FAMILY); MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); writer.setMultiPut(true); writer.start(startKey, endKey, numThreads); System.out.printf("Started loading data..."); writer.waitForFinish(); System.out.printf("Finished loading data..."); }
@Override public void start(long startKey, long endKey, int numThreads) throws IOException { super.start(startKey, endKey, numThreads); if (verbose) { LOG.debug("Inserting keys [" + startKey + ", " + endKey + ")"); } createWriterThreads(numThreads); startThreads(writers); }
protected void runLoadTestOnExistingTable() throws IOException { writerThreads.start(0, numKeys, NUM_THREADS); writerThreads.waitForFinish(); assertEquals(0, writerThreads.getNumWriteFailures()); readerThreads.start(0, numKeys, NUM_THREADS); readerThreads.waitForFinish(); assertEquals(0, readerThreads.getNumReadFailures()); assertEquals(0, readerThreads.getNumReadErrors()); assertEquals(numKeys, readerThreads.getNumKeysVerified()); }