public TransactionImpl(Environment env) { this(env, null, allocateTimestamp(env).getTxTimestamp()); }
public TransactionImpl(Environment env) { this(env, null, allocateTimestamp(env).getTxTimestamp()); }
public TransactionImpl(Environment env, Notification trigger) { this(env, trigger, allocateTimestamp(env).getTxTimestamp()); }
public TransactionImpl(Environment env, Notification trigger) { this(env, trigger, allocateTimestamp(env).getTxTimestamp()); }
@Override public void run() { OracleClient oclient = env.getSharedResources().getOracleClient(); for (int i = 0; i < numToGet; i++) { try { output.add(oclient.getStamp().getTxTimestamp()); } catch (Exception e) { e.printStackTrace(); } } cdl.countDown(); } }
/** * Allocate a timestamp */ public Stamp allocateTimestamp() { synchronized (this) { Preconditions.checkState(!closed, "tracker closed "); if (node == null) { Preconditions.checkState(allocationsInProgress == 0, "expected allocationsInProgress == 0 when node == null"); Preconditions.checkState(!updatingZk, "unexpected concurrent ZK update"); createZkNode(getTimestamp().getTxTimestamp()); } allocationsInProgress++; } try { Stamp ts = getTimestamp(); synchronized (this) { timestamps.add(ts.getTxTimestamp()); } return ts; } catch (RuntimeException re) { synchronized (this) { allocationsInProgress--; } throw re; } }
/** * Test that bogus input into the oracle server doesn't cause an OOM exception. This essentially * tests for THRIFT-602 */ @Test public void bogusDataTest() throws Exception { Socket socket = new Socket(); socket.connect(new InetSocketAddress(HostUtil.getHostName(), oserver.getPort())); OutputStream outstream = socket.getOutputStream(); try (PrintWriter out = new PrintWriter(outstream)) { out.print("abcd"); out.flush(); } socket.close(); OracleClient client = env.getSharedResources().getOracleClient(); assertEquals(2, client.getStamp().getTxTimestamp()); }
@Override CompletableFuture<Boolean> getMainOp(CommitData cd) { return getStampOp().thenApply(commitStamp -> { if (startTs < commitStamp.getGcTimestamp()) { return false; } else { getStats().setCommitTs(commitStamp.getTxTimestamp()); return true; } }); }
@VisibleForTesting public boolean finishCommit(CommitData cd, Stamp commitStamp) { cd.commitObserver = new SyncCommitObserver(); getStats().setCommitTs(commitStamp.getTxTimestamp()); CommitStep firstStep = new DeleteLocksStep(); firstStep.andThen(new FinishCommitStep()); firstStep.compose(cd).exceptionally(throwable -> { System.err.println("Unexpected exception in finish commit test method : "); throwable.printStackTrace(); return null; }); return true; }
@Override CompletableFuture<Boolean> getMainOp(CommitData cd) { return getStampOp().thenApply(commitStamp -> { if (startTs < commitStamp.getGcTimestamp()) { return false; } else { getStats().setCommitTs(commitStamp.getTxTimestamp()); return true; } }); }
@Test public void testRestart() throws Exception { OracleClient client = env.getSharedResources().getOracleClient(); long ts1 = client.getStamp().getTxTimestamp(); long ts2 = client.getStamp().getTxTimestamp(); oserver.stop(); oserver.start(); long ts3 = client.getStamp().getTxTimestamp(); long ts4 = client.getStamp().getTxTimestamp(); assertTrue(ts1 + " " + ts2, ts1 < ts2); assertTrue(ts2 + " " + ts3, ts2 < ts3); assertTrue(ts3 + " " + ts4, ts3 < ts4); }
@VisibleForTesting public boolean finishCommit(CommitData cd, Stamp commitStamp) { cd.commitObserver = new SyncCommitObserver(); getStats().setCommitTs(commitStamp.getTxTimestamp()); CommitStep firstStep = new DeleteLocksStep(); firstStep.andThen(new FinishCommitStep()); firstStep.compose(cd).exceptionally(throwable -> { System.err.println("Unexpected exception in finish commit test method : "); throwable.printStackTrace(); return null; }); return true; }
@Override public void waitForObservers() { try { Scanner scanner = env.getConnector().createScanner(env.getTable(), env.getAuthorizations()); Notification.configureScanner(scanner); while (true) { long ts1 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); long ntfyCount = Iterables.size(scanner); long ts2 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); if (ntfyCount == 0 && ts1 == (ts2 - 1)) { break; } long sleepTime = ntfyCount / 2; sleepTime = Math.min(Math.max(10, sleepTime), 10000); Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void waitForObservers() { try { Scanner scanner = env.getAccumuloClient().createScanner(env.getTable(), env.getAuthorizations()); Notification.configureScanner(scanner); while (true) { long ts1 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); long ntfyCount = Iterables.size(scanner); long ts2 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); if (ntfyCount == 0 && ts1 == (ts2 - 1)) { break; } long sleepTime = ntfyCount / 2; sleepTime = Math.min(Math.max(10, sleepTime), 10000); Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setClientInfo(conf, AccumuloUtil.getClientInfo(fconfig)); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link org.apache.fluo.api.config.FluoConfiguration} to configure * programmatically */ public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setClientInfo(conf, AccumuloUtil.getClientInfo(fconfig)); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * If an {@link OracleServer} goes away and comes back, the client should automatically reconnect * and start a new block of timestamps (making sure that no timestamp should ever go backwards). */ @Test public void singleOracle_goesAwayAndComesBack() throws Exception { sleepUntil(oserver::isConnected); OracleClient client = env.getSharedResources().getOracleClient(); long timestamp; for (long i = 2; i <= 7; i++) { timestamp = client.getStamp().getTxTimestamp(); assertEquals(i, timestamp); } oserver.stop(); sleepWhile(oserver::isConnected); while (client.getOracle() != null) { Thread.sleep(100); } assertNull(client.getOracle()); oserver.start(); sleepUntil(oserver::isConnected); assertEquals(1002, client.getStamp().getTxTimestamp()); assertTrue(client.getOracle().endsWith(Integer.toString(oserver.getPort()))); oserver.stop(); }
@Test public void testTimestampUtilGetOldestTs() throws Exception { Assert.assertEquals(0, getOldestTs()); TimestampTracker tr1 = new TimestampTracker(env, new TransactorID(env), 5); final long ts1 = tr1.allocateTimestamp().getTxTimestamp(); Thread.sleep(15); Assert.assertEquals(tr1.getZookeeperTimestamp(), getOldestTs()); TimestampTracker tr2 = new TimestampTracker(env, new TransactorID(env), 5); final long ts2 = tr2.allocateTimestamp().getTxTimestamp(); TimestampTracker tr3 = new TimestampTracker(env, new TransactorID(env), 5); final long ts3 = tr3.allocateTimestamp().getTxTimestamp(); Thread.sleep(15); Assert.assertEquals(ts1, getOldestTs()); tr1.removeTimestamp(ts1); Thread.sleep(15); Assert.assertEquals(ts2, getOldestTs()); tr2.removeTimestamp(ts2); Thread.sleep(15); Assert.assertEquals(ts3, getOldestTs()); tr3.removeTimestamp(ts3); tr1.close(); tr2.close(); tr3.close(); }
@Test public void testTrackingWithZkUpdate() throws Exception { TimestampTracker tracker = new TimestampTracker(env, new TransactorID(env), 5); final long ts1 = tracker.allocateTimestamp().getTxTimestamp(); Thread.sleep(15); Assert.assertNotNull(ts1); Assert.assertTrue(zkNodeExists(tracker)); Assert.assertNotNull(zkNodeValue(tracker)); Assert.assertEquals(tracker.getZookeeperTimestamp(), zkNodeValue(tracker)); Assert.assertEquals(ts1, tracker.getOldestActiveTimestamp()); final long ts2 = tracker.allocateTimestamp().getTxTimestamp(); Assert.assertEquals(ts1, tracker.getOldestActiveTimestamp()); Thread.sleep(15); tracker.removeTimestamp(ts1); Thread.sleep(15); Assert.assertEquals(ts2, tracker.getOldestActiveTimestamp()); Assert.assertEquals(ts2, zkNodeValue(tracker)); tracker.removeTimestamp(ts2); Thread.sleep(15); Assert.assertTrue(tracker.isEmpty()); Assert.assertFalse(zkNodeExists(tracker)); tracker.close(); }
@Test public void testTrackingWithNoUpdate() throws Exception { TimestampTracker tracker = new TimestampTracker(env, new TransactorID(env)); Assert.assertTrue(tracker.isEmpty()); Assert.assertFalse(zkNodeExists(tracker)); final long ts1 = tracker.allocateTimestamp().getTxTimestamp(); Assert.assertFalse(tracker.isEmpty()); Assert.assertTrue(zkNodeExists(tracker)); Assert.assertTrue(ts1 > zkNodeValue(tracker)); Assert.assertEquals(tracker.getZookeeperTimestamp(), zkNodeValue(tracker)); Assert.assertEquals(ts1, tracker.getOldestActiveTimestamp()); final long ts2 = tracker.allocateTimestamp().getTxTimestamp(); Assert.assertEquals(ts1, tracker.getOldestActiveTimestamp()); tracker.removeTimestamp(ts1); Assert.assertFalse(tracker.isEmpty()); Assert.assertEquals(ts2, tracker.getOldestActiveTimestamp()); Assert.assertFalse(tracker.isEmpty()); Assert.assertTrue(ts1 > zkNodeValue(tracker)); Assert.assertEquals(tracker.getZookeeperTimestamp(), zkNodeValue(tracker)); tracker.removeTimestamp(ts2); Assert.assertTrue(tracker.isEmpty()); Assert.assertTrue(zkNodeExists(tracker)); tracker.close(); }