eventsToProcess = eventsDataset.toBefore("timestamp", currentMinute);
testDataset.to("timestamp", octInstant)); assertContentEquals(Sets.newHashSet(sepEvent, octEvent), testDataset.toBefore("timestamp", novStart));
@Override public int run(String[] args) throws Exception { final long startOfToday = startOfDay(); // the destination dataset Dataset<Record> persistent = Datasets.load( "dataset:file:/tmp/data/logs", Record.class); // the source: anything before today in the staging area Dataset<Record> staging = Datasets.load( "dataset:file:/tmp/data/logs_staging", Record.class); View<Record> ready = staging.toBefore("timestamp", startOfToday); ReadableSource<Record> source = CrunchDatasets.asSource(ready); PCollection<Record> stagedLogs = read(source); getPipeline().write(stagedLogs, CrunchDatasets.asTarget(persistent), Target.WriteMode.APPEND); PipelineResult result = run(); if (result.succeeded()) { // remove the source data partition from staging ready.deleteAll(); return 0; } else { return 1; } }
@Test public void testSimpleViews() { assertViewUriEquivalent("dataset", "dataset:file:/tmp/test_name", test); assertViewUriEquivalent("to constraint", "view:file:/tmp/test_name?timestamp=(,0]", test.to("timestamp", 0L)); assertViewUriEquivalent("View with toBefore constraint", "view:file:/tmp/test_name?timestamp=(,0)", test.toBefore("timestamp", 0L)); assertViewUriEquivalent("View with from constraint", "view:file:/tmp/test_name?timestamp=[0,)", test.from("timestamp", 0L)); assertViewUriEquivalent("View with fromAfter constraint", "view:file:/tmp/test_name?timestamp=(0,)", test.fromAfter("timestamp", 0L)); assertViewUriEquivalent("View with in(\"\") constraint", "view:file:/tmp/test_name?color=in()", test.with("color", "")); assertViewUriEquivalent("View with in constraint", "view:file:/tmp/test_name?color=orange,red", test.with("color", "orange", "red")); assertViewUriEquivalent("View with exists constraint", "view:file:/tmp/test_name?id=", test.with("id")); }
notPartitioned.to("timestamp", now)); Assert.assertNotNull("toBefore should succeed", notPartitioned.toBefore("timestamp", now)); Assert.assertNotNull("with should succeed", notPartitioned.with("timestamp", now));