@PUT @Path("lines") public void write(HttpServiceRequest request, HttpServiceResponder responder, @QueryParam("time") Long time) { PartitionKey key = PartitionKey.builder().addLongField("time", time).build(); PartitionOutput partitionOutput = lines.getPartitionOutput(key); Location location = partitionOutput.getLocation(); try { try (WritableByteChannel channel = Channels.newChannel(location.getOutputStream())) { channel.write(request.getContent()); } partitionOutput.addPartition(); } catch (IOException e) { responder.sendError(400, String.format("Unable to write path '%s'", location)); return; } responder.sendStatus(200); }
private void addTimePartition(DataSetManager<TimePartitionedFileSet> tpfsManager, long inputTime) throws IOException, TransactionFailureException, InterruptedException { TimePartitionedFileSet tpfs = tpfsManager.get(); PartitionOutput partitionOutput = tpfs.getPartitionOutput(inputTime); Location location = partitionOutput.getLocation(); prepareFileInput(location); partitionOutput.addPartition(); tpfsManager.flush(); } }
@Override public void run(DatasetContext context) throws Exception { Map<String, Long> wordCounts = new HashMap<>(); for (PartitionDetail partition : partitions) { ByteBuffer content; Location location = partition.getLocation(); content = ByteBuffer.wrap(ByteStreams.toByteArray(location.getInputStream())); String string = Bytes.toString(Bytes.toBytes(content)); for (String token : string.split(" ")) { Long count = Objects.firstNonNull(wordCounts.get(token), 0L); wordCounts.put(token, count + 1); } } IncrementingKeyValueTable counts = context.getDataset("counts"); for (Map.Entry<String, Long> entry : wordCounts.entrySet()) { counts.write(Bytes.toBytes(entry.getKey()), entry.getValue()); } PartitionedFileSet outputLines = context.getDataset("outputLines"); PartitionKey partitionKey = PartitionKey.builder().addLongField("time", System.currentTimeMillis()).build(); PartitionOutput outputPartition = outputLines.getPartitionOutput(partitionKey); Location partitionDir = outputPartition.getLocation(); partitionDir.mkdirs(); Location outputLocation = partitionDir.append("file"); outputLocation.createNew(); try (OutputStream outputStream = outputLocation.getOutputStream()) { outputStream.write(Bytes.toBytes(Joiner.on("\n").join(wordCounts.values()))); } outputPartition.addPartition(); } });
private Location createPartition(PartitionedFileSet pfs, PartitionKey key, String fileName, int intToWrite) throws IOException { PartitionOutput partitionOutput = pfs.getPartitionOutput(key); Location outputLocation = partitionOutput.getLocation().append(fileName); Assert.assertFalse(outputLocation.exists()); try (OutputStream outputStream = outputLocation.getOutputStream()) { outputStream.write(intToWrite); } Assert.assertTrue(outputLocation.exists()); partitionOutput.addPartition(); return outputLocation; }
PartitionedFileSet pfs = getContext().getDataset(dataset); final PartitionOutput partitionOutput = pfs.getPartitionOutput(partitionKey); final Location partitionDir = partitionOutput.getLocation(); if (!partitionDir.mkdirs()) { responder.sendError(HttpURLConnection.HTTP_CONFLICT,
location1 = output1.getLocation(); try (Writer writer = new OutputStreamWriter(location1.append("file").getOutputStream())) { writer.write("1,1\n"); location3 = output3.getLocation(); String basePath = pfs.getEmbeddedFileSet().getBaseLocation().toURI().getPath(); String absPath3 = location3.toURI().getPath();
final Location partitionDir = output.getLocation(); if (!partitionDir.mkdirs()) { responder.sendString(409, "Partition exists.", Charsets.UTF_8);
final Location location4x = output4x.getLocation(); try (Writer writer = new OutputStreamWriter(location4x.append("file").getOutputStream())) { writer.write("4x,4x\n"); final Location location5x = output5x.getLocation(); try (Writer writer = new OutputStreamWriter(location5x.append("file").getOutputStream())) { writer.write("5x,5x\n"); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); Assert.assertFalse(pfs.getPartitionOutput(KEY_2).getLocation().exists()); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); Assert.assertFalse(pfs.getPartitionOutput(KEY_4).getLocation().exists()); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); Assert.assertFalse(pfs.getPartitionOutput(KEY_5).getLocation().exists()); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); Assert.assertFalse(pfs.getPartitionOutput(KEY_2).getLocation().exists()); Assert.assertFalse(pfs.getPartitionOutput(KEY_5).getLocation().exists()); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); Assert.assertFalse(pfs.getPartitionOutput(KEY_0).getLocation().exists()); Assert.assertFalse(pfs.getPartitionOutput(KEY_4).getLocation().exists());
PartitionedFileSet pfs = pfsManager.get(); PartitionOutput partitionOutput = pfs.getPartitionOutput(PartitionKey.builder().addStringField("x", "nn").build()); Location location = partitionOutput.getLocation(); prepareFileInput(location); partitionOutput.addPartition();
Location partitionLocation = partitionOutput.getLocation(); int numInputFiles = 100; List<String> writtenData = writeSmallOrcFiles(partitionLocation, numInputFiles);