/** Write a DatasetGraph in streaming fashion * * @param output OutputStream * @param datasetGraph DatasetGraph to write * @param format Syntax */ public static void write(OutputStream output, DatasetGraph datasetGraph, RDFFormat format) { StreamRDF stream = getWriterStream(output, format) ; StreamOps.datasetToStream(datasetGraph, stream) ; }
/** Write a DatasetGraph in streaming fashion * * @param output OutputStream * @param datasetGraph DatasetGraph to write * @param lang Syntax */ public static void write(OutputStream output, DatasetGraph datasetGraph, Lang lang) { RDFFormat fmt = registry.choose(lang) ; write(output, datasetGraph, fmt) ; }
private static void printRegistered(PrintStream out) { out.println("Streaming languages:") ; Set<Lang> seen = new HashSet<>() ; for ( RDFFormat fmt : StreamRDFWriter.registered()) { Lang lang = fmt.getLang() ; if ( hiddenLanguages.contains(lang)) continue ; if ( seen.contains(lang) ) continue ; seen.add(lang) ; out.println(" "+lang.getLabel()) ; } System.err.println("Non-streaming languages:") ; for ( RDFFormat fmt : RDFWriterRegistry.registered() ) { Lang lang = fmt.getLang() ; if ( hiddenLanguages.contains(lang)) continue ; if ( seen.contains(lang) ) continue ; seen.add(lang) ; out.println(" "+lang.getLabel()) ; } }
new RuntimeTrellisException("Invalid content type: " + syntax.mediaType())); final RDFFormat format = defaultSerialization(lang); final StreamRDF stream = getWriterStream(output, format); stream.start(); ofNullable(nsService).ifPresent(svc -> svc.getNamespaces().forEach(stream::prefix));
if ( lang == null ) throw new CmdException("Not recognized as an RDF language : '"+langName+"'") ; streamOutput = StreamRDFWriter.defaultSerialization(lang) ; if ( streamOutput == null ) { System.err.println("Language '"+lang.getLabel()+"' not registered for streaming.") ; throw new CmdException("Not recognized as an RDF language : '"+langName+"'") ; if ( StreamRDFWriter.registered(lang) ) { streamOutput = StreamRDFWriter.defaultSerialization(lang) ; } else { formattedOutput = RDFWriterRegistry.defaultSerialization(lang) ;
private static void write(final RdfStream rdfStream, final OutputStream output, final Lang dataFormat, final MediaType dataMediaType, final Map<String, String> nsPrefixes) { final RDFFormat format = defaultSerialization(dataFormat); // For formats that can be block-streamed (n-triples, turtle) if (format != null) { LOGGER.debug("Stream-based serialization of {}", dataFormat.toString()); if (RDFFormat.NTRIPLES.equals(format)) { serializeNTriples(rdfStream, format, output); } else { serializeBlockStreamed(rdfStream, output, format, nsPrefixes); } // For formats that require analysis of the entire model and cannot be streamed directly (rdfxml, n3) } else { LOGGER.debug("Non-stream serialization of {}", dataFormat.toString()); serializeNonStreamed(rdfStream, output, dataFormat, dataMediaType, nsPrefixes); } }
new RuntimeTrellisException("Invalid content type: " + syntax.mediaType())); final RDFFormat format = defaultSerialization(lang); final StreamRDF stream = getWriterStream(output, format); stream.start(); ofNullable(nsService).ifPresent(svc -> svc.getNamespaces().forEach(stream::prefix));
if ( lang == null ) throw new CmdException("Not recognized as an RDF language : '"+langName+"'") ; streamOutput = StreamRDFWriter.defaultSerialization(lang) ; if ( streamOutput == null ) { System.err.println("Language '"+lang.getLabel()+"' not registered for streaming.") ; throw new CmdException("Not recognized as an RDF language : '"+langName+"'") ; if ( StreamRDFWriter.registered(lang) ) { streamOutput = StreamRDFWriter.defaultSerialization(lang) ; } else { formattedOutput = RDFWriterRegistry.defaultSerialization(lang) ;
/** Get a StreamRDF destination that will output in syntax {@code Lang} * and is guaranteed to do so in a scaling, streaming fashion. * @param output OutputStream * @param lang The syntax * @return StreamRDF, or null if Lang does not have a streaming format. * @see StreamOps#graphToStream * @see StreamOps#datasetToStream */ public static StreamRDF getWriterStream(OutputStream output, Lang lang) { RDFFormat fmt = registry.choose(lang) ; return getWriterStream(output, fmt) ; }
/** Write a Graph in streaming fashion * * @param output OutputStream * @param graph Graph to write * @param lang Syntax */ public static void write(OutputStream output, Graph graph, Lang lang) { RDFFormat fmt = registry.choose(lang) ; write(output, graph, fmt) ; }
private static void printRegistered(PrintStream out) { out.println("Streaming languages:") ; Set<Lang> seen = new HashSet<>() ; for ( RDFFormat fmt : StreamRDFWriter.registered()) { Lang lang = fmt.getLang() ; if ( hiddenLanguages.contains(lang)) continue ; if ( seen.contains(lang) ) continue ; seen.add(lang) ; out.println(" "+lang.getLabel()) ; } System.err.println("Non-streaming languages:") ; for ( RDFFormat fmt : RDFWriterRegistry.registered() ) { Lang lang = fmt.getLang() ; if ( hiddenLanguages.contains(lang)) continue ; if ( seen.contains(lang) ) continue ; seen.add(lang) ; out.println(" "+lang.getLabel()) ; } }
/** Write a Graph in streaming fashion * * @param output OutputStream * @param graph Graph to write * @param lang Syntax */ public static void write(OutputStream output, Graph graph, RDFFormat lang) { StreamRDF stream = getWriterStream(output, lang) ; StreamOps.graphToStream(graph, stream) ; }
/** Create a streaming output sink if possible */ protected StreamRDF createStreamSink() { if ( modLangParse.toBitBucket() ) return StreamRDFLib.sinkNull() ; RDFFormat fmt = modLangOutput.getOutputStreamFormat() ; if ( fmt == null ) return null ; /** Create an accumulating output stream for later pretty printing */ return StreamRDFWriter.getWriterStream(outputWrite, fmt) ; }
@Test public void graph_02() { ByteArrayOutputStream out = new ByteArrayOutputStream() ; StreamRDFWriter.write(out, graph, Lang.RDFTHRIFT) ; byte[] bytes = out.toByteArray() ; ByteArrayInputStream in = new ByteArrayInputStream(bytes) ; Graph g2 = GraphFactory.createGraphMem() ; RDFDataMgr.read(g2, in, Lang.RDFTHRIFT) ; boolean b = IsoMatcher.isomorphic(graph, g2) ; assertTrue(b) ; // Stronger - same bNodes. // ** Java8 //graph.find(null, null, null).forEachRemaining(t -> assertTrue(g2.contains(t))) ; // Stronger - same bNodes. sameTerms(graph, g2) ; }
/** Create a streaming output sink if possible */ protected StreamRDF createStreamSink() { if ( modLangParse.toBitBucket() ) return StreamRDFLib.sinkNull() ; RDFFormat fmt = modLangOutput.getOutputStreamFormat() ; if ( fmt == null ) return null ; /** Create an accumulating output stream for later pretty printing */ return StreamRDFWriter.getWriterStream(outputWrite, fmt) ; }
@Test public void dataset_02() { DatasetGraph dsg1 = datasetGraph ; ByteArrayOutputStream out = new ByteArrayOutputStream() ; StreamRDFWriter.write(out, dsg1, Lang.RDFTHRIFT) ; byte[] bytes = out.toByteArray() ; ByteArrayInputStream in = new ByteArrayInputStream(bytes) ; DatasetGraph dsg2 = DatasetGraphFactory.create() ; StreamRDF stream2 = StreamRDFLib.dataset(dsg2) ; BinRDF.inputStreamToStream(in, stream2) ; boolean b = IsoMatcher.isomorphic(dsg1, dsg2) ; assertTrue(b) ; // Stronger - same bNode and same as in original data. Node obj = Iter.first(dsg1.listGraphNodes(), Node::isBlank) ; termAsObject(dsg1, obj) ; }
@Override public Observable<JsonNode> generate(final Observable<GDMModel> recordGDM, final OutputStream outputStream) throws XMLStreamException { final StreamRDF writer = StreamRDFWriter.getWriterStream(outputStream, rdfSerializationFormat); writer.start(); final ConcurrentHashMap<String, org.apache.jena.graph.Node> resourceNodeCache = new ConcurrentHashMap<>(); final ConcurrentHashMap<String, org.apache.jena.graph.Node> predicateCache = new ConcurrentHashMap<>(); return recordGDM .doOnSubscribe(() -> LOG.debug("subscribed to RDF export; will return data as '{}'", mediaType.toString())) .onBackpressureBuffer(10000) .map(recordGDMModel -> processRecordGDMModel(writer, resourceNodeCache, predicateCache, recordGDMModel)) .map(org.dswarm.persistence.model.internal.Model::toGDMCompactJSON) .flatMapIterable(nodes -> { final ArrayList<JsonNode> nodeList = new ArrayList<>(); Iterators.addAll(nodeList, nodes.elements()); return nodeList; }) .doOnCompleted(writer::finish) .doOnCompleted(() -> LOG.debug("finished RDF export; return data as '{}'", mediaType.toString())); }
private static void serializeNTriples(final RdfStream rdfStream, final RDFFormat format, final OutputStream output) { final StreamRDF stream = new SynchonizedStreamRDFWrapper(getWriterStream(output, format)); stream.start(); rdfStream.forEach(stream::triple); stream.finish(); }
private static void serializeBlockStreamed(final RdfStream rdfStream, final OutputStream output, final RDFFormat format, final Map<String, String> nsPrefixes) { final Set<String> namespacesPresent = new HashSet<>(); final StreamRDF stream = new SynchonizedStreamRDFWrapper(getWriterStream(output, format)); stream.start(); // Must read the rdf stream before writing out ns prefixes, otherwise the prefixes come after the triples final List<Triple> tripleList = rdfStream.peek(t -> { // Collect the namespaces present in the RDF stream, using the same // criteria for where to look that jena's model.listNameSpaces() does namespacesPresent.add(t.getPredicate().getNameSpace()); if (RDF_TYPE.equals(t.getPredicate().getURI()) && t.getObject().isURI()) { namespacesPresent.add(t.getObject().getNameSpace()); } }).collect(Collectors.toList()); nsPrefixes.forEach((prefix, uri) -> { // Only add namespace prefixes if the namespace is present in the rdf stream if (namespacesPresent.contains(uri)) { stream.prefix(prefix, uri); } }); tripleList.forEach(stream::triple); stream.finish(); }