protected Tokenizer makeTokenizer(InputStream in) { Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ; return tokenizer ; }
public BindingInputStream(InputStream in) { this(TokenizerFactory.makeTokenizerUTF8(in)) ; }
protected Tokenizer makeTokenizer(InputStream in) { Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ; return tokenizer ; }
public BindingInputStream(InputStream in) { this(TokenizerFactory.makeTokenizerUTF8(in)) ; }
protected Tokenizer makeTokenizer(InputStream in) { Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ; return tokenizer ; }
/** Create an iterator for parsing N-Quads. */ public static Iterator<Quad> createIteratorNQuads(InputStream input, StreamRDF dest, ParserProfile profile) { // LangNQuads supports iterator use. Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input); return createParserNQuads(tokenizer, null, profile); } }
/** Create an iterator for parsing N-Triples. */ public static Iterator<Triple> createIteratorNTriples(InputStream input, StreamRDF dest, ParserProfile profile) { // LangNTriples supports iterator use. Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input); return createParserNTriples(tokenizer, null, profile); }
/** Create a parser for Turtle * @deprecated use an RDFDataMgr operation with argument Lang.Turtle */ @Deprecated public static LangTurtle createParserTurtle(InputStream input, String baseIRI, StreamRDF dest) { Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input) ; return createParserTurtle(tokenizer, baseIRI, dest) ; }
/** Create a parser for a quads (or triples) language */ public static LangRIOT createParserQuads(InputStream input, Lang lang, String baseIRI, StreamRDF dest) { Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input) ; return createParserQuads(tokenizer, lang, baseIRI, dest) ; }
/** Create a parser for Turtle, with default behaviour */ public static LangTurtle createParserTurtle(InputStream input, String baseIRI, StreamRDF dest) { Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input) ; return createParserTurtle(tokenizer, baseIRI, dest) ; }
/** Create a parser for TriG, with default behaviour */ public static LangTriG createParserTriG(InputStream input, String baseIRI, StreamRDF dest) { Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input) ; return createParserTriG(tokenizer, baseIRI, dest) ; }
/** Create a parser for TriG * @deprecated use an RDFDataMgr operation with argument Lang.TRIG */ @Deprecated public static LangTriG createParserTriG(InputStream input, String baseIRI, StreamRDF dest) { Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input) ; return createParserTriG(tokenizer, baseIRI, dest) ; }
private /*public*/ IteratorStreamRDFText(InputStream input) { Tokenizer t = TokenizerFactory.makeTokenizerUTF8(input) ; in = new TokenInputStream(null, t) ; }
static protected Tokenizer tokenizer(String string) { // UTF-8 byte b[] = StrUtils.asUTF8bytes(string); ByteArrayInputStream in = new ByteArrayInputStream(b); Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in); return tokenizer; }
Tokenizer tokenize = TokenizerFactory.makeTokenizerUTF8(in) ; Timer timer = new Timer() ; long count = 0 ;
static protected Tokenizer tokenizer(CharSpace charSpace, String string) { byte b[] = StrUtils.asUTF8bytes(string); ByteArrayInputStream in = new ByteArrayInputStream(b); Tokenizer tokenizer = charSpace == CharSpace.ASCII ? TokenizerFactory.makeTokenizerASCII(in) : TokenizerFactory.makeTokenizerUTF8(in); return tokenizer; }
/** Create a parser */ public static LangRIOT createParser(InputStream input, Lang lang, String baseIRI, StreamRDF dest) { if ( lang == RDFXML ) { if ( baseIRI != null ) baseIRI = IRIResolver.resolveString(baseIRI) ; return LangRDFXML.create(input, baseIRI, baseIRI, ErrorHandlerFactory.getDefaultErrorHandler(), dest) ; } Tokenizer tokenizer = ( lang == RDFJSON ) ? new TokenizerJSON(PeekReader.makeUTF8(input)) : TokenizerFactory.makeTokenizerUTF8(input) ; return createParser(tokenizer, lang, baseIRI, dest) ; }
@Test(expected=IllegalArgumentException.class) public void rdfjson_invalid_tokenizer() { byte b[] = StrUtils.asUTF8bytes("") ; ByteArrayInputStream in = new ByteArrayInputStream(b); Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ; StreamRDFCounting sink = StreamRDFLib.count() ; LangRDFJSON parser = RiotParsers.createParserRdfJson(tokenizer, sink, RiotLib.dftProfile()) ; }
/** InputStream input */ public static LangRIOT createParser(InputStream input, Lang lang, StreamRDF dest, ParserProfile profile) { if ( RDFLanguages.sameLang(RDFJSON, lang) ) { Tokenizer tokenizer = new TokenizerJSON(PeekReader.makeUTF8(input)); return createParserRdfJson(tokenizer, dest, profile); } Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input); if ( RDFLanguages.sameLang(TURTLE, lang) || RDFLanguages.sameLang(N3, lang) ) return createParserTurtle(tokenizer, dest, profile); if ( RDFLanguages.sameLang(NTRIPLES, lang) ) return createParserNTriples(tokenizer, dest, profile); if ( RDFLanguages.sameLang(NQUADS, lang) ) return createParserNQuads(tokenizer, dest, profile); if ( RDFLanguages.sameLang(TRIG, lang) ) return createParserTriG(tokenizer, dest, profile); return null; }
@Test public void tokenizer_BOM_1() { // BOM ByteArrayInputStream in = bytes("\uFEFF'abc'") ; Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ; assertTrue(tokenizer.hasNext()) ; Token token = tokenizer.next() ; assertNotNull(token) ; assertEquals(TokenType.STRING, token.getType()) ; assertEquals("abc", token.getImage()) ; assertFalse(tokenizer.hasNext()) ; }