@Override public void close() throws IOException { closed = true; out.close(); if (count != countWritten) { throw new IllegalStateException("wrote " + countWritten + " values, but expected " + count); } }
@Override public void close() throws IOException { closed = true; out.close(); if (count != countWritten) { throw new IllegalStateException("wrote " + countWritten + " values, but expected " + count); } }
@Override public void close() throws IOException { closed = true; out.close(); if (count != countWritten) { throw new IllegalStateException("wrote " + countWritten + " values, but expected " + count); } }
protected URI createFst(SortedSet<String> values) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { FST fst = DatawaveFieldIndexListIteratorJexl.getFST(values); // now serialize to our file system CompressionCodec codec = null; String extension = ""; if (config.getHdfsFileCompressionCodec() != null) { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { classLoader = this.getClass().getClassLoader(); } Class<? extends CompressionCodec> clazz = Class.forName(config.getHdfsFileCompressionCodec(), true, classLoader).asSubclass(CompressionCodec.class); codec = clazz.newInstance(); extension = codec.getDefaultExtension(); } int fstCount = config.getFstCount().incrementAndGet(); Path fstFile = new Path(fstHdfsUri, "PushdownLargeFileFst." + fstCount + ".fst" + extension); OutputStream fstFileOut = new BufferedOutputStream(fs.create(fstFile, false)); if (codec != null) { fstFileOut = codec.createOutputStream(fstFileOut); } OutputStreamDataOutput outStream = new OutputStreamDataOutput(fstFileOut); fst.save(outStream); outStream.close(); return fstFile.toUri(); }
@Override public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { if (weight < -1 || weight > Integer.MAX_VALUE) { throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE"); } for (int i = 0; i < surfaceForm.length; i++) { if (surfaceForm.bytes[i] == UNIT_SEPARATOR) { throw new IllegalArgumentException( "surface form cannot contain unit separator character U+001F; this character is reserved"); } } ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream); output.writeVLong(weight + 1); output.writeVInt(surfaceForm.length); output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); output.writeVInt(payload.length); output.writeBytes(payload.bytes, 0, payload.length); output.close(); return new BytesRef(byteArrayOutputStream.toByteArray()); }
@Override public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { if (weight < -1 || weight > Integer.MAX_VALUE) { throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE"); } for (int i = 0; i < surfaceForm.length; i++) { if (surfaceForm.bytes[i] == UNIT_SEPARATOR) { throw new IllegalArgumentException( "surface form cannot contain unit separator character U+001F; this character is reserved"); } } ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream); output.writeVLong(weight + 1); output.writeVInt(surfaceForm.length); output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); output.writeVInt(payload.length); output.writeBytes(payload.bytes, 0, payload.length); output.close(); return new BytesRef(byteArrayOutputStream.toByteArray()); }
@Override public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { if (weight < -1 || weight > Integer.MAX_VALUE) { throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE"); } for (int i = 0; i < surfaceForm.length; i++) { if (surfaceForm.bytes[i] == UNIT_SEPARATOR) { throw new IllegalArgumentException( "surface form cannot contain unit separator character U+001F; this character is reserved"); } } ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream); output.writeVLong(weight + 1); output.writeVInt(surfaceForm.length); output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); output.writeVInt(payload.length); output.writeBytes(payload.bytes, 0, payload.length); output.close(); return new BytesRef(byteArrayOutputStream.toByteArray()); }
private static void serialize(String file, BytesRef [] all) throws IOException { final Object nothing = NoOutputs.getSingleton().getNoOutput(); final Builder<Object> builder = new Builder<Object>(INPUT_TYPE.BYTE4, NoOutputs.getSingleton()); final IntsRefBuilder intsRef = new IntsRefBuilder(); for (BytesRef br : all) { intsRef.clear(); intsRef.copyUTF8Bytes(br); builder.add(intsRef.get(), nothing); } final FST<Object> fst = builder.finish(); final OutputStreamDataOutput out = new OutputStreamDataOutput(new FileOutputStream(file)); fst.save(out); out.close(); } }