/** Loads elements from a file given by a pathname, storing them in a given array. * * @param filename a filename. * @param array an array which will be filled with data from the specified file. * @return the number of elements actually read from the given file (it might be less than the array length if the file is too short). */ public static int loadBytes(final CharSequence filename, final byte[] array) throws IOException { return loadBytes(new File(filename.toString()), array); } /** Loads elements from a file given by a {@link File} object, storing them in a new array.
/** Loads elements from a file given by a pathname, storing them in a given big-array fragment. * * @param filename a filename. * @param array an array which will be filled with data from the specified file. * @param offset the index of the first element of {@code array} to be filled. * @param length the number of elements of {@code array} to be filled. * @return the number of elements actually read from the given file (it might be less than {@code length} if the file is too short). */ public static long loadBytes(final CharSequence filename, final byte[][] array, final long offset, final long length) throws IOException { return loadBytes(new File(filename.toString()), array, offset, length); } /** Loads elements from a file given by a {@link File} object, storing them in a given big array.
/** Loads elements from a file given by a pathname, storing them in a given array fragment. * * @param filename a filename. * @param array an array which will be filled with data from the specified file. * @param offset the index of the first element of {@code array} to be filled. * @param length the number of elements of {@code array} to be filled. * @return the number of elements actually read from the given file (it might be less than {@code length} if the file is too short). */ public static int loadBytes(final CharSequence filename, final byte[] array, final int offset, final int length) throws IOException { return loadBytes(new File(filename.toString()), array, offset, length); } /** Loads elements from a file given by a {@link File} object, storing them in a given array.
/** Loads elements from a file given by a pathname, storing them in a given big array. * * @param filename a filename. * @param array a big array which will be filled with data from the specified file. * @return the number of elements actually read from the given file (it might be less than the array length if the file is too short). */ public static long loadBytes(final CharSequence filename, final byte[][] array) throws IOException { return loadBytes(new File(filename.toString()), array); } /** Loads elements from a file given by a {@link File} object, storing them in a new big array.
/** Creates a new multi-array input stream loading it from an input stream. * * @param is the input stream that will fill the array. * @param size the number of bytes to be read from {@code is}. */ public FastMultiByteArrayInputStream(final InputStream is, long size) throws IOException { length = size; array = new byte[(int)((size + SLICE_SIZE - 1) / SLICE_SIZE) + 1][]; for(int i = 0; i < array.length - 1; i++) { array[i] = new byte[size >= SLICE_SIZE ? SLICE_SIZE : (int)size]; // It is important *not* to use is.read() directly because of bug #6478546 if (BinIO.loadBytes(is, array[i]) != array[i].length) throw new EOFException(); size -= array[i].length; } current = array[0]; }
/** Loads elements from a file given by a filename, storing them in a new array. * * <p>Note that the length of the returned array will be computed * dividing the specified file size by the number of bytes used to * represent each element. * * @param filename a filename. * @return an array filled with the content of the specified file. */ public static byte[] loadBytes(final CharSequence filename) throws IOException { return loadBytes(new File(filename.toString())); } /** Stores an array fragment to a given data output.
private void write( final int b ) { if ( avail == 0 ) { final int oldLength = buffer.length; try { buffer = ByteArrays.grow( buffer, buffer.length + 1 ); } catch( OutOfMemoryError e ) { outOfMemoryError = true; try { // We try at all costs to avoid out-of-memory errors: we dump the buffer, try to allocate a slightly larger array and reload it. File temp = File.createTempFile( ByteArrayPostingList.class.getSimpleName(), "dump" ); temp.deleteOnExit(); BinIO.storeBytes( buffer, temp ); buffer = null; buffer = new byte[ oldLength + POSTINGS_EMERGENCY_INCREMENT ]; BinIO.loadBytes( temp, buffer ); temp.delete(); } catch ( IOException f ) { throw new RuntimeException( f ); } } avail += buffer.length - oldLength; } avail--; buffer[ pos++ ] = (byte)b; }
private void write( final int b ) { if ( avail == 0 ) { final int oldLength = buffer.length; try { buffer = ByteArrays.grow( buffer, buffer.length + 1 ); } catch( OutOfMemoryError e ) { outOfMemoryError = true; try { // We try at all costs to avoid out-of-memory errors: we dump the buffer, try to allocate a slightly larger array and reload it. File temp = File.createTempFile( ByteArrayPostingList.class.getSimpleName(), "dump" ); temp.deleteOnExit(); BinIO.storeBytes( buffer, temp ); buffer = null; buffer = new byte[ oldLength + POSTINGS_EMERGENCY_INCREMENT ]; BinIO.loadBytes( temp, buffer ); temp.delete(); } catch ( IOException f ) { throw new RuntimeException( f ); } } avail += buffer.length - oldLength; } avail--; buffer[ pos++ ] = (byte)b; }
private void write( final int b ) { if ( avail == 0 ) { final int oldLength = buffer.length; try { buffer = ByteArrays.grow( buffer, buffer.length + 1 ); } catch( OutOfMemoryError e ) { outOfMemoryError = true; try { // We try at all costs to avoid out-of-memory errors: we dump the buffer, try to allocate a slightly larger array and reload it. File temp = File.createTempFile( ByteArrayPostingList.class.getSimpleName(), "dump" ); temp.deleteOnExit(); BinIO.storeBytes( buffer, temp ); buffer = null; buffer = new byte[ oldLength + POSTINGS_EMERGENCY_INCREMENT ]; BinIO.loadBytes( temp, buffer ); temp.delete(); } catch ( IOException f ) { throw new RuntimeException( f ); } } avail += buffer.length - oldLength; } avail--; buffer[ pos++ ] = (byte)b; }
private void write( final int b ) { if ( avail == 0 ) { final int oldLength = buffer.length; try { buffer = ByteArrays.grow( buffer, buffer.length + 1 ); } catch( OutOfMemoryError e ) { outOfMemoryError = true; try { // We try at all costs to avoid out-of-memory errors: we dump the buffer, try to allocate a slightly larger array and reload it. File temp = File.createTempFile( ByteArrayPostingList.class.getSimpleName(), "dump" ); temp.deleteOnExit(); BinIO.storeBytes( buffer, temp ); buffer = null; buffer = new byte[ oldLength + POSTINGS_EMERGENCY_INCREMENT ]; BinIO.loadBytes( temp, buffer ); temp.delete(); } catch ( IOException f ) { throw new RuntimeException( f ); } } avail += buffer.length - oldLength; } avail--; buffer[ pos++ ] = (byte)b; }
/** Utility method to load a compressed size file into an {@linkplain EliasFanoLongBigList Elias–Fano compressed list}. * * @param filename the filename containing the γ-coded sizes (see {@link BitStreamIndexWriter}). * @param N the number of documents indexed. * @return a list of integers backed by an {@linkplain EliasFanoLongBigList Elias–Fano compressed list}. * @throws IllegalStateException if <code>ioFactory</code> is not {@link IOFactory#FILESYSTEM_FACTORY}. * * @deprecated This method is an ancestral residue. */ @Deprecated public static IntList readSizesSuccinct( final CharSequence filename, final int N ) throws IOException { LOGGER.debug( "Loading sizes..." ); final IntList sizes = new AbstractIntList() { final EliasFanoLongBigList list = new EliasFanoLongBigList( new GammaCodedIterableList( BinIO.loadBytes( filename ), N ) ); public int getInt( int index ) { return (int)list.getLong( index ); } public int size() { return (int)list.size64(); } }; LOGGER.debug( "Completed." ); return sizes; }
/** Utility method to load a compressed size file into an {@linkplain EliasFanoLongBigList Elias–Fano compressed list}. * * @param filename the filename containing the γ-coded sizes (see {@link BitStreamIndexWriter}). * @param N the number of documents indexed. * @return a list of integers backed by an {@linkplain EliasFanoLongBigList Elias–Fano compressed list}. * @throws IllegalStateException if <code>ioFactory</code> is not {@link IOFactory#FILESYSTEM_FACTORY}. * * @deprecated This method is an ancestral residue. */ @Deprecated public static IntBigList readSizesSuccinct( final CharSequence filename, final long N ) throws IOException { LOGGER.debug( "Loading sizes..." ); final IntBigList sizes = new AbstractIntBigList() { final EliasFanoLongBigList list = new EliasFanoLongBigList( new GammaCodedIterableList( BinIO.loadBytes( filename ), N ) ); public int getInt( long index ) { return (int)list.getLong( index ); } public long size64() { return list.size64(); } }; LOGGER.debug( "Completed." ); return sizes; }
/** Utility method to load a compressed size file into an {@linkplain EliasFanoLongBigList Elias–Fano compressed list}. * * @param filename the filename containing the γ-coded sizes (see {@link BitStreamIndexWriter}). * @param N the number of documents indexed. * @return a list of integers backed by an {@linkplain EliasFanoLongBigList Elias–Fano compressed list}. * @throws IOException */ public static IntList readSizesSuccinct( final CharSequence filename, final int N ) throws IOException { LOGGER.debug( "Loading sizes..." ); final IntList sizes = new AbstractIntList() { final EliasFanoLongBigList list = new EliasFanoLongBigList( new GammaCodedIterableList( BinIO.loadBytes( filename ), N ) ); public int getInt( int index ) { return (int)list.getLong( index ); } public int size() { return (int)list.size64(); } }; LOGGER.debug( "Completed." ); return sizes; }
/** Utility method to load a compressed size file into an {@linkplain EliasFanoLongBigList Elias–Fano compressed list}. * * @param filename the filename containing the γ-coded sizes (see {@link BitStreamIndexWriter}). * @param N the number of documents indexed. * @return a list of integers backed by an {@linkplain EliasFanoLongBigList Elias–Fano compressed list}. * @throws IOException */ public static IntBigList readSizesSuccinct( final CharSequence filename, final long N ) throws IOException { LOGGER.debug( "Loading sizes..." ); final IntBigList sizes = new AbstractIntBigList() { final EliasFanoLongBigList list = new EliasFanoLongBigList( new GammaCodedIterableList( BinIO.loadBytes( filename ), N ) ); public int getInt( long index ) { return (int)list.getLong( index ); } public long size64() { return list.size64(); } }; LOGGER.debug( "Completed." ); return sizes; }
BinIO.loadBytes(fis, graphMemory); fis.close(); isMemory = true;
if (size <= Integer.MAX_VALUE) byteArray = BinIO.loadBytes(basename + LABELS_EXTENSION); else labelStream = new FastMultiByteArrayInputStream(fis, size);
? new InMemoryHPIndex( BinIO.loadBytes( indexFile.toString() ), BinIO.loadBytes( basename + POSITIONS_EXTENSION ), numberOfDocuments, numberOfTerms, numberOfPostings, numberOfOccurrences, maxCount, payload, frequencyCoding, pointerCoding, countCoding, positionCoding, quantum, height, termProcessor, field, properties, termMap, prefixMap, sizes, offsets ) : new InMemoryIndex( BinIO.loadBytes( indexFile.toString() ), numberOfDocuments, numberOfTerms, numberOfPostings, numberOfOccurrences, maxCount, payload, frequencyCoding, pointerCoding, countCoding, positionCoding, quantum, height,
? new InMemoryHPIndex( BinIO.loadBytes( indexFile.toString() ), BinIO.loadBytes( basename + POSITIONS_EXTENSION ), numberOfDocuments, numberOfTerms, numberOfPostings, numberOfOccurrences, maxCount, payload, frequencyCoding, pointerCoding, countCoding, positionCoding, quantum, height, termProcessor, field, properties, termMap, prefixMap, sizes, offsets ) : new InMemoryIndex( BinIO.loadBytes( indexFile.toString() ), numberOfDocuments, numberOfTerms, numberOfPostings, numberOfOccurrences, maxCount, payload, frequencyCoding, pointerCoding, countCoding, positionCoding, quantum, height,