/** * <!-- <init>(Learner,String,boolean) --> Creates a new trainer that doesn't produce status * messages. * * @param l The learner to be trained. * @param p The path to an example file. * @param z Whether or not the example file is compressed. **/ public BatchTrainer(Learner l, String p, boolean z) { this(l, new ArrayFileParser(p, z)); }
/** * Initializes the parser with a data array, specifying whether the data is zipped. * * @param data The examples can be parsed out of this array. * @param zip Whether or not the input stream is zipped. **/ public ArrayFileParser(byte[] data, boolean zip) { exampleData = data; zipped = zip; reset(); }
+ "feature pre-extraction has already been performed."); ArrayFileParser afp = (ArrayFileParser) parser; afp.setIncludePruned(true); for (Object example = afp.next(); example != null; example = afp.next()) { if (progressOutput > 0 && examples % progressOutput == 0) System.out.println(" " + learner.name + ", pruning: " + examples + " examples at " parser = new ArrayFileParser(fTempFile.getPath(), zip); else parser = new ArrayFileParser(baos.toByteArray(), zip);
public static void main(String[] args) { String exFileName = null; String lexFileName = null; String lcFileName = null; try { exFileName = args[0]; lexFileName = args[1]; lcFileName = args[2]; if (args.length > 3) throw new Exception(); } catch (Exception e) { System.err .println("usage: java edu.illinois.cs.cogcomp.lbjava.parse.ArrayFileParser <example file> <lexicon file> <lc file>"); System.exit(1); } ArrayFileParser parser = new ArrayFileParser(exFileName); Learner learner = Learner.readLearner(lcFileName); learner.readLexicon(lexFileName); for (Object e = parser.next(); e != null; e = parser.next()) { FeatureVector v = new FeatureVector((Object[]) e, learner.getLexicon(), learner.getLabelLexicon()); v.sort(); System.out.println(v); } } }
((ArrayFileParser) parser).setIncludePruned(true); parser = new ArrayFileParser(fTempFile.getPath(), zip); else parser = new ArrayFileParser(baos.toByteArray(), zip);
afp.setIncludePruned(true); result = metric.test(testLearner, null, testParser); afp.setIncludePruned(false); } else { setIsTraining(false);
/** * <!-- fillInSizes() --> This method sets the {@link #examples} and {@link #lexiconSize} * variables by querying {@link #parser} and {@link #learner} respectively. It sets * {@link #examples} to 0 if {@link #parser} is not an * {@link edu.illinois.cs.cogcomp.lbjava.parse.ArrayFileParser} and {@link #lexiconSize} to 0 if * {@link #learner} doesn't either have the lexicon loaded or know where to find it. **/ public void fillInSizes() { if (parser instanceof ArrayFileParser) { ArrayFileParser afp = (ArrayFileParser) parser; examples = afp.getNumExamples(); } else examples = 0; lexiconSize = learner.getPrunedLexiconSize(); }
/** Resets the example file stream to the beginning. */ public void reset() { close(); try { if (exampleFileName != null) { if (zipped) { ZipFile zip = new ZipFile(exampleFileName); in = new DataInputStream(new BufferedInputStream(zip.getInputStream(zip .getEntry(ExceptionlessInputStream.zipEntryName)))); } else in = new DataInputStream(new BufferedInputStream(new FileInputStream( exampleFileName))); } else if (zipped) { ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(exampleData)); zip.getNextEntry(); in = new DataInputStream(new BufferedInputStream(zip)); } else in = new DataInputStream(new ByteArrayInputStream(exampleData)); } catch (Exception e) { System.err.println("Can't open '" + exampleFileName + "' for input:"); e.printStackTrace(); System.exit(1); } }
afp.setIncludePruned(true); result = metric.test(testLearner, null, foldParser); afp.setIncludePruned(false); } else { setIsTraining(false);
/** * <!-- <init>(Learner,String,boolean,int) --> Creates a new trainer that produces status * messages. * * @param l The learner to be trained. * @param p The path to an example file. * @param z Whether or not the example file is compressed. * @param o The number of examples in between status messages on STDOUT. **/ public BatchTrainer(Learner l, String p, boolean z, int o) { this(l, new ArrayFileParser(p, z), o); }
/** * Initializes the parser with a file name, specifying whether the data is zipped. * * @param exampleFile The name of the file containing the examples. * @param zip Whether or not the input stream is zipped. **/ public ArrayFileParser(String exampleFile, boolean zip) { exampleFileName = exampleFile; zipped = zip; reset(); }
/** * <!-- <init>(Learner,String,boolean,int,String) --> Creates a new trainer that produces status * messages with the specified indentation spacing for status messages. * * @param l The learner to be trained. * @param p The path to an example file. * @param z Whether or not the example file is compressed. * @param o The number of examples in between status messages on STDOUT. * @param i The indentation spacing for status messages. **/ public BatchTrainer(Learner l, String p, boolean z, int o, String i) { this(l, new ArrayFileParser(p, z), o, i); }
/** * Returns the number of examples left in the example file. This may be slow to compute as it * must read through the entire file and increment the count. {@link #reset()} is called after * the examples are counted. * * @return The number of examples left in the example file. **/ public int getNumExamples() { int result = 0; try { while (true) { int L = in.readInt(); if (L == -1) continue; ++result; in.skipBytes(12 * L); // 4 for label index, 8 for its value L = in.readInt() + in.readInt(); in.skipBytes(12 * L); // 4 for feature index, 8 for its value } } catch (EOFException eof) { } catch (Exception e) { System.err.println("Can't read from '" + exampleFileName + "':"); e.printStackTrace(); System.exit(1); } reset(); return result; }
new ArrayFileParser(exFilePath, lce.preExtract.value.endsWith("Zip\"")); if (lce.pruneStatus != RevisionAnalysis.UNAFFECTED) learner.readLexiconOnDemand(classDir + getName() + ".lex"); new ArrayFileParser(testExFilePath, lce.preExtract.value.endsWith("Zip\"")); else