private static void runCsvBench(String type, int numRecords, Times times) throws IOException { Record[] records = makeArray(type, numRecords, times); ByteArrayOutputStream bout = new ByteArrayOutputStream(); CsvRecordOutput rout = new CsvRecordOutput(bout); for(int idx = 0; idx < numRecords; idx++) { records[idx].serialize(rout); } bout.reset(); times.serialize = System.nanoTime(); for(int idx = 0; idx < numRecords; idx++) { records[idx].serialize(rout); } times.serialize = System.nanoTime() - times.serialize; byte[] serialized = bout.toByteArray(); ByteArrayInputStream bin = new ByteArrayInputStream(serialized); CsvRecordInput rin = new CsvRecordInput(bin); times.deserialize = System.nanoTime(); for(int idx = 0; idx < numRecords; idx++) { records[idx].deserialize(rin); } times.deserialize = System.nanoTime() - times.deserialize; }
CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))); KBEntry<Integer,Integer> entry = deserialiseCsvRecord(cri); if (entry != null) {
CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))); KBEntry<String,Record> entry = deserialiseCsvRecord(cri); if (entry != null) {
CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))); try { KBEntry<Integer,Record> entry = deserialiseCsvRecord(cri);
CsvRecordInput linksInRecord = new CsvRecordInput(new ByteArrayInputStream((inLinkLine + "\n").getBytes("UTF-8"))) ; WEntry<Integer, DbLinkLocationList> inLinkEntry = deserializePageLinkCsvRecord(linksInRecord) ; CsvRecordInput linksOutRecord = new CsvRecordInput(new ByteArrayInputStream((outLinkLine + "\n").getBytes("UTF-8"))) ; WEntry<Integer, DbLinkLocationList> outLinkEntry = deserializePageLinkCsvRecord(linksOutRecord) ; if (inLinkLine != null) { bytesRead += (inLinkLine.length() + 1) ; linksInRecord = new CsvRecordInput(new ByteArrayInputStream((inLinkLine + "\n").getBytes("UTF-8"))) ; inLinkEntry = deserializePageLinkCsvRecord(linksInRecord) ; } else { if (outLinkLine != null) { bytesRead += (outLinkLine.length() + 1) ; linksOutRecord = new CsvRecordInput(new ByteArrayInputStream((outLinkLine + "\n").getBytes("UTF-8"))) ; outLinkEntry = deserializePageLinkCsvRecord(linksOutRecord) ; } else {
lineNum++ ; CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))) ;
CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))); KBEntry<Integer,Long> entry = deserialiseCsvRecord(cri); if (entry != null) {
tx = environment.createWriteTransaction(); CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))); KBEntry<String,Integer> entry = deserialiseCsvRecord(cri); if (entry != null) {
CsvRecordInput linksInRecord = new CsvRecordInput(new ByteArrayInputStream((inLinkLine + "\n").getBytes("UTF-8"))); KBEntry<Integer, DbLinkLocationList> inLinkEntry = deserializePageLinkCsvRecord(linksInRecord); CsvRecordInput linksOutRecord = new CsvRecordInput(new ByteArrayInputStream((outLinkLine + "\n").getBytes("UTF-8"))); KBEntry<Integer, DbLinkLocationList> outLinkEntry = deserializePageLinkCsvRecord(linksOutRecord); inLinkLine = linksInInput.readLine(); if (inLinkLine != null) { linksInRecord = new CsvRecordInput(new ByteArrayInputStream((inLinkLine + "\n").getBytes("UTF-8"))); inLinkEntry = deserializePageLinkCsvRecord(linksInRecord); } else { outLinkLine = linksOutInput.readLine(); if (outLinkLine != null) { linksOutRecord = new CsvRecordInput(new ByteArrayInputStream((outLinkLine + "\n").getBytes("UTF-8"))); outLinkEntry = deserializePageLinkCsvRecord(linksOutRecord); } else {
CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))); KBEntry<String,Integer> entry = deserialiseCsvRecord(cri);
tx = environment.createWriteTransaction(); CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))); KBEntry<Integer,DbIntList> entry = deserialiseCsvRecord(cri); try {
tx = environment.createWriteTransaction(); CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))); try { KBEntry<Integer,DbPage> entry = deserialiseCsvRecord(cri);
@Override public void loadFromCsvFile(File dataFile, boolean overwrite, ProgressTracker tracker) throws IOException { if (exists() && !overwrite) return ; if (tracker == null) tracker = new ProgressTracker(1, WDatabase.class) ; tracker.startTask(dataFile.length(), "Loading " + getName()) ; BufferedReader input = new BufferedReader(new InputStreamReader(new FileInputStream(dataFile), "UTF-8")) ; long bytesRead = 0 ; int lineNum = 0 ; Database db = getDatabase(false) ; String line ; while ((line=input.readLine()) != null) { bytesRead = bytesRead + line.length() + 1 ; lineNum++ ; CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))) ; WEntry<Integer,DbIntList> entry = deserialiseCsvRecord(cri) ; if (entry != null) { DatabaseEntry k = new DatabaseEntry() ; keyBinding.objectToEntry(entry.getKey(), k) ; DatabaseEntry v = new DatabaseEntry() ; valueBinding.objectToEntry(entry.getValue(), v) ; db.put(null, k, v) ; } tracker.update(bytesRead) ; } input.close(); env.cleanAndCheckpoint() ; getDatabase(true) ; }
lineNum++ ; CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))) ;
CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream(line.getBytes("UTF8"))) ; currKeys[pass] = cri.readString(null) ; currValues[pass] = new DbLabel() ; CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream(line.getBytes("UTF8"))) ; currKeys[pass] = cri.readString(null) ; currValues[pass] = new DbLabel() ;
public void testCsv() { File tmpfile; try { tmpfile = new File("/temp/hadooptmp.txt"); RecRecord1 r1 = new RecRecord1(); r1.setBoolVal(true); r1.setByteVal((byte)0x66); r1.setFloatVal(3.145F); r1.setDoubleVal(1.5234); r1.setIntVal(4567); r1.setLongVal(0x5a5a5a5a5a5aL); r1.setStringVal("random text"); r1.setBufferVal(new Buffer()); r1.setVectorVal(new ArrayList<String>()); r1.setMapVal(new TreeMap<String,String>()); FileInputStream istream = new FileInputStream(tmpfile); CsvRecordInput in = new CsvRecordInput(istream); RecRecord1 r2 = new RecRecord1(); r2.deserialize(in, ""); istream.close(); assertTrue(r1.equals(r2)); } catch (IOException ex) { ex.printStackTrace(); } }
ostream.close(); FileInputStream istream = new FileInputStream(tmpfile); CsvRecordInput in = new CsvRecordInput(istream); RecRecord1 r2 = new RecRecord1(); r2.deserialize(in, "");
lineNum++ ; CsvRecordInput cri = new CsvRecordInput(new ByteArrayInputStream((line + "\n").getBytes("UTF-8"))) ;