static void extractNMessages(String filein, int n, String fileout) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(filein, "r"); FileOutputStream fos = new FileOutputStream(fileout); WritableByteChannel wbc = fos.getChannel()) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext() && (count < n)) { Message m = scan.next(); scan.writeCurrentMessage(wbc); count++; } } }
public void execute(String filename) throws IOException { try (RandomAccessFile mraf = new RandomAccessFile(filename, "r")) { MessageScanner scanner = new MessageScanner(mraf); while (scanner.hasNext()) { Message m = scanner.next(); if (m == null) continue; total_msgs++; if (m.getNumberDatasets() == 0) continue; // LOOK check on tables complete etc ?? m.setRawBytes(scanner.getMessageBytes(m)); // decide what to do with the message dispatcher.dispatch(m); } dispatcher.resetBufrTableMessages(); } }
public static void main(String arg[]) throws Exception { //String filename = "C:/temp/cache/uniqueMessages.bufr"; String filename = "C:/data/formats/bufr/uniqueExamples.bufr"; int size = 0; int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r"); OutputStream out = new FileOutputStream("C:/data/formats/bufr/uniqueE/" + count + ".xml")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message message = scan.next(); if (message == null || !message.isTablesComplete() || !message.isBitCountOk()) continue; byte[] mbytes = scan.getMessageBytesFromLast(message); NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp"); NetcdfDataset ncd = new NetcdfDataset(ncfile); new Bufr2Xml(message, ncd, out, true); out.close(); count++; size += message.getMessageSize(); } } catch (Throwable e) { e.printStackTrace(); } System.out.printf("total size= %f Kb %n", .001 * size); } }
static private TableB readEmbeddedTableB(String location) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(location, "r")) { MessageScanner scan = new MessageScanner(raf); TableLookup lookup = scan.getTableLookup(); if (lookup != null) { return lookup.getLocalTableB(); } return null; } }
public void scanBufrFile(String filename) throws Exception { int count = 0; RandomAccessFile raf = null; try { raf = new RandomAccessFile(filename, "r"); MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; out.format("%sMessage %d header=%s%n", indent, count++, m.getHeader()); processBufrMessageAsDataset(scan, m); } } finally { if (raf != null) raf.close(); } }
@Override public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException { super.open(raf, ncfile, cancelTask); scanner = new MessageScanner(raf); protoMessage = scanner.getFirstDataMessage(); if (protoMessage == null) throw new IOException("No data messages in the file= "+ncfile.getLocation()); // DataDescriptor dds = protoMessage.getRootDataDescriptor(); // construct the data descriptors, check for complete tables if (!protoMessage.isTablesComplete()) throw new IllegalStateException("BUFR file has incomplete tables"); // just get the fields config = BufrConfig.openFromMessage(raf, protoMessage, iospParam); // this fills the netcdf object Construct2 construct = new Construct2(protoMessage, config, ncfile); obsStructure = construct.getObsStructure(); ncfile.finish(); isSingle = false; }
private void processBufrMessageAsDataset(MessageScanner scan, Message m, Counter counter) throws Exception { byte[] mbytes = scan.getMessageBytes(m); NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp"); Sequence obs = (Sequence) ncfile.findVariable(BufrIosp2.obsRecord); StructureDataIterator sdataIter = obs.getStructureIterator(-1); processSequence(obs, sdataIter, counter); }
@Override public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) throws IOException { return MessageScanner.isValidFile(raf); }
log.warn("Illegal edition - BUFR message at pos " + start + " header= " + cleanup(header)); return null; log.warn("Edition "+ is.getBufrEdition()+" is not supported - BUFR message at pos " + start + " header= " +cleanup(header)); return null; for (int i = 0; i < 3; i++) { if (raf.read() != 55) { log.warn("Missing End of BUFR message at pos=" + ending + " header= " + cleanup(header)); return null; raf.seek(dataPos + dataLength-1); // see if byte before is a '7' if (raf.read() != 55) { log.warn("Missing End of BUFR message at pos=" +ending+ " header= " + cleanup(header)+" edition= "+is.getBufrEdition()); return null; } else { log.warn("End of BUFR message off-by-one at pos= " +ending+ " header= " + cleanup(header)+" edition= "+is.getBufrEdition()); lastPos--; m.setHeader( cleanup(header)); m.setStartPos( start);
public int scanBufrFile(String filename, Counter total) throws Exception { int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; try { if (showMess) out.format("%sMessage %d header=%s%n", indent, count, m.getHeader()); count++; Counter counter = new Counter(); processBufrMessageAsDataset(scan, m, counter); if (showMess) out.format("%scount=%d miss=%d%n", indent, counter.nvals, counter.nmiss); total.add(counter); } catch (Exception e) { System.out.printf(" BARF:%s on %s%n", e.getMessage(), m.getHeader()); indent.setIndentLevel(0); } } } return count; }
static private TableD readEmbeddedTableD(String location) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(location, "r")) { MessageScanner scan = new MessageScanner(raf); TableLookup lookup = scan.getTableLookup(); if (lookup != null) { return lookup.getLocalTableD(); } return null; } }
private void scanBufrFile(RandomAccessFile raf) throws Exception { NetcdfFile ncd = null; countObs = 0; try { MessageScanner scanner = new MessageScanner(raf); Message protoMessage = scanner.getFirstDataMessage(); if (protoMessage == null) throw new IOException("No message found!"); messHash = protoMessage.hashCode(); standardFields = StandardFields.extract(protoMessage); rootConverter = new FieldConverter(protoMessage.ids.getCenterId(), protoMessage.getRootDataDescriptor()); if (standardFields.hasStation()) { hasStations = true; map = new HashMap<>(1000); } featureType = guessFeatureType(standardFields); hasDate = standardFields.hasTime(); //ncd = NetcdfDataset.openDataset(raf.getLocation(), BufrIosp2.enhance, -1, null, null); // LOOK opening another raf ncd = NetcdfFile.open(raf.getLocation()); // LOOK opening another raf Attribute centerAtt = ncd.findGlobalAttribute(BufrIosp2.centerId); int center = (centerAtt == null) ? 0 : centerAtt.getNumericValue().intValue(); Sequence seq = (Sequence) ncd.findVariable(null, BufrIosp2.obsRecord); extract = new StandardFields.StandardFieldsFromStructure(center, seq); StructureDataIterator iter = seq.getStructureIterator(); processSeq( iter, rootConverter, true); setStandardActions(rootConverter); } finally { if (ncd != null) ncd.close(); } System.out.printf("nobs = %d%n", countObs); }
private void processBufrMessageAsDataset(MessageScanner scan, Message m, Counter counter) throws Exception { byte[] mbytes = scan.getMessageBytes(m); NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp"); Sequence obs = (Sequence) ncfile.findVariable(BufrIosp.obsRecord); StructureDataIterator sdataIter = obs.getStructureIterator(-1); processSequence(obs, sdataIter, counter); }
public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) throws IOException { return MessageScanner.isValidFile(raf); }
log.warn("Illegal edition - BUFR message at pos " + start + " header= " + cleanup(header)); return null; log.warn("Edition "+ is.getBufrEdition()+" is not supported - BUFR message at pos " + start + " header= " +cleanup(header)); return null; for (int i = 0; i < 3; i++) { if (raf.read() != 55) { log.warn("Missing End of BUFR message at pos= {} header= {} file= {}", ending, cleanup(header), raf.getLocation()); return null; raf.seek(dataPos + dataLength-1); // see if byte before is a '7' if (raf.read() != 55) { log.warn("Missing End of BUFR message at pos= {} header= {} edition=() file= {}", ending, cleanup(header), is.getBufrEdition(), raf.getLocation()); return null; } else { log.info("End of BUFR message off-by-one at pos= {} header= {} edition=() file= {}", ending, cleanup(header), is.getBufrEdition(), raf.getLocation()); lastPos--; m.setHeader( cleanup(header)); m.setStartPos( start);
static void writeUniqueDDS(String filename, WritableByteChannel wbc) throws IOException { System.out.printf("open %s ",filename); try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) { bad_msgs++; System.out.printf("Bad Message%n"); continue; } if (!messSet.contains(m)) { scan.writeCurrentMessage(wbc); messSet.add(m); } count++; } System.out.printf(" read = %d%n ", count); } }
public int scanBufrFile(String filename, Counter total) throws Exception { int count = 0; RandomAccessFile raf = null; try { raf = new RandomAccessFile(filename, "r"); MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; try { if (showMess) out.format("%sMessage %d header=%s%n", indent, count, m.getHeader()); count++; Counter counter = new Counter(); processBufrMessageAsDataset(scan, m, counter); if (showMess) out.format("%scount=%d miss=%d%n", indent, counter.nvals, counter.nmiss); total.add(counter); } catch (Exception e) { System.out.printf(" BARF:%s on %s%n", e.getMessage(), m.getHeader()); indent.setIndentLevel(0); } } } finally { if (raf != null) raf.close(); } return count; }
private boolean scanBufrFile(String filename) throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException { int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; byte[] mbytes = scan.getMessageBytes(m); try( NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp2")) { NetcdfDataset ncd = new NetcdfDataset(ncfile); } } } return true; }