/** * Handles initializing this class with objects specific to it (i.e., the parser). * Common initialization that might be leveraged by a subclass is done in * <code>doSetup</code>. Hence a subclass may choose to override this method * and call <code>doSetup</code> as well before handling it's own custom params. * * @param context */ @Override protected void setup(Context context) { doSetup(context); Configuration conf = context.getConfiguration(); parser = new ImportTsv.TsvParser(conf.get(ImportTsv.COLUMNS_CONF_KEY), separator); if (parser.getRowKeyColumnIndex() == -1) { throw new RuntimeException("No row key column specified"); } }
/** * Convert a line of TSV text into an HBase table row. */ @Override public void map(LongWritable offset, Text value, Context context) throws IOException { try { Pair<Integer,Integer> rowKeyOffests = parser.parseRowKey(value.getBytes(), value.getLength()); ImmutableBytesWritable rowKey = new ImmutableBytesWritable( value.getBytes(), rowKeyOffests.getFirst(), rowKeyOffests.getSecond()); context.write(rowKey, value); } catch (ImportTsv.TsvParser.BadTsvLineException|IllegalArgumentException badLine) { if (logBadLines) { System.err.println(value); } System.err.println("Bad line at offset: " + offset.get() + ":\n" + badLine.getMessage()); if (skipBadLines) { incrementBadLineCount(1); return; } throw new IOException(badLine); } catch (InterruptedException e) { LOG.error("Interrupted while emitting TSV text", e); Thread.currentThread().interrupt(); } } }
/** * Handles initializing this class with objects specific to it (i.e., the parser). * Common initialization that might be leveraged by a subclass is done in * <code>doSetup</code>. Hence a subclass may choose to override this method * and call <code>doSetup</code> as well before handling it's own custom params. * * @param context */ @Override protected void setup(Context context) { doSetup(context); Configuration conf = context.getConfiguration(); parser = new ImportTsv.TsvParser(conf.get(ImportTsv.COLUMNS_CONF_KEY), separator); if (parser.getRowKeyColumnIndex() == -1) { throw new RuntimeException("No row key column specified"); } }
/** * Convert a line of TSV text into an HBase table row. */ @Override public void map(LongWritable offset, Text value, Context context) throws IOException { try { Pair<Integer,Integer> rowKeyOffests = parser.parseRowKey(value.getBytes(), value.getLength()); ImmutableBytesWritable rowKey = new ImmutableBytesWritable( value.getBytes(), rowKeyOffests.getFirst(), rowKeyOffests.getSecond()); context.write(rowKey, value); } catch (ImportTsv.TsvParser.BadTsvLineException|IllegalArgumentException badLine) { if (logBadLines) { System.err.println(value); } System.err.println("Bad line at offset: " + offset.get() + ":\n" + badLine.getMessage()); if (skipBadLines) { incrementBadLineCount(1); return; } throw new IOException(badLine); } catch (InterruptedException e) { e.printStackTrace(); Thread.currentThread().interrupt(); } } }
/** * Handles initializing this class with objects specific to it (i.e., the parser). * Common initialization that might be leveraged by a subsclass is done in * <code>doSetup</code>. Hence a subclass may choose to override this method * and call <code>doSetup</code> as well before handling it's own custom params. * * @param context */ @Override protected void setup(Context context) { doSetup(context); Configuration conf = context.getConfiguration(); parser = new ImportTsv.TsvParser(conf.get(ImportTsv.COLUMNS_CONF_KEY), separator); if (parser.getRowKeyColumnIndex() == -1) { throw new RuntimeException("No row key column specified"); } }
/** * Convert a line of TSV text into an HBase table row. */ @Override public void map(LongWritable offset, Text value, Context context) throws IOException { try { Pair<Integer,Integer> rowKeyOffests = parser.parseRowKey(value.getBytes(), value.getLength()); ImmutableBytesWritable rowKey = new ImmutableBytesWritable( value.getBytes(), rowKeyOffests.getFirst(), rowKeyOffests.getSecond()); context.write(rowKey, value); } catch (ImportTsv.TsvParser.BadTsvLineException|IllegalArgumentException badLine) { if (logBadLines) { System.err.println(value); } System.err.println("Bad line at offset: " + offset.get() + ":\n" + badLine.getMessage()); if (skipBadLines) { incrementBadLineCount(1); return; } throw new IOException(badLine); } catch (InterruptedException e) { e.printStackTrace(); Thread.currentThread().interrupt(); } } }
/** * Handles initializing this class with objects specific to it (i.e., the parser). * Common initialization that might be leveraged by a subsclass is done in * <code>doSetup</code>. Hence a subclass may choose to override this method * and call <code>doSetup</code> as well before handling it's own custom params. * * @param context */ @Override protected void setup(Context context) { doSetup(context); Configuration conf = context.getConfiguration(); parser = new ImportTsv.TsvParser(conf.get(ImportTsv.COLUMNS_CONF_KEY), separator); if (parser.getRowKeyColumnIndex() == -1) { throw new RuntimeException("No row key column specified"); } }
if (skipBadLines) { System.err.println("Bad line at offset: " + offset.get() + ":\n" + badLine.getMessage()); incrementBadLineCount(1); return; if (skipBadLines) { System.err.println("Bad line at offset: " + offset.get() + ":\n" + e.getMessage()); incrementBadLineCount(1); return; } else {