/** * Private constructor that opens a new JobHistoryFijiTable, creating it if necessary. * This method also updates an existing layout to the latest layout for the job * history table. * * @param fiji The fiji instance to retrieve the job history table from. * @throws IOException If there's an error opening the underlying HBaseFijiTable. */ private JobHistoryFijiTable(Fiji fiji) throws IOException { install(fiji); mFijiTable = fiji.openTable(TABLE_NAME); }
/** * Opens a connection. * @param fijiURI The fijiURI * @throws IOException If there is a connection error. */ public Connection(FijiURI fijiURI) throws IOException { final Configuration conf = HBaseConfiguration.create(); mFiji = Fiji.Factory.open(fijiURI); mFijiTable = mFiji.openTable(fijiURI.getTable()); }
/** {@inheritDoc} */ @Override public final FijiTable openResource() throws Exception { if (null != mTableURI) { return new GetTable(mTableURI).eval(); } else { return mFiji.openTable(mTableName); } } }
/** {@inheritDoc} */ @Override protected void setup() throws IOException { mFiji = Fiji.Factory.open(mTableURI, getConf()); mTable = mFiji.openTable(mTableURI.getTable()); }
/** {@inheritDoc} */ @Override protected void setup(Context hadoopContext) throws IOException, InterruptedException { super.setup(hadoopContext); final Configuration conf = hadoopContext.getConfiguration(); FijiURI tableURI; try { tableURI = FijiURI.newBuilder(conf.get(FijiConfKeys.OUTPUT_FIJI_TABLE_URI)).build(); } catch (FijiURIException kue) { throw new IOException(kue); } mFiji = Fiji.Factory.open(tableURI, conf); mTable = mFiji.openTable(TABLE_NAME); mWriter = mTable.openTableWriter(); }
/** {@inheritDoc} */ @Override protected void setup(Context hadoopContext) throws IOException, InterruptedException { super.setup(hadoopContext); final Configuration conf = hadoopContext.getConfiguration(); FijiURI tableURI; try { tableURI = FijiURI.newBuilder(conf.get(FijiConfKeys.OUTPUT_FIJI_TABLE_URI)).build(); } catch (FijiURIException kue) { throw new IOException(kue); } mFiji = Fiji.Factory.open(tableURI, conf); mTable = mFiji.openTable(TABLE_NAME); mTableWriter = mTable.openTableWriter(); }
/** {@inheritDoc} */ @Override protected void setup(Context hadoopContext) throws IOException, InterruptedException { super.setup(hadoopContext); final Configuration conf = hadoopContext.getConfiguration(); FijiURI tableURI; try { tableURI = FijiURI.newBuilder(conf.get(FijiConfKeys.OUTPUT_FIJI_TABLE_URI)).build(); } catch (FijiURIException kue) { throw new IOException(kue); } mFiji = Fiji.Factory.open(tableURI, hadoopContext.getConfiguration()); mTable = mFiji.openTable(tableURI.getTable()); mWriter = mTable.openTableWriter(); }
/** * Initializes a new table-wide record writer. * * @param oformat FijiHFileOutputFormat this writer is built from. * @param context Context of the task. * @throws IOException on I/O error. */ public TableRecordWriter(FijiHFileOutputFormat oformat, TaskAttemptContext context) throws IOException { mContext = Preconditions.checkNotNull(context); mConf = mContext.getConfiguration(); mLatestTimestamp = mConf.getLong(CONF_LATEST_TIMESTAMP, System.currentTimeMillis()); mLatestTimestampBytes = toBytes(mLatestTimestamp); mOutputDir = oformat.getDefaultWorkFile(mContext, OUTPUT_EXTENSION); mFileSystem = mOutputDir.getFileSystem(mConf); mTableURI = FijiURI.newBuilder(mConf.get(FijiConfKeys.FIJI_OUTPUT_TABLE_URI)).build(); final Fiji fiji = Fiji.Factory.open(mTableURI, mConf); final FijiTable table = fiji.openTable(mTableURI.getTable()); mLayout = table.getLayout(); ResourceUtils.releaseOrLog(table); ResourceUtils.releaseOrLog(fiji); }
/** {@inheritDoc} */ @Override protected void setup() throws IOException { Preconditions.checkArgument((mColumnURIFlag != null) && !mColumnURIFlag.isEmpty(), "Specify a target table to write synthesized data to with " + "--table=fiji://hbase-address/fiji-instance/table"); mColumnURI = FijiURI.newBuilder(mColumnURIFlag).build(); Preconditions.checkArgument(mColumnURI.getTable() != null, "No table specified in target URI '{}'. " + "Specify a target table to write synthesized data to with " + "--table=fiji://hbase-address/fiji-instance/table", mColumnURI); Preconditions.checkArgument(mColumnURI.getColumns().size() == 1, "Invalid target column '{}', specify exactly one column in URI with " + "--target=fiji://hbase-address/fiji-instance/table/family:qualifier", mColumnURI); Preconditions.checkArgument(mColumnURI.getColumns().get(0).isFullyQualified(), "Missing column qualifier in '{}', specify exactly one column in URI with " + "--target=fiji://hbase-address/fiji-instance/table/family:qualifier", mColumnURI); mFiji = Fiji.Factory.open(mColumnURI, getConf()); mTable = mFiji.openTable(mColumnURI.getTable()); }
/** * Constructor. * * @param conf The job configuration. * @throws java.io.IOException If the input split cannot be opened. */ public FijiTableRecordWriter(Configuration conf) throws IOException { String fijiURIString = conf.get(FijiTableOutputFormat.CONF_FIJI_TABLE_URI); FijiURI fijiURI = FijiURI.newBuilder(fijiURIString).build(); mFiji = Fiji.Factory.open(fijiURI); mFijiTable = mFiji.openTable(fijiURI.getTable()); mFijiTableWriter = mFijiTable.openTableWriter(); }
@Override protected int run(List<String> nonFlagArgs) throws Exception { Fiji fiji = null; FijiTable fijiTable = null; FijiSchemaTable schemaTable = null; try { fiji = Fiji.Factory.open(mTableURI); fijiTable = fiji.openTable(mTableURI.getTable()); FijiTableLayout fijiTableLayout = fijiTable.getLayout(); schemaTable = fiji.getSchemaTable(); getPrintStream().println(generateHiveDDLStatement(mTableURI, fijiTableLayout, schemaTable)); } catch (IOException ioe) { LOG.warn(ioe.getMessage()); return FAILURE; } finally { if (null != fiji) { ResourceUtils.releaseOrLog(fiji); } if (null != fijiTable) { ResourceUtils.releaseOrLog(fijiTable); } } return SUCCESS; } }
/** {@inheritDoc} */ @Override protected void setup() throws Exception { super.setup(); mFiji = Fiji.Factory.open(mOutput.getOutputTableURI(), getConf()); mTable = mFiji.openTable(mOutput.getOutputTableURI().getTable()); }
/** {@inheritDoc} */ @Override protected void setup() throws Exception { super.setup(); mFiji = Fiji.Factory.open(mOutput.getOutputTableURI(), getConf()); mTable = mFiji.openTable(mOutput.getOutputTableURI().getTable()); }
/** {@inheritDoc} */ @Override public List<InputSplit> getSplits(JobContext context) throws IOException { final Configuration conf = context.getConfiguration(); final FijiURI inputTableURI = getInputTableURI(conf); final Fiji fiji = Fiji.Factory.open(inputTableURI, conf); final FijiTable table = fiji.openTable(inputTableURI.getTable()); final HTableInterface htable = HBaseFijiTable.downcast(table).openHTableConnection(); try { final List<InputSplit> splits = Lists.newArrayList(); for (FijiRegion region : table.getRegions()) { final byte[] startKey = region.getStartKey(); // TODO: a smart way to get which location is most relevant. final String location = region.getLocations().isEmpty() ? null : region.getLocations().iterator().next(); final TableSplit tableSplit = new TableSplit( htable.getTableName(), startKey, region.getEndKey(), location); splits.add(new FijiTableSplit(tableSplit, startKey)); } return splits; } finally { htable.close(); } }
/** {@inheritDoc} */ @Override protected void configureJob(Job job) throws IOException { // Configure the input, mapper, combiner, and reducer, output. super.configureJob(job); // Validate the Fiji data request against the current table layout: Preconditions.checkNotNull(mInputTableURI, "Input Fiji table was never set."); final Fiji fiji = Fiji.Factory.open(mInputTableURI, getConf()); try { final FijiTable table = fiji.openTable(mInputTableURI.getTable()); try { validateInputTable(table); } finally { ResourceUtils.releaseOrLog(table); } } finally { ResourceUtils.releaseOrLog(fiji); } }
/** * Constructs a new context that can write cells directly to a Fiji table. * * @param hadoopContext is the Hadoop {@link TaskInputOutputContext} that will be used to perform * the writes. * @throws IOException on I/O error. */ public DirectFijiTableWriterContext(TaskInputOutputContext<?, ?, ?, ?> hadoopContext) throws IOException { super(hadoopContext); final Configuration conf = new Configuration(hadoopContext.getConfiguration()); final FijiURI outputURI = FijiURI.newBuilder(conf.get(FijiConfKeys.FIJI_OUTPUT_TABLE_URI)).build(); mFiji = Fiji.Factory.open(outputURI, conf); mTable = mFiji.openTable(outputURI.getTable()); mWriter = mTable.getWriterFactory().openBufferedWriter(); mEntityIdFactory = EntityIdFactory.getFactory(mTable.getLayout()); }
/** * Constructs a new context that can write cells to an HFile that can be loaded into an HBase * table. * * @param hadoopContext is the Hadoop {@link TaskInputOutputContext} that will be used to perform * the writes. * @throws IOException on I/O error. */ public HFileWriterContext(TaskInputOutputContext<?, ?, ?, ?> hadoopContext) throws IOException { super(hadoopContext); final Configuration conf = new Configuration(hadoopContext.getConfiguration()); final FijiURI outputURI = FijiURI.newBuilder(conf.get(FijiConfKeys.FIJI_OUTPUT_TABLE_URI)).build(); mFiji = Fiji.Factory.open(outputURI, conf); mTable = mFiji.openTable(outputURI.getTable()); mReader = mTable.openTableReader(); mColumnNameTranslator = HBaseColumnNameTranslator.from(mTable.getLayout()); mEntityIdFactory = EntityIdFactory.getFactory(mTable.getLayout()); }
/** {@inheritDoc} */ @Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException { assert split instanceof FijiTableSplit; mSplit = (FijiTableSplit) split; final Configuration conf = context.getConfiguration(); final FijiURI inputURI = getInputTableURI(conf); mFiji = Fiji.Factory.open(inputURI, conf); mTable = mFiji.openTable(inputURI.getTable()); mReader = mTable.openTableReader(); final FijiScannerOptions scannerOptions = new FijiScannerOptions() .setStartRow(HBaseEntityId.fromHBaseRowKey(mSplit.getStartRow())) .setStopRow(HBaseEntityId.fromHBaseRowKey(mSplit.getEndRow())); mScanner = mReader.getScanner(mDataRequest, scannerOptions); mIterator = mScanner.iterator(); mCurrentRow = null; }
/** {@inheritDoc} */ @Override protected int run(final List<String> nonFlagArgs) throws Exception { final Fiji fiji = Fiji.Factory.open(mURI); try { final FijiTable table = fiji.openTable(mURI.getTable()); try { mAnnotator = table.openTableAnnotator(); try { switch (mDoMode) { case SET: return set(); case REMOVE: return remove(); case GET: return get(); default: throw new InternalFijiError("Unknown NotesTool DoMode: " + mDoMode); } } finally { mAnnotator.close(); } } finally { table.release(); } } finally { fiji.release(); } } }
/** {@inheritDoc} */ @Override protected void configure(B jobBuilder) throws ClassNotFoundException, IOException { // Basic job configuration (base JobConf, jars and KV stores): super.configure(jobBuilder); // Configure job input: jobBuilder.withJobInput(mJobInput); final Fiji fiji = Fiji.Factory.open(mJobInput.getInputTableURI(), getConf()); try { final FijiTable table = fiji.openTable(mJobInput.getInputTableURI().getTable()); try { if (mStartRowFlag != null) { final EntityId eidLimit = ToolUtils.createEntityIdFromUserInputs(mStartRowFlag, table.getLayout()); jobBuilder.withStartRow(eidLimit); } if (mLimitRowFlag != null) { final EntityId eidLimit = ToolUtils.createEntityIdFromUserInputs(mLimitRowFlag, table.getLayout()); jobBuilder.withLimitRow(eidLimit); } } finally { ResourceUtils.releaseOrLog(table); } } finally { ResourceUtils.releaseOrLog(fiji); } }