public static void validatePut(Put put, int maxKeyValueSize) throws IllegalArgumentException { if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert"); } if (maxKeyValueSize > 0) { for (List<Cell> list : put.getFamilyCellMap().values()) { for (Cell cell : list) { if (cell.getSerializedSize() > maxKeyValueSize) { throw new IllegalArgumentException("KeyValue size too large"); } } } } }
/** * Submit immediately the list of rows, whatever the server status. Kept for backward * compatibility: it allows to be used with the batch interface that return an array of objects. * @param task The setting and data */ private <CResult> AsyncRequestFuture submitAll(AsyncProcessTask task) { RowAccess<? extends Row> rows = task.getRowAccess(); List<Action> actions = new ArrayList<>(rows.size()); // The position will be used by the processBatch to match the object array returned. int posInList = -1; NonceGenerator ng = this.connection.getNonceGenerator(); int highestPriority = HConstants.PRIORITY_UNSET; for (Row r : rows) { posInList++; if (r instanceof Put) { Put put = (Put) r; if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert for #" + (posInList+1)+ " item"); } highestPriority = Math.max(put.getPriority(), highestPriority); } Action action = new Action(r, posInList, highestPriority); setNonce(ng, r, action); actions.add(action); } AsyncRequestFutureImpl<CResult> ars = createAsyncRequestFuture(task, actions, ng.getNonceGroup()); ars.groupAndSendMultiAction(actions, 1); return ars; }
deletes.add(delete); if (!put.isEmpty()) { puts.add(put);
if (!put.isEmpty()) { mutator.mutate(put);
table.delete(delete); if (!put.isEmpty()) { table.put(put);
public static void validatePut(Put put, int maxKeyValueSize) throws IllegalArgumentException { if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert"); } if (maxKeyValueSize > 0) { for (List<Cell> list : put.getFamilyCellMap().values()) { for (Cell cell : list) { if (KeyValueUtil.length(cell) > maxKeyValueSize) { throw new IllegalArgumentException("KeyValue size too large"); } } } } }
/** * Submit immediately the list of rows, whatever the server status. Kept for backward * compatibility: it allows to be used with the batch interface that return an array of objects. * @param task The setting and data */ private <CResult> AsyncRequestFuture submitAll(AsyncProcessTask task) { RowAccess<? extends Row> rows = task.getRowAccess(); List<Action> actions = new ArrayList<>(rows.size()); // The position will be used by the processBatch to match the object array returned. int posInList = -1; NonceGenerator ng = this.connection.getNonceGenerator(); int highestPriority = HConstants.PRIORITY_UNSET; for (Row r : rows) { posInList++; if (r instanceof Put) { Put put = (Put) r; if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert for #" + (posInList+1)+ " item"); } highestPriority = Math.max(put.getPriority(), highestPriority); } Action action = new Action(r, posInList, highestPriority); setNonce(ng, r, action); actions.add(action); } AsyncRequestFutureImpl<CResult> ars = createAsyncRequestFuture(task, actions, ng.getNonceGroup()); ars.groupAndSendMultiAction(actions, 1); return ars; }
@Override public boolean isEmpty() { return put.isEmpty(); }
@Override public boolean isEmpty() { return put.isEmpty(); }
@Override public List<Put> apply(List<Variant> list) throws IOException { List<Put> puts = new ArrayList<>(list.size()); for (Variant variant : list) { Put put = fillGaps(variant); if (put != null && !put.isEmpty()) { puts.add(put); } } return puts; }
public static void validatePut(Put put, int maxKeyValueSize) throws IllegalArgumentException { if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert"); } if (maxKeyValueSize > 0) { for (List<Cell> list : put.getFamilyCellMap().values()) { for (Cell cell : list) { if (KeyValueUtil.length(cell) > maxKeyValueSize) { throw new IllegalArgumentException("KeyValue size too large"); } } } } }
private void validatePut(final Put put) throws IllegalArgumentException{ if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert"); } if (maxKeyValueSize > 0) { for (List<KeyValue> list : put.getFamilyMap().values()) { for (KeyValue kv : list) { if (kv.getLength() > maxKeyValueSize) { throw new IllegalArgumentException("KeyValue size too large"); } } } } }
public static void validatePut(Put put, int maxKeyValueSize) throws IllegalArgumentException { if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert"); } if (maxKeyValueSize > 0) { for (List<Cell> list : put.getFamilyCellMap().values()) { for (Cell cell : list) { if (KeyValueUtil.length(cell) > maxKeyValueSize) { throw new IllegalArgumentException("KeyValue size too large"); } } } } }
protected List<Put> getPuts(int remain) { List<Put> puts = new LinkedList<>(); while (buffer.size() > remain) { IndexChunk indexChunk = buffer.keySet().iterator().next(); List<Map<String, Set<String>>> sampleList = buffer.remove(indexChunk); Iterator<Integer> sampleIterator = sampleIds.iterator(); for (Map<String, Set<String>> gtsMap : sampleList) { Integer sampleId = sampleIterator.next(); byte[] rk = SampleIndexConverter.toRowKey(sampleId, indexChunk.chromosome, indexChunk.position); Put put = new Put(rk); for (Map.Entry<String, Set<String>> gtsEntry : gtsMap.entrySet()) { put.addColumn(family, toGenotypeColumn(gtsEntry.getKey()), Bytes.toBytes(String.join(",", gtsEntry.getValue()))); put.addColumn(family, toGenotypeCountColumn(gtsEntry.getKey()), Bytes.toBytes(gtsEntry.getValue().size())); } if (!put.isEmpty()) { puts.add(put); } } } return puts; }
@Override protected void map(Object key, Variant variant, Context context) throws IOException, InterruptedException { Put put = fillGapsTask.fillGaps(variant); if (put != null && !put.isEmpty()) { context.write(new ImmutableBytesWritable(put.getRow()), put); } } }
@Override public void process(Row value) { Object rowkey = value.getAs(rowkeyIndex); if (rowkey == null) { return; } Put put = new Put(BytesUtil.toBytes(rowkey)); try { for (String fieldName : schema.getFieldNames()) { if (!config.rowkey.equals(fieldName)) { Tuple2<String, String> tuple2 = columMapping.get(fieldName); if (tuple2 != null) { hbaseHelper.addColumn(tuple2.f0(), tuple2.f1(), value.getAs(fieldName), put); } else { logger.warn("Field:" + fieldName + " not defined in table " + tableName); } } } if (!put.isEmpty()) { hbaseHelper.store(put); } } catch (Exception e) { logger.error("put record to hbase fail.", e); } }
if (r instanceof Put) { Put put = (Put) r; if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert for #" + (posInList+1)+ " item");
/** * Submit immediately the list of rows, whatever the server status. Kept for backward * compatibility: it allows to be used with the batch interface that return an array of objects. * @param task The setting and data */ private <CResult> AsyncRequestFuture submitAll(AsyncProcessTask task) { RowAccess<? extends Row> rows = task.getRowAccess(); List<Action> actions = new ArrayList<>(rows.size()); // The position will be used by the processBatch to match the object array returned. int posInList = -1; NonceGenerator ng = this.connection.getNonceGenerator(); int highestPriority = HConstants.PRIORITY_UNSET; for (Row r : rows) { posInList++; if (r instanceof Put) { Put put = (Put) r; if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert for #" + (posInList+1)+ " item"); } highestPriority = Math.max(put.getPriority(), highestPriority); } Action action = new Action(r, posInList, highestPriority); setNonce(ng, r, action); actions.add(action); } AsyncRequestFutureImpl<CResult> ars = createAsyncRequestFuture(task, actions, ng.getNonceGroup()); ars.groupAndSendMultiAction(actions, 1); return ars; }
@Override protected void adaptMutations(Put operation, com.google.cloud.bigtable.data.v2.models.MutationApi<?> mutation) { if (operation.isEmpty()) { throw new IllegalArgumentException("No columns to insert");
public static void fillGapsLocal(HadoopVariantStorageEngine variantStorageEngine, StudyConfiguration studyConfiguration, Collection<Integer> sampleIds) throws StorageEngineException, IOException { VariantHadoopDBAdaptor dbAdaptor = variantStorageEngine.getDBAdaptor(); String variantTableName = variantStorageEngine.getVariantTableName(); Table variantsTable = dbAdaptor.getHBaseManager().getConnection().getTable(TableName.valueOf(variantTableName)); FillGapsFromVariantTask fillGapsTask = new FillGapsFromVariantTask(dbAdaptor.getHBaseManager(), variantStorageEngine.getArchiveTableName(studyConfiguration.getStudyId()), studyConfiguration, dbAdaptor.getGenomeHelper(), sampleIds); fillGapsTask.pre(); ProgressLogger progressLogger = new ProgressLogger("Fill gaps:", dbAdaptor.count(new Query()).first(), 10); for (Variant variant : dbAdaptor) { progressLogger.increment(1, variant::toString); Put put = fillGapsTask.fillGaps(variant); if (put != null && !put.isEmpty()) { variantsTable.put(put); } } variantsTable.close(); fillGapsTask.post(); }