/** * Perform any required validation on the table being bulk loaded into: * - ensure no column family names start with '_', as they'd be ignored leading to problems. * @throws SQLException */ private static void validateTable() throws SQLException { Connection conn = DriverManager.getConnection(getUrl()); try { ResultSet rs = conn.getMetaData().getColumns(null, StringUtil.escapeLike(schemaName), StringUtil.escapeLike(tableName), null); while (rs.next()) { String familyName = rs.getString(1); if (familyName != null && familyName.startsWith(UNDERSCORE)) { String msg; if (QueryConstants.DEFAULT_COLUMN_FAMILY.equals(familyName)) { msg = "CSV Bulk Loader error: All column names that are not part of the primary key constraint must be prefixed with a column family name (i.e. f.my_column VARCHAR)"; } else { msg = "CSV Bulk Loader error: Column family name must not start with '_': " + familyName; } throw new SQLException(msg); } } } finally{ conn.close(); } }
private void appendKeyRanges(StringBuilder buf) { ScanRanges scanRanges = context.getScanRanges(); KeyRange minMaxRange = context.getMinMaxRange(); if (minMaxRange == null && (scanRanges == ScanRanges.EVERYTHING || scanRanges == ScanRanges.NOTHING)) { return; } buf.append(" ["); StringBuilder buf1 = new StringBuilder(); appendScanRow(buf1, Bound.LOWER); buf.append(buf1); buf.setCharAt(buf.length()-1, ']'); StringBuilder buf2 = new StringBuilder(); appendScanRow(buf2, Bound.UPPER); if (!StringUtil.equals(buf1, buf2)) { buf.append( " - ["); buf.append(buf2); } buf.setCharAt(buf.length()-1, ']'); } }
public static int getByteLengthForUtf8SubStr(byte[] bytes, int offset, int length, ColumnModifier columnModifier) throws UnsupportedEncodingException { int byteLength = 0; while(length > 0 && offset + byteLength < bytes.length) { int charLength = getBytesInChar(bytes[offset + byteLength], columnModifier); byteLength += charLength; length--; } return byteLength; }
if (type == PDataType.VARCHAR || type == PDataType.CHAR) { if (type == PDataType.CHAR && maxLength != null && b.length < maxLength) { b = StringUtil.padChar(b, maxLength); } else if (value != null) { maxLength = ((String)value).length();
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { if (!getStringExpression().evaluate(tuple, ptr)) { return false; } if (ptr.getLength() == 0) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); return true; } byte[] string = ptr.get(); int offset = ptr.getOffset(); int length = ptr.getLength(); ColumnModifier columnModifier = getColumnModifier(); int end = StringUtil.getFirstNonBlankCharIdxFromEnd(string, offset, length, columnModifier); if (end == offset - 1) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); return true; } int head = StringUtil.getFirstNonBlankCharIdxFromStart(string, offset, length, columnModifier); ptr.set(string, head, end - head + 1); return true; }
boolean isCharType = getStrExpression().getDataType() == PDataType.CHAR; ColumnModifier columnModifier = getStrExpression().getColumnModifier(); int strlen = isCharType ? ptr.getLength() : StringUtil.calculateUTF8Length(ptr.get(), ptr.getOffset(), ptr.getLength(), columnModifier); length = length == -1 ? maxLength : Math.min(length,maxLength); int byteOffset = isCharType ? offset : StringUtil.getByteLengthForUtf8SubStr(ptr.get(), ptr.getOffset(), offset, columnModifier); int byteLength = isCharType ? length : StringUtil.getByteLengthForUtf8SubStr(ptr.get(), ptr.getOffset() + byteOffset, length, columnModifier); ptr.set(ptr.get(), ptr.getOffset() + byteOffset, byteLength); return true;
public static int getUnpaddedCharLength(byte[] b, int offset, int length, ColumnModifier columnModifier) { return getFirstNonBlankCharIdxFromEnd(b, offset, length, columnModifier) - offset + 1; }
@Override public Object toObject(byte[] bytes, int offset, int length, PDataType actualType) { if (!actualType.isCoercibleTo(this)) { // TODO: have isCoercibleTo that takes bytes, offset? throw new ConstraintViolationException(actualType + " cannot be coerced to " + this); } if (length == 0) { return null; } length = StringUtil.getUnpaddedCharLength(bytes, offset, length, null); String s = Bytes.toString(bytes, offset, length); if (length != s.length()) { throw new IllegalDataException("CHAR types may only contain single byte characters (" + s + ")"); } return s; }
@Override public Object toObject(String value) { if (value == null || value.length() == 0) { throw new ConstraintViolationException(this + " may not be null"); } if (StringUtil.hasMultiByteChars(value)) { throw new IllegalDataException("CHAR types may only contain single byte characters (" + value + ")"); } return value; }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { // Starting from the front of the byte, look for all single bytes at the end of the string // that is below SPACE_UTF8 (space and control characters) or 0x7f (control chars). if (!getStringExpression().evaluate(tuple, ptr)) { return false; } if (ptr.getLength() == 0) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); return true; } byte[] string = ptr.get(); int offset = ptr.getOffset(); int length = ptr.getLength(); ColumnModifier columnModifier = getStringExpression().getColumnModifier(); // TODO: when we have ColumnModifier.REVERSE, we'll need to trim from the end instead of // the beginning (just delegate to RTrimFunction or replace from ExpressionCompiler instead?) int i = StringUtil.getFirstNonBlankCharIdxFromStart(string, offset, length, columnModifier); if (i == offset + length) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); return true; } ptr.set(string, i, offset + length - i); return true; }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { Expression child = getStringExpression(); if (!child.evaluate(tuple, ptr)) { return false; } if (ptr.getLength() == 0) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); return true; } int len; if (child.getDataType() == PDataType.CHAR) { // Only single-byte characters allowed in CHAR len = ptr.getLength(); } else { try { len = StringUtil.calculateUTF8Length(ptr.get(), ptr.getOffset(), ptr.getLength(), child.getColumnModifier()); } catch (UnsupportedEncodingException e) { return false; } } ptr.set(PDataType.INTEGER.toBytes(len)); return true; }
@Test public void testPartialRangeFilter() throws SQLException { // I know these id's are ridiculous, but users can write queries that look like this String tenantId1 = "001"; String tenantId2 = "02"; String query = String.format("select * from %s where organization_id > '%s' AND organization_id < '%s'", ATABLE_NAME, tenantId1, tenantId2); PhoenixConnection pconn = DriverManager.getConnection(getUrl(), TEST_PROPERTIES).unwrap(PhoenixConnection.class); PhoenixPreparedStatement pstmt = new PhoenixPreparedStatement(pconn, query); QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); assertNull(scan.getFilter()); byte[] wideLower = ByteUtil.nextKey(StringUtil.padChar(Bytes.toBytes(tenantId1), 15)); byte[] wideUpper = StringUtil.padChar(Bytes.toBytes(tenantId2), 15); assertArrayEquals(wideLower, scan.getStartRow()); assertArrayEquals(wideUpper, scan.getStopRow()); }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { // Starting from the end of the byte, look for all single bytes at the end of the string // that is below SPACE_UTF8 (space and control characters) or above (control chars). if (!getStringExpression().evaluate(tuple, ptr)) { return false; } if (ptr.getLength() == 0) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); return true; } byte[] string = ptr.get(); int offset = ptr.getOffset(); int length = ptr.getLength(); ColumnModifier columnModifier = getStringExpression().getColumnModifier(); int i = StringUtil.getFirstNonBlankCharIdxFromEnd(string, offset, length, columnModifier); if (i == offset - 1) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); return true; } ptr.set(string, offset, i - offset + 1); return true; }
ColumnModifier rhsColumnModifier = children.get(1).getColumnModifier(); if (rhsDataType == PDataType.CHAR) { rhsLength = StringUtil.getUnpaddedCharLength(rhsBytes, rhsOffset, rhsLength, rhsColumnModifier); lhsLength = StringUtil.getUnpaddedCharLength(lhsBytes, lhsOffset, lhsLength, lhsColumnModifier);
s = StringUtil.escapeLike(s); StringBuilder buf = new StringBuilder();
@Test public void testPaddedStartStopKey() throws SQLException { String tenantId = "000000000000001"; String keyPrefix = "fo"; String query = "select * from atable where organization_id=? AND entity_id=?"; List<Object> binds = Arrays.<Object>asList(tenantId,keyPrefix); PhoenixConnection pconn = DriverManager.getConnection(getUrl(), TEST_PROPERTIES).unwrap(PhoenixConnection.class); PhoenixPreparedStatement pstmt = new PhoenixPreparedStatement(pconn, query); bindParams(pstmt, binds); QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); assertArrayEquals(ByteUtil.concat(Bytes.toBytes(tenantId), StringUtil.padChar(Bytes.toBytes(keyPrefix), 15)),scan.getStartRow()); assertArrayEquals(ByteUtil.nextKey(scan.getStartRow()),scan.getStopRow()); }
public static int calculateUTF8Length(byte[] bytes, int offset, int length, ColumnModifier columnModifier) throws UnsupportedEncodingException { int i = offset, endOffset = offset + length; length = 0; while (i < endOffset) { int charLength = getBytesInChar(bytes[i], columnModifier); i += charLength; length++; } return length; }
DatabaseMetaData dbmd = conn.getMetaData(); String escapedTableName = StringUtil.escapeLike(tableName); String[] schemaAndTable = escapedTableName.split("\\."); ResultSet rs = null;
byteValue = StringUtil.padChar(byteValue, byteSize); } else if (byteSize != null && byteValue.length > byteSize) { throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not exceed " + byteSize + " bytes (" + SchemaUtil.toString(type, byteValue) + ")");
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { Expression arg = getChildren().get(0); if (!arg.evaluate(tuple, ptr)) { return false; } int targetOffset = ptr.getLength(); if (targetOffset == 0) { return true; } byte[] source = ptr.get(); byte[] target = new byte[targetOffset]; int sourceOffset = ptr.getOffset(); int endOffset = sourceOffset + ptr.getLength(); ColumnModifier modifier = arg.getColumnModifier(); while (sourceOffset < endOffset) { int nBytes = StringUtil.getBytesInChar(source[sourceOffset], modifier); targetOffset -= nBytes; System.arraycopy(source, sourceOffset, target, targetOffset, nBytes); sourceOffset += nBytes; } ptr.set(target); return true; }