@Test public void testNullableField() throws IOException, SQLException { String[] argv = {}; final int TOTAL_RECORDS = 1 * 10; List<Schema> childSchemas = new ArrayList<Schema>(); childSchemas.add(Schema.create(Schema.Type.NULL)); childSchemas.add(Schema.create(Schema.Type.STRING)); Schema schema = Schema.createUnion(childSchemas); ColumnGenerator gen0 = colGenerator(null, schema, null, "VARCHAR(64)"); ColumnGenerator gen1 = colGenerator("s", schema, "s", "VARCHAR(64)"); createAvroFile(0, TOTAL_RECORDS, gen0, gen1); createTable(gen0, gen1); runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1))); verifyExport(TOTAL_RECORDS); assertColMinAndMax(forIdx(0), gen0); assertColMinAndMax(forIdx(1), gen1); }
@Test public void testSpecifiedColumnsAsAvroFields() throws IOException, SQLException { final int TOTAL_RECORDS = 10; ColumnGenerator[] gens = new ColumnGenerator[] { colGenerator(000, Schema.create(Schema.Type.INT), 100, "INTEGER"), //col0 colGenerator(111, Schema.create(Schema.Type.INT), 100, "INTEGER"), //col1 colGenerator(222, Schema.create(Schema.Type.INT), 100, "INTEGER"), //col2 colGenerator(333, Schema.create(Schema.Type.INT), 100, "INTEGER") //col3 }; createAvroFile(0, TOTAL_RECORDS, gens); createTable(gens); runExport(getArgv(true, 10, 10, newStrArray(null, "-m", "" + 1, "--columns", "ID,MSG,COL1,COL2"))); verifyExport(TOTAL_RECORDS); assertColValForRowId(0, "col0", null); assertColValForRowId(0, "col1", 111); assertColValForRowId(0, "col2", 222); assertColValForRowId(0, "col3", null); assertColValForRowId(9, "col0", null); assertColValForRowId(9, "col1", 111); assertColValForRowId(9, "col2", 222); assertColValForRowId(9, "col3", null); }
@Test public void testAvroWithUpdateKey() throws IOException, SQLException { String[] argv = { "--update-key", "ID" }; final int TOTAL_RECORDS = 1; // ColumnGenerator gen = colGenerator("100", // Schema.create(Schema.Type.STRING), null, "VARCHAR(64)"); createAvroFile(0, TOTAL_RECORDS); createTableWithInsert(); runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1))); verifyExport(getMsgPrefix() + "0"); }
@Test public void testMissingDatabaseFields() throws IOException, SQLException { String[] argv = {}; final int TOTAL_RECORDS = 1; // null column type means don't create a database column // the Avro value will not be exported ColumnGenerator gen = colGenerator(100, Schema.create(Schema.Type.INT), null, null); createAvroFile(0, TOTAL_RECORDS, gen); createTable(gen); runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1))); verifyExport(TOTAL_RECORDS); }
@Test public void testPathPatternInExportDir() throws IOException, SQLException { final int TOTAL_RECORDS = 10; ColumnGenerator[] gens = new ColumnGenerator[] { colGenerator(true, Schema.create(Schema.Type.BOOLEAN), true, "BIT"), }; createAvroFile(0, TOTAL_RECORDS, gens); createTable(gens); // Converts path to an unary set while preserving the leading '/' String pathPattern = new StringBuilder(getTablePath().toString()) .insert(1, "{") .append("}") .toString(); runExport(getArgv(true, 10, 10, "--export-dir", pathPattern)); verifyExport(TOTAL_RECORDS); }
@Test public void testAvroWithUpsert() throws IOException, SQLException { String[] argv = { "--update-key", "ID", "--update-mode", "allowinsert" }; final int TOTAL_RECORDS = 2; // ColumnGenerator gen = colGenerator("100", // Schema.create(Schema.Type.STRING), null, "VARCHAR(64)"); createAvroFile(0, TOTAL_RECORDS); createTableWithInsert(); thrown.expect(Exception.class); thrown.reportMissingExceptionWithMessage("Expected Exception during Avro export with --update-mode"); runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1))); }
/** Verify that for the max and min values of the 'id' column, the values for a given column meet the expected values. */ protected void assertColMinAndMax(String colName, ColumnGenerator generator) throws SQLException { Connection conn = getConnection(); int minId = getMinRowId(conn); int maxId = getMaxRowId(conn); LOG.info("Checking min/max for column " + colName + " with type " + generator.getColumnType()); Object expectedMin = generator.getVerifyValue(minId); Object expectedMax = generator.getVerifyValue(maxId); assertColValForRowId(minId, colName, expectedMin); assertColValForRowId(maxId, colName, expectedMax); }
Connection conn = getConnection(); PreparedStatement statement = conn.prepareStatement( getDropTableStatement(getTableName()), ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); try { sb.append(getTableName()); sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)"); int colNum = 0; for (ColumnGenerator gen : extraColumns) { if (gen.getColumnType() != null) { sb.append(", " + forIdx(colNum++) + " " + gen.getColumnType());
ColumnGenerator... extraCols) throws IOException { Path tablePath = getTablePath(); Path filePath = new Path(tablePath, "part" + fileNum); OutputStream os = fs.create(filePath); Schema schema = buildAvroSchema(extraCols); DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(); GenericRecord record = new GenericData.Record(schema); record.put("id", i); record.put("msg", getMsgPrefix() + i); addExtraColumns(record, i, extraCols); dataFileWriter.append(record);
private Schema buildAvroSchema(ColumnGenerator... extraCols) { List<Field> fields = new ArrayList<Field>(); fields.add(buildAvroField("id", Schema.Type.INT)); fields.add(buildAvroField("msg", Schema.Type.STRING)); int colNum = 0; // Issue [SQOOP-2846] if (null != extraCols) { for (ColumnGenerator gen : extraCols) { if (gen.getColumnAvroSchema() != null) { fields.add(buildAvroField(forIdx(colNum++), gen.getColumnAvroSchema())); } } } Schema schema = Schema.createRecord("myschema", null, null, false); schema.setFields(fields); return schema; }
private void addExtraColumns(GenericRecord record, int rowNum, ColumnGenerator[] extraCols) { int colNum = 0; // Issue [SQOOP-2846] if (null != extraCols) { for (ColumnGenerator gen : extraCols) { if (gen.getColumnAvroSchema() != null) { record.put(forIdx(colNum++), gen.getExportValue(rowNum)); } } } }
@Test public void testMissingAvroFields() throws IOException, SQLException { String[] argv = {}; final int TOTAL_RECORDS = 1; // null Avro schema means don't create an Avro field ColumnGenerator gen = colGenerator(null, null, null, "VARCHAR(64)"); createAvroFile(0, TOTAL_RECORDS, gen); createTable(gen); thrown.expect(Exception.class); thrown.reportMissingExceptionWithMessage("Expected Exception on missing Avro fields"); runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1))); }
colGenerator(true, Schema.create(Schema.Type.BOOLEAN), true, "BIT"), colGenerator(100, Schema.create(Schema.Type.INT), 100, "INTEGER"), colGenerator(200L, Schema.create(Schema.Type.LONG), 200L, "BIGINT"), colGenerator(1.0f, Schema.create(Schema.Type.FLOAT), 1.0d, "REAL"), colGenerator(2.0d, Schema.create(Schema.Type.DOUBLE), 2.0d, "DOUBLE"), colGenerator("s", Schema.create(Schema.Type.STRING), "s", "VARCHAR(8)"), colGenerator(ByteBuffer.wrap(b), Schema.create(Schema.Type.BYTES), b, "VARBINARY(8)"), colGenerator(new GenericData.Fixed(fixed, b), fixed, b, "BINARY(2)"), colGenerator(new GenericData.EnumSymbol(enumeration, "a"), enumeration, "a", "VARCHAR(8)"), colGenerator(new BigDecimal("2.00"), decimalSchema, new BigDecimal("2.00"), "DECIMAL(3,2)"), colGenerator("22.00", Schema.create(Schema.Type.STRING), new BigDecimal("22.00"), "DECIMAL(4,2)"), }; createAvroFile(0, TOTAL_RECORDS, gens); createTable(gens); runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1))); verifyExport(TOTAL_RECORDS); for (int i = 0; i < gens.length; i++) { assertColMinAndMax(forIdx(i), gens[i]);
@Test public void testAvroRecordsNotSupported() throws IOException, SQLException { String[] argv = {}; final int TOTAL_RECORDS = 1; Schema schema = Schema.createRecord("nestedrecord", null, null, false); schema.setFields(Lists.newArrayList(buildAvroField("myint", Schema.Type.INT))); GenericRecord record = new GenericData.Record(schema); record.put("myint", 100); // DB type is not used so can be anything: ColumnGenerator gen = colGenerator(record, schema, null, "VARCHAR(64)"); createAvroFile(0, TOTAL_RECORDS, gen); createTable(gen); thrown.expect(Exception.class); thrown.reportMissingExceptionWithMessage("Expected Exception as Avro records are not supported"); runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1))); }