conf, n); OrcInputFormat.FileGenerator gen = new OrcInputFormat.FileGenerator( context, fs, new MockPath(fs, "mock:/a/b"), false, null); List<SplitStrategy<?>> splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size()); conf, n); OrcInputFormat.FileGenerator gen = new OrcInputFormat.FileGenerator( context, fs, new MockPath(fs, "mock:/a/b"), false, null); List<SplitStrategy<?>> splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size());
MockFileSystem fs = new MockFileSystem(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktbl2"); conf.set("hive.orc.cache.use.soft.references", "true"); conf.set("mapred.input.dir", mockPath.toString()); conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName());
MockFileSystem fs = new MockFileSystem(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktbl"); conf.set(ConfVars.HIVE_ORC_CACHE_STRIPE_DETAILS_MEMORY_SIZE.varname, "0"); conf.set("mapred.input.dir", mockPath.toString()); conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName());
OrcInputFormat.FileGenerator gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a"), false, null); List<OrcInputFormat.SplitStrategy<?>> splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size()); new MockFile("mock:/a/base_0000001/bucket_00000", 1000, new byte[1], new MockBlock("host1")), new MockFile("mock:/a/base_0000001/bucket_00001", 1000, new byte[1], new MockBlock("host1"))); gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a"), false, null); splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size()); new MockFile("mock:/a/delete_delta_0000001_0000001_0000/bucket_00000", 1000, new byte[1], new MockBlock("host1")), new MockFile("mock:/a/delete_delta_0000001_0000001_0000/bucket_00001", 1000, new byte[1], new MockBlock("host1"))); gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a"), false, null); splitStrategies = createSplitStrategies(context, gen); assertEquals(2, splitStrategies.size()); new MockFile("mock:/a/delete_delta_0000001_0000001_0000/bucket_00000", 1000, new byte[1], new MockBlock("host1")), new MockFile("mock:/a/delete_delta_0000001_0000001_0000/bucket_00001", 1000, new byte[1], new MockBlock("host1"))); gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a"), false, null); splitStrategies = createSplitStrategies(context, gen); assertEquals(2, splitStrategies.size()); new MockFile("mock:/a/delete_delta_0000002_0000002_0000/bucket_00000", 1000, new byte[1], new MockBlock("host1")), new MockFile("mock:/a/delete_delta_0000002_0000002_0000/bucket_00001", 1000, new byte[1], new MockBlock("host1"))); gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a"), false, null); splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size());
OrcInputFormat.FileGenerator gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a/b"), false, null); List<OrcInputFormat.SplitStrategy<?>> splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size()); new MockFile("mock:/a/b/part-04", 1000, new byte[1000], new MockBlock("host1", "host2"))); gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a/b"), false, null); splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size()); new MockBlock("host1", "host2"))); gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a/b"), false, null); splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size()); new MockBlock("host1", "host2"))); gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a/b"), false, null); splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size()); new MockBlock("host1", "host2"), new MockBlock("host1", "host2"))); gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a/b"), false, null); splitStrategies = createSplitStrategies(context, gen); assertEquals(1, splitStrategies.size());
@Test public void testACIDReaderNoFooterSerialize() throws Exception { MockFileSystem fs = new MockFileSystem(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktable5"); conf.set(ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN.varname, "true"); conf.setBoolean(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, true); conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES, MyRow.getColumnTypesProperty()); conf.set("hive.orc.splits.include.file.footer", "false"); conf.set("mapred.input.dir", mockPath.toString()); conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName());
conf.set("fs.mock.impl", MockFileSystem.class.getName()); MockPath mockPath = new MockPath(fs, "mock:///mocktable8"); conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS, MyRow.getColumnNamesProperty()); conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES, MyRow.getColumnTypesProperty()); conf.set("hive.orc.splits.include.file.footer", "true"); conf.set("mapred.input.dir", mockPath.toString()); StructObjectInspector inspector; synchronized (TestOrcFile.class) {
@Test public void testACIDReaderFooterSerialize() throws Exception { MockFileSystem fs = new MockFileSystem(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktable6"); conf.set(ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN.varname, "true"); conf.setBoolean(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, true); conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES, MyRow.getColumnTypesProperty()); conf.set("hive.orc.splits.include.file.footer", "true"); conf.set("mapred.input.dir", mockPath.toString()); conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName());
@Test public void testVectorReaderNoFooterSerialize() throws Exception { MockFileSystem fs = new MockFileSystem(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktable3"); conf.set("hive.orc.splits.include.file.footer", "false"); conf.set("mapred.input.dir", mockPath.toString()); conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName());
@Test public void testNonVectorReaderNoFooterSerialize() throws Exception { MockFileSystem fs = new MockFileSystem(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktable1"); conf.set("hive.orc.splits.include.file.footer", "false"); conf.set("mapred.input.dir", mockPath.toString()); conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName());
@Test public void testNonVectorReaderFooterSerialize() throws Exception { MockFileSystem fs = new MockFileSystem(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktable2"); conf.set("hive.orc.splits.include.file.footer", "true"); conf.set("mapred.input.dir", mockPath.toString()); conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName());
conf.set("fs.mock.impl", MockFileSystem.class.getName()); FileSystem fs = FileSystem.get(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktable7"); conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS, MyRow.getColumnNamesProperty()); conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES, MyRow.getColumnTypesProperty()); conf.set("hive.orc.splits.include.file.footer", "false"); conf.set("mapred.input.dir", mockPath.toString()); StructObjectInspector inspector; synchronized (TestOrcFile.class) {
TestVectorizedOrcAcidRowBatchReader.DummyRow.getColumnTypesProperty()); conf.setBoolean(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED.varname, true); MockPath mockPath = new MockPath(fs, "mock:/a"); conf.set("mapred.input.dir", mockPath.toString()); conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName()); OrcInputFormat.Context context = new OrcInputFormat.Context(conf); OrcInputFormat.FileGenerator gen = new OrcInputFormat.FileGenerator(context, fs, new MockPath(fs, "mock:/a"), false, null); List<OrcInputFormat.SplitStrategy<?>> splitStrategies = createSplitStrategies(context, gen);
@Test public void testVectorReaderFooterSerialize() throws Exception { MockFileSystem fs = new MockFileSystem(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktable4"); conf.set("hive.orc.splits.include.file.footer", "true"); conf.set("mapred.input.dir", mockPath.toString()); conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName());
conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName()); MockPath mockPath = new MockPath(fs, "mock:///mocktable"); StructObjectInspector inspector; synchronized (TestOrcFile.class) {
new ValidReadTxnList(new long[0], new BitSet(), 1000, Long.MAX_VALUE).writeToString()); AcidUtils.Directory dir = AcidUtils.getAcidState(new MockPath(fs, "mock:/tbl/part1"), conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); assertEquals(null, dir.getBaseDirectory());
assertEquals(123, opts.getBucketId()); opts = AcidUtils.parseBaseOrDeltaBucketFilename( new MockPath(fs, dir + "/delta_000005_000006/bucket_00001"), conf); assertEquals(false, opts.getOldStyle()); assertEquals(false, opts.isWritingBase()); assertEquals(1, opts.getBucketId()); opts = AcidUtils.parseBaseOrDeltaBucketFilename( new MockPath(fs, dir + "/delete_delta_000005_000006/bucket_00001"), conf); assertEquals(false, opts.getOldStyle()); assertEquals(false, opts.isWritingBase());
new MockFile("mock:/tbl/part1/delta_058_58/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_50/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); conf.set(ValidTxnList.VALID_TXNS_KEY, new ValidReadTxnList(new long[0], new BitSet(), 1000, Long.MAX_VALUE).writeToString());
new ValidReadTxnList(new long[0], new BitSet(), 1000, Long.MAX_VALUE).writeToString()); AcidUtils.Directory dir = AcidUtils.getAcidState(new MockPath(fs, "mock:/tbl/part1"), conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); assertEquals("mock:/tbl/part1/base_49", dir.getBaseDirectory().toString());