/** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setMetaCacheConfiguration(cacheConfiguration("meta")); igfsCfg.setDataCacheConfiguration(cacheConfiguration("data")); igfsCfg.setBlockSize(IGFS_BLOCK_SIZE); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(BLOCK_SIZE); cfg.setFileSystemConfiguration(igfsCfg); return cfg; }
/** * @throws Exception If failed. */ @Test public void testRemoteIfDataBlockSizeDiffers() throws Exception { IgniteConfiguration g2Cfg = getConfiguration("g2"); FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); g2IgfsCfg1.setBlockSize(g2IgfsCfg1.getBlockSize() + 100); g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g1IgfsCfg2); G.start(g1Cfg); checkGridStartFails(g2Cfg, "Data block size should be the same on all nodes in grid for IGFS", false); }
/** * @return IGFS configuration. * @throws Exception If failed. */ public FileSystemConfiguration igfsConfiguration() throws Exception { FileSystemConfiguration cfg = new FileSystemConfiguration(); cfg.setName(igfsName); cfg.setBlockSize(igfsBlockSize); cfg.setDataCacheConfiguration(dataCacheConfiguration()); cfg.setMetaCacheConfiguration(metaCacheConfiguration()); cfg.setFragmentizerEnabled(false); return cfg; }
/** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); cfg.setCacheConfiguration(); FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setMetaCacheConfiguration(cacheConfiguration("meta")); igfsCfg.setDataCacheConfiguration(cacheConfiguration("data")); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(CFG_BLOCK_SIZE); igfsCfg.setFragmentizerEnabled(true); cfg.setFileSystemConfiguration(igfsCfg); return cfg; }
/** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(IGFS_BLOCK_SIZE); // Need to set this to avoid thread starvation. igfsCfg.setPerNodeParallelBatchCount(8); igfsCfg.setFragmentizerThrottlingBlockLength(16 * IGFS_BLOCK_SIZE); igfsCfg.setFragmentizerThrottlingDelay(10); igfsCfg.setMetaCacheConfiguration(metaConfiguration()); igfsCfg.setDataCacheConfiguration(dataConfiguration()); cfg.setFileSystemConfiguration(igfsCfg); return cfg; }
/** * Gets IGFS configuration. * * @param igniteInstanceName Ignite instance name. * @return IGFS configuration. */ protected FileSystemConfiguration fsConfiguration(String igniteInstanceName) throws IgniteCheckedException { FileSystemConfiguration cfg = new FileSystemConfiguration(); cfg.setName("igfs"); cfg.setPrefetchBlocks(1); cfg.setDefaultMode(mode); if (mode != PRIMARY) cfg.setSecondaryFileSystem( new IgniteHadoopIgfsSecondaryFileSystem(secondaryFsUriStr, secondaryConfFullPath)); cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(igniteInstanceName)); cfg.setManagementPort(-1); cfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups. cfg.setDataCacheConfiguration(dataCacheConfiguration()); cfg.setMetaCacheConfiguration(metaCacheConfiguration()); return cfg; }
/** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); cfg.setDiscoverySpi(discoSpi); FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheConfiguration(dataCacheConfiguration()); igfsCfg.setMetaCacheConfiguration(metaCacheConfiguration()); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(512 * 1024); IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration(); endpointCfg.setType(IgfsIpcEndpointType.TCP); endpointCfg.setPort(DFLT_IPC_PORT); igfsCfg.setIpcEndpointConfiguration(endpointCfg); cfg.setFileSystemConfiguration(igfsCfg); return cfg; }
/** * Gets IGFS configuration. * * @param igniteInstanceName Ignite instance name. * @return IGFS configuration. */ protected FileSystemConfiguration igfsConfiguration(String igniteInstanceName) throws IgniteCheckedException { FileSystemConfiguration cfg = new FileSystemConfiguration(); cfg.setName("igfs"); cfg.setPrefetchBlocks(1); cfg.setDefaultMode(mode); cfg.setMetaCacheConfiguration(metaCacheConfiguration(igniteInstanceName)); cfg.setDataCacheConfiguration(dataCacheConfiguration(igniteInstanceName)); if (mode != PRIMARY) cfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(secondaryFileSystemUriPath(), secondaryFileSystemConfigPath(), SECONDARY_FS_USER)); cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(igniteInstanceName)); cfg.setManagementPort(-1); cfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups. return cfg; }
/** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName("igfs"); igfsCfg.setDataCacheConfiguration(cacheConfiguration(igniteInstanceName, "data") .setNearConfiguration(new NearCacheConfiguration())); igfsCfg.setMetaCacheConfiguration(cacheConfiguration(igniteInstanceName, "meta") .setNearConfiguration(new NearCacheConfiguration())); IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration(); endpointCfg.setType(IgfsIpcEndpointType.SHMEM); endpointCfg.setPort(IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT + cnt); igfsCfg.setIpcEndpointConfiguration(endpointCfg); igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups. cfg.setFileSystemConfiguration(igfsCfg); cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED); if (cnt == 0) cfg.setClientMode(true); cnt++; return cfg; }
/** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName("igfs"); igfsCfg.setManagementPort(FileSystemConfiguration.DFLT_MGMT_PORT + cnt); igfsCfg.setDataCacheConfiguration(dataCacheConfiguration()); igfsCfg.setMetaCacheConfiguration(metaCacheConfiguration()); IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration(); endpointCfg.setType(IgfsIpcEndpointType.SHMEM); endpointCfg.setPort(IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT + cnt); igfsCfg.setIpcEndpointConfiguration(endpointCfg); igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups. cfg.setFileSystemConfiguration(igfsCfg); cfg.setCacheConfiguration(dataCacheConfiguration()); cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED); TcpCommunicationSpi commSpi = new TcpCommunicationSpi(); commSpi.setSharedMemoryPort(-1); cfg.setCommunicationSpi(commSpi); cnt++; return cfg; }
igfsCfg.setBlockSize(igfsBlockSize); igfsCfg.setDefaultMode(mode); igfsCfg.setSecondaryFileSystem(secondaryFs);
/** * @return IGFS configuration for this test. */ protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException { FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups. CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); dataCacheCfg.setCacheMode(PARTITIONED); dataCacheCfg.setNearConfiguration(null); dataCacheCfg.setWriteSynchronizationMode(FULL_SYNC); dataCacheCfg.setEvictionPolicy(null); dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128)); dataCacheCfg.setBackups(0); dataCacheCfg.setAtomicityMode(TRANSACTIONAL); CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); metaCacheCfg.setCacheMode(REPLICATED); metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC); metaCacheCfg.setEvictionPolicy(null); metaCacheCfg.setAtomicityMode(TRANSACTIONAL); igfsCfg.setMetaCacheConfiguration(metaCacheCfg); igfsCfg.setDataCacheConfiguration(dataCacheCfg); return igfsCfg; }
/** * Gets IGFS configuration. * * @param igniteInstanceName Ignite instance name. * @return IGFS configuration. */ protected FileSystemConfiguration igfsConfiguration(String igniteInstanceName) throws IgniteCheckedException { FileSystemConfiguration cfg = new FileSystemConfiguration(); cfg.setDataCacheConfiguration(dataCacheConfiguration(igniteInstanceName)); cfg.setMetaCacheConfiguration(metaCacheConfiguration(igniteInstanceName)); cfg.setName("igfs"); cfg.setPrefetchBlocks(1); cfg.setDefaultMode(mode); if (mode != PRIMARY) { CachingHadoopFileSystemFactory fac = new CachingHadoopFileSystemFactory(); fac.setUri(SECONDARY_URI); fac.setConfigPaths(SECONDARY_CFG_PATH); IgniteHadoopIgfsSecondaryFileSystem sec = new IgniteHadoopIgfsSecondaryFileSystem(); sec.setFileSystemFactory(fac); sec.setDefaultUserName(SECONDARY_FS_USER); // NB: start() will be invoked upon IgfsImpl init. cfg.setSecondaryFileSystem(sec); } cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(igniteInstanceName)); cfg.setManagementPort(-1); cfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups. return cfg; }
/** * Create grid configuration. * * @param idx Node index. * @return Grid configuration */ private IgniteConfiguration config(int idx) { FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(BLOCK_SIZE); igfsCfg.setDefaultMode(PRIMARY); igfsCfg.setFragmentizerEnabled(false); CacheConfiguration dataCacheCfg = new CacheConfiguration(DEFAULT_CACHE_NAME); dataCacheCfg.setCacheMode(PARTITIONED); dataCacheCfg.setAtomicityMode(TRANSACTIONAL); dataCacheCfg.setWriteSynchronizationMode(FULL_SYNC); dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(1)); dataCacheCfg.setBackups(0); CacheConfiguration metaCacheCfg = new CacheConfiguration(DEFAULT_CACHE_NAME); metaCacheCfg.setCacheMode(REPLICATED); metaCacheCfg.setAtomicityMode(TRANSACTIONAL); metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); igfsCfg.setMetaCacheConfiguration(metaCacheCfg); igfsCfg.setDataCacheConfiguration(dataCacheCfg); IgniteConfiguration cfg = new IgniteConfiguration(); cfg.setFileSystemConfiguration(igfsCfg); cfg.setIgniteInstanceName("node-" + idx); return cfg; }
/** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(512); igfsCfg.setDefaultMode(PRIMARY); CacheConfiguration dataCacheCfg = new CacheConfiguration(DEFAULT_CACHE_NAME); dataCacheCfg.setCacheMode(PARTITIONED); dataCacheCfg.setAtomicityMode(TRANSACTIONAL); dataCacheCfg.setNearConfiguration(new NearCacheConfiguration()); dataCacheCfg.setWriteSynchronizationMode(FULL_SYNC); dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128)); dataCacheCfg.setBackups(0); CacheConfiguration metaCacheCfg = new CacheConfiguration(DEFAULT_CACHE_NAME); metaCacheCfg.setCacheMode(REPLICATED); metaCacheCfg.setAtomicityMode(TRANSACTIONAL); metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC); igfsCfg.setMetaCacheConfiguration(metaCacheCfg); igfsCfg.setDataCacheConfiguration(dataCacheCfg); IgniteConfiguration cfg = new IgniteConfiguration(); cfg.setIgniteInstanceName("grid"); cfg.setFileSystemConfiguration(igfsCfg); Ignite g = G.start(cfg); igfs = g.fileSystem("igfs"); }
igfsCfg.setBlockSize(BLOCK_SIZE); igfsCfg.setFragmentizerEnabled(false); igfsCfg.setManagementPort(++mgmtPort);
igfsCfg.setBlockSize(512 * 1024); igfsCfg.setPrefetchBlocks(1);
igfsCfg.setBlockSize(SECONDARY_BLOCK_SIZE); igfsCfg.setDefaultMode(PRIMARY); igfsCfg.setIpcEndpointConfiguration(SECONDARY_REST_CFG);
igfsCfg.setBlockSize(512 * 1024); igfsCfg.setDefaultMode(PRIMARY);
igfsCfg.setBlockSize(512); igfsCfg.setDefaultMode(PRIMARY); igfsCfg.setIpcEndpointConfiguration(SECONDARY_REST_CFG);