/** * @return IGFS configuration. * @throws Exception If failed. */ public FileSystemConfiguration igfsConfiguration() throws Exception { FileSystemConfiguration cfg = new FileSystemConfiguration(); cfg.setName(igfsName); cfg.setBlockSize(igfsBlockSize); cfg.setDataCacheConfiguration(dataCacheConfiguration()); cfg.setMetaCacheConfiguration(metaCacheConfiguration()); cfg.setFragmentizerEnabled(false); return cfg; }
blockSize = cfg.getBlockSize(); bufSize = cfg.getBufferSize(); colocateMeta = cfg.isColocateMetadata(); dataCacheCfg = cfg.getDataCacheConfiguration(); dfltMode = cfg.getDefaultMode(); fragmentizerConcurrentFiles = cfg.getFragmentizerConcurrentFiles(); fragmentizerEnabled = cfg.isFragmentizerEnabled(); fragmentizerThrottlingBlockLen = cfg.getFragmentizerThrottlingBlockLength(); fragmentizerThrottlingDelay = cfg.getFragmentizerThrottlingDelay(); secondaryFs = cfg.getSecondaryFileSystem(); ipcEndpointCfg = cfg.getIpcEndpointConfiguration(); ipcEndpointEnabled = cfg.isIpcEndpointEnabled(); maxTaskRangeLen = cfg.getMaximumTaskRangeLength(); metaCacheCfg = cfg.getMetaCacheConfiguration(); mgmtPort = cfg.getManagementPort(); name = cfg.getName(); pathModes = cfg.getPathModes(); perNodeBatchSize = cfg.getPerNodeBatchSize(); perNodeParallelBatchCnt = cfg.getPerNodeParallelBatchCount(); prefetchBlocks = cfg.getPrefetchBlocks(); relaxedConsistency = cfg.isRelaxedConsistency(); seqReadsBeforePrefetch = cfg.getSequentialReadsBeforePrefetch(); updateFileLenOnFlush = cfg.isUpdateFileLengthOnFlush();
/** {@inheritDoc} */ @Override protected FileSystemConfiguration getSecondaryIgfsConfiguration() throws IgniteCheckedException { FileSystemConfiguration igfsCfg = super.getSecondaryIgfsConfiguration(); igfsCfg.setName("igfs-secondary"); igfsCfg.setDefaultMode(PRIMARY); IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration(); endpointCfg.setType(IgfsIpcEndpointType.TCP); endpointCfg.setPort(11500); igfsCfg.setIpcEndpointConfiguration(endpointCfg); return igfsCfg; } }
/** {@inheritDoc} */ @Override public boolean isProxy(URI path) { IgfsMode mode = F.isEmpty(cfg.getPathModes()) ? cfg.getDefaultMode() : modeRslvr.resolveMode(new IgfsPath(path)); return mode == PROXY; }
FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName(IGFS_PRIMARY); igfsCfg.setBlockSize(512); igfsCfg.setDefaultMode(DUAL_SYNC); igfsCfg.setPrefetchBlocks(1); igfsCfg.setSequentialReadsBeforePrefetch(Integer.MAX_VALUE); igfsCfg.setSecondaryFileSystem(secondaryFs.asSecondary()); igfsCfg.setPathModes(pathModes); metaCacheCfg.setAtomicityMode(TRANSACTIONAL); igfsCfg.setMetaCacheConfiguration(metaCacheCfg); igfsCfg.setDataCacheConfiguration(dataCacheCfg); igfsPrimary.context().configuration().getDataCacheConfiguration().getName());
/** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(IGFS_BLOCK_SIZE); // Need to set this to avoid thread starvation. igfsCfg.setPerNodeParallelBatchCount(8); igfsCfg.setFragmentizerThrottlingBlockLength(16 * IGFS_BLOCK_SIZE); igfsCfg.setFragmentizerThrottlingDelay(10); igfsCfg.setMetaCacheConfiguration(metaConfiguration()); igfsCfg.setDataCacheConfiguration(dataConfiguration()); cfg.setFileSystemConfiguration(igfsCfg); return cfg; }
FileSystemConfiguration fsConfig = new FileSystemConfiguration(); fsConfig.setName("igfs"); fsConfig.setMetaCacheName("igfs-meta"); fsConfig.setDataCacheName("igfs-data"); fsConfig.setBlockSize(128 * 1024); fsConfig.setPerNodeBatchSize(512); fsConfig.setPerNodeParallelBatchCount(16); fsConfig.setPrefetchBlocks(32);
assert cfg.getName() != null; FileSystemConfiguration cfg0 = new FileSystemConfiguration(cfg); String metaCacheName = cfg.getMetaCacheConfiguration().getName(); ctx, cfg0, new IgfsMetaManager(cfg0.isRelaxedConsistency(), metaClient), new IgfsDataManager(), new IgfsServerManager(), mgr.start(igfsCtx); igfsCache.put(cfg0.getName(), igfsCtx); String dataCacheName = igfsCfg.getDataCacheConfiguration().getName(); igfsCfg.getName(), igfsCfg.getBlockSize(), ((IgfsGroupDataBlocksKeyMapper)affMapper).getGroupSize(), igfsCfg.getMetaCacheConfiguration().getName(), dataCacheName, igfsCfg.getDefaultMode(), igfsCfg.getPathModes(), igfsCfg.isFragmentizerEnabled()));
/** * @throws Exception If failed. */ @Test public void testPreConfiguredCache() throws Exception { FileSystemConfiguration igfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); igfsCfg1.setName("igfs"); g1Cfg.setFileSystemConfiguration(igfsCfg1); CacheConfiguration ccfgData = dataCache(1024); ccfgData.setRebalanceTimeout(10001); CacheConfiguration ccfgMeta = metaCache(); ccfgMeta.setRebalanceTimeout(10002); igfsCfg1.setDataCacheConfiguration(ccfgData); igfsCfg1.setMetaCacheConfiguration(ccfgMeta); IgniteEx g = (IgniteEx)G.start(g1Cfg); assertEquals(10001, g.cachex(g.igfsx("igfs").configuration().getDataCacheConfiguration().getName()) .configuration().getRebalanceTimeout()); assertEquals(10002, g.cachex(g.igfsx("igfs").configuration().getMetaCacheConfiguration().getName()) .configuration().getRebalanceTimeout()); }
String name = cfg.getName(); "assign unique name to each): " + name); CacheConfiguration ccfgData = cfg.getDataCacheConfiguration(); CacheConfiguration ccfgMeta = cfg.getMetaCacheConfiguration(); throw new IgniteCheckedException("IGFS metadata cache should be transactional: " + cfg.getName()); IgfsGroupDataBlocksKeyMapper.class.getSimpleName() + "): " + cfg); IgfsIpcEndpointConfiguration ipcCfg = cfg.getIpcEndpointConfiguration(); boolean secondary = cfg.getDefaultMode() == IgfsMode.PROXY; if (cfg.getPathModes() != null) { for (Map.Entry<String, IgfsMode> mode : cfg.getPathModes().entrySet()) { if (mode.getValue() == IgfsMode.PROXY) secondary = true; if (secondary && cfg.getSecondaryFileSystem() == null) {
/** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); FileSystemConfiguration igfs = new FileSystemConfiguration(); igfs.setName("igfs"); igfs.setIpcEndpointConfiguration(new IgfsIpcEndpointConfiguration()); cfg.setFileSystemConfiguration(igfs); DataStorageConfiguration dsCfg = new DataStorageConfiguration(); DataRegionConfiguration drCfg = new DataRegionConfiguration(); drCfg.setName("testDataRegion"); drCfg.setMaxSize(100L * 1024 * 1024); dsCfg.setDefaultDataRegionConfiguration(drCfg); cfg.setDataStorageConfiguration(dsCfg); return cfg; }
/** * @throws Exception If failed. */ @Test public void testRemoteIfDataBlockSizeDiffers() throws Exception { IgniteConfiguration g2Cfg = getConfiguration("g2"); FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); g2IgfsCfg1.setBlockSize(g2IgfsCfg1.getBlockSize() + 100); g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g1IgfsCfg2); G.start(g1Cfg); checkGridStartFails(g2Cfg, "Data block size should be the same on all nodes in grid for IGFS", false); }
/** * @throws Exception If failed. */ @Test public void testLocalIfAffinityMapperIsWrongClass() throws Exception { for (FileSystemConfiguration igfsCfg : g1Cfg.getFileSystemConfiguration()) { igfsCfg.setDataCacheConfiguration(dataCache(1024)); igfsCfg.setMetaCacheConfiguration(metaCache()); igfsCfg.getMetaCacheConfiguration().setAffinityMapper(new GridCacheDefaultAffinityKeyMapper()); igfsCfg.getDataCacheConfiguration().setAffinityMapper(new GridCacheDefaultAffinityKeyMapper()); } checkGridStartFails(g1Cfg, "Invalid IGFS data cache configuration (key affinity mapper class should be", true); }
meta = igfsCtx.meta(); data = igfsCtx.data(); secondaryFs = cfg.getSecondaryFileSystem(); if (cfg.getDefaultMode() == PROXY) throw new IgniteCheckedException("Mode cannot be PROXY if secondary file system hasn't been defined."); dfltMode = cfg.getDefaultMode(); if (cfg.getPathModes() != null) { for (Map.Entry<String, IgfsMode> e : cfg.getPathModes().entrySet()) cfgModes.put(e.getKey(), e.getValue()); String dataCacheName = igfsCtx.configuration().getDataCacheConfiguration().getName(); new SynchronousQueue<Runnable>(), new IgfsThreadFactory(cfg.getName())) : null;
/** * @throws Exception If failed. */ @Test public void testLocalNullIgfsNameIsNotSupported() throws Exception { try { g1IgfsCfg1.setName(null); fail("IGFS name cannot be null"); } catch (IllegalArgumentException e) { // No-op. } ArrayList<FileSystemConfiguration> fsCfgs = new ArrayList<>(Arrays.asList(g1Cfg.getFileSystemConfiguration())); fsCfgs.add(new FileSystemConfiguration()); // IGFS doesn't have default name (name == null). g1Cfg.setFileSystemConfiguration(fsCfgs.toArray(new FileSystemConfiguration[fsCfgs.size()])); checkGridStartFails(g1Cfg, "IGFS name cannot be null", true); }
/** * Constructor. * * @param igfs Target IGFS. * @param log Log. * @param userName User name. */ private HadoopIgfsInProc(IgfsEx igfs, Log log, String userName) { this.igfs = igfs; this.log = log; bufSize = igfs.configuration().getBlockSize() * 2; user = IgfsUtils.fixUserName(userName); }
/** * Initialize internal cache names */ private void initializeInternalCacheNames() { FileSystemConfiguration[] igfsCfgs = ctx.grid().configuration().getFileSystemConfiguration(); if (igfsCfgs != null) { for (FileSystemConfiguration igfsCfg : igfsCfgs) { internalCaches.add(igfsCfg.getMetaCacheConfiguration().getName()); internalCaches.add(igfsCfg.getDataCacheConfiguration().getName()); } } if (IgniteComponentType.HADOOP.inClassPath()) internalCaches.add(CU.SYS_CACHE_HADOOP_MR); }
/** * Get data cache for node. * * @param node Node. * @return Data cache. * @throws Exception If failed. */ private GridCacheAdapter dataCache(Ignite node) throws Exception { return ((IgniteKernal)node).internalCache(((IgniteKernal)node).igfsx(IGFS_NAME).configuration() .getDataCacheConfiguration().getName()); }
/** * Check invalid port handling. * * @param port Port. * @throws Exception If failed. */ private void checkInvalidPort(int port) throws Exception { final String failMsg = "IGFS endpoint TCP port is out of range"; final String igfsCfgName = "igfs-cfg"; final IgfsIpcEndpointConfiguration igfsEndpointCfg = new IgfsIpcEndpointConfiguration(); igfsEndpointCfg.setPort(port); g1IgfsCfg1.setName(igfsCfgName); g1IgfsCfg1.setIpcEndpointConfiguration(igfsEndpointCfg); checkGridStartFails(g1Cfg, failMsg, true); }
/** * @throws Exception If failed. */ @Test public void testRemoteIfDefaultModeDiffers() throws Exception { IgniteConfiguration g2Cfg = getConfiguration("g2"); FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2); g1IgfsCfg1.setDefaultMode(DUAL_ASYNC); g1IgfsCfg2.setDefaultMode(DUAL_ASYNC); g2IgfsCfg1.setDefaultMode(DUAL_SYNC); g2IgfsCfg2.setDefaultMode(DUAL_SYNC); g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2); G.start(g1Cfg); checkGridStartFails(g2Cfg, "Default mode should be the same on all nodes in grid for IGFS", false); }