private ExtTableSnapshotInfo load(String resourcePath) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); ExtTableSnapshotInfo snapshot = store.getResource(resourcePath, SNAPSHOT_SERIALIZER); return snapshot; }
public void removeSnapshot(String tableName, String snapshotID) throws IOException { String snapshotResPath = ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID); snapshotCache.invalidate(snapshotResPath); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); store.deleteResource(snapshotResPath); }
public List<SnapshotTable> getSnapshots(String tableName, TableSignature sourceTableSignature) throws IOException { List<SnapshotTable> result = Lists.newArrayList(); String tableSnapshotsPath = SnapshotTable.getResourceDir(tableName); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); result.addAll(store.getAllResources(tableSnapshotsPath, SnapshotTableSerializer.INFO_SERIALIZER)); if (sourceTableSignature != null) { String oldTableSnapshotsPath = SnapshotTable.getOldResourceDir(sourceTableSignature); result.addAll(store.getAllResources(oldTableSnapshotsPath, SnapshotTableSerializer.INFO_SERIALIZER)); } return result; }
public Set<String> getAllExtSnapshotResPaths() throws IOException { Set<String> result = Sets.newHashSet(); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); Set<String> snapshotTablePaths = store.listResources(ResourceStore.EXT_SNAPSHOT_RESOURCE_ROOT); if (snapshotTablePaths == null) { return result; } for (String snapshotTablePath : snapshotTablePaths) { Set<String> snapshotPaths = store.listResources(snapshotTablePath); if (snapshotPaths != null) { result.addAll(snapshotPaths); } } return result; }
public void save(ExtTableSnapshotInfo snapshot) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); String path = snapshot.getResourcePath(); store.checkAndPutResource(path, snapshot, SNAPSHOT_SERIALIZER); }
public List<ExtTableSnapshotInfo> getSnapshots(String tableName) throws IOException { String tableSnapshotsPath = ExtTableSnapshotInfo.getResourceDir(tableName); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); return store.getAllResources(tableSnapshotsPath, SNAPSHOT_SERIALIZER); }
private TableExtDesc convertOldTableExtToNewer(String resourceName) { ResourceStore store = getStore(); Map<String, String> attrs = Maps.newHashMap(); try { RawResource res = store.getResource( ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + resourceName + MetadataConstants.FILE_SURFIX); InputStream is = res.content(); try { attrs.putAll(JsonUtil.readValue(is, HashMap.class)); } finally { if (is != null) is.close(); } } catch (IOException ex) { throw new RuntimeException(ex); } String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY); // parse table identity from file name String tableIdentity = TableDesc.parseResourcePath(resourceName).getFirst(); TableExtDesc result = new TableExtDesc(); result.setIdentity(tableIdentity); result.setUuid(RandomUtil.randomUUID().toString()); result.setLastModified(0); result.setCardinality(cardinality); return result; }
private void initExtFilter() throws IOException { this.extFilterMap = new CaseInsensitiveStringCache<>(config, "external_filter"); this.extFilterCrud = new CachedCrudAssist<ExternalFilterDesc>(getStore(), ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT, ExternalFilterDesc.class, extFilterMap) { @Override protected ExternalFilterDesc initEntityAfterReload(ExternalFilterDesc t, String resourceName) { return t; // noop } }; extFilterCrud.reloadAll(); Broadcaster.getInstance(config).registerListener(new ExtFilterSyncListener(), "external_filter"); }
private void initSrcTable() throws IOException { this.srcTableMap = new CaseInsensitiveStringCache<>(config, "table"); this.srcTableCrud = new CachedCrudAssist<TableDesc>(getStore(), ResourceStore.TABLE_RESOURCE_ROOT, TableDesc.class, srcTableMap) { @Override protected TableDesc initEntityAfterReload(TableDesc t, String resourceName) { String prj = TableDesc.parseResourcePath(resourceName).getSecond(); t.init(config, prj); return t; } }; srcTableCrud.reloadAll(); Broadcaster.getInstance(config).registerListener(new SrcTableSyncListener(), "table"); }
private void initSrcExt() throws IOException { this.srcExtMap = new CaseInsensitiveStringCache<>(config, "table_ext"); this.srcExtCrud = new CachedCrudAssist<TableExtDesc>(getStore(), ResourceStore.TABLE_EXD_RESOURCE_ROOT, TableExtDesc.class, srcExtMap) { @Override protected TableExtDesc initEntityAfterReload(TableExtDesc t, String resourceName) { // convert old tableExt json to new one if (t.getIdentity() == null) { t = convertOldTableExtToNewer(resourceName); } String prj = TableDesc.parseResourcePath(resourceName).getSecond(); t.init(prj); return t; } }; srcExtCrud.reloadAll(); Broadcaster.getInstance(config).registerListener(new SrcTableExtSyncListener(), "table_ext"); }
private void loadH2Table(String tableName) throws SQLException { TableMetadataManager metaMgr = TableMetadataManager.getInstance(config); TableDesc tableDesc = metaMgr.getTableDesc(tableName.toUpperCase(Locale.ROOT), project); File tempFile = null; try { tempFile = File.createTempFile("tmp_h2", ".csv"); FileOutputStream tempFileStream = new FileOutputStream(tempFile); String path = path(tableDesc); InputStream csvStream = metaMgr.getStore().getResource(path).content(); IOUtils.copy(csvStream, tempFileStream); csvStream.close(); tempFileStream.close(); } catch (IOException e) { e.printStackTrace(); } String cvsFilePath = tempFile.getPath(); try (Statement stmt = h2Connection.createStatement()) { String createDBSql = "CREATE SCHEMA IF NOT EXISTS DEFAULT;\nCREATE SCHEMA IF NOT EXISTS EDW;\nSET SCHEMA DEFAULT;\n"; stmt.executeUpdate(createDBSql); String sql = generateCreateH2TableSql(tableDesc, cvsFilePath); stmt.executeUpdate(sql); List<String> createIndexStatements = generateCreateH2IndexSql(tableDesc); for (String indexSql : createIndexStatements) { stmt.executeUpdate(indexSql); } } tempFile.delete(); }
private ExtTableSnapshotInfo checkDupByInfo(ExtTableSnapshotInfo snapshot) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { ExtTableSnapshotInfo existingSnapshot = load(existing); // direct load from store if (existingSnapshot != null && sig.equals(existingSnapshot.getSignature())) return existingSnapshot; } return null; }
public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException { try (AutoLock lock = srcExtMapLock.lockForWrite()) { if (tableExt.getUuid() == null || tableExt.getIdentity() == null) { throw new IllegalArgumentException(); } // updating a legacy global table if (tableExt.getProject() == null) { if (getTableExt(tableExt.getIdentity(), prj).getProject() != null) throw new IllegalStateException( "Updating a legacy global TableExtDesc while a project level version exists: " + tableExt.getIdentity() + ", " + prj); prj = tableExt.getProject(); } tableExt.init(prj); // what is this doing?? String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj); ResourceStore store = getStore(); TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER); if (t != null && t.getIdentity() == null) store.deleteResource(path); srcExtCrud.save(tableExt); } }
InputStream hbaseDataStream = metaMgr.getStore().getResource("/data/" + tablename + ".csv").content(); FileOutputStream localFileStream = new FileOutputStream(localBufferFile); IOUtils.copy(hbaseDataStream, localFileStream);
private ExtTableSnapshotInfo load(String resourcePath) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); ExtTableSnapshotInfo snapshot = store.getResource(resourcePath, SNAPSHOT_SERIALIZER); return snapshot; }
public void removeSnapshot(String tableName, String snapshotID) throws IOException { String snapshotResPath = ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID); snapshotCache.invalidate(snapshotResPath); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); store.deleteResource(snapshotResPath); }
public List<SnapshotTable> getSnapshots(String tableName, TableSignature sourceTableSignature) throws IOException { List<SnapshotTable> result = Lists.newArrayList(); String tableSnapshotsPath = SnapshotTable.getResourceDir(tableName); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); result.addAll(store.getAllResources(tableSnapshotsPath, SnapshotTableSerializer.INFO_SERIALIZER)); if (sourceTableSignature != null) { String oldTableSnapshotsPath = SnapshotTable.getOldResourceDir(sourceTableSignature); result.addAll(store.getAllResources(oldTableSnapshotsPath, SnapshotTableSerializer.INFO_SERIALIZER)); } return result; }
public void save(ExtTableSnapshotInfo snapshot) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); String path = snapshot.getResourcePath(); store.checkAndPutResource(path, snapshot, SNAPSHOT_SERIALIZER); }
public List<ExtTableSnapshotInfo> getSnapshots(String tableName) throws IOException { String tableSnapshotsPath = ExtTableSnapshotInfo.getResourceDir(tableName); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); return store.getAllResources(tableSnapshotsPath, SNAPSHOT_SERIALIZER); }
private void initExtFilter() throws IOException { this.extFilterMap = new CaseInsensitiveStringCache<>(config, "external_filter"); this.extFilterCrud = new CachedCrudAssist<ExternalFilterDesc>(getStore(), ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT, ExternalFilterDesc.class, extFilterMap) { @Override protected ExternalFilterDesc initEntityAfterReload(ExternalFilterDesc t, String resourceName) { return t; // noop } }; extFilterCrud.reloadAll(); Broadcaster.getInstance(config).registerListener(new ExtFilterSyncListener(), "external_filter"); }