/** * {@inheritDoc} */ @Override public List<String> get_tables(final String dbName, final String hivePattern) throws TException { return requestWrapper("get_tables", new Object[]{dbName, hivePattern}, () -> { List<String> result = v1.getDatabase(catalogName, dbName, false, true).getTables(); if (hivePattern != null) { final Pattern pattern = PATTERNS.getUnchecked("(?i)" + hivePattern.replaceAll("\\*", ".*")); result = result.stream().filter(name -> pattern.matcher(name).matches()).collect(Collectors.toList()); } return result; }); }
private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); definitionMetadata = deserializeObjectNode(in); }
private void writeObject(final ObjectOutputStream out) throws IOException { out.defaultWriteObject(); serializeObjectNode(out, definitionMetadata); } }
/** * {@inheritDoc} */ @Override @SuppressWarnings("unchecked") public Database metacatToHiveDatabase(final DatabaseDto dto) { final Database database = new Database(); String name = ""; String description = ""; final QualifiedName databaseName = dto.getName(); if (databaseName != null) { name = databaseName.getDatabaseName(); // Since this is required setting it to the same as the DB name for now description = databaseName.getDatabaseName(); } database.setName(name); database.setDescription(description); String dbUri = dto.getUri(); if (Strings.isNullOrEmpty(dbUri)) { dbUri = ""; } database.setLocationUri(dbUri); Map<String, String> metadata = dto.getMetadata(); if (metadata == null) { metadata = Collections.EMPTY_MAP; } database.setParameters(metadata); return database; }
final List<ListenableFuture<Void>> processDatabaseFutures = input.stream().filter(NOT_NULL) .map(databaseDto -> { final List<QualifiedName> tableNames = databaseDto.getTables().stream() .map(s -> QualifiedName.ofTable(databaseDto.getName().getCatalogName(), databaseDto.getName().getDatabaseName(), s)) .collect(Collectors.toList()); log.info("Full refresh of database {} for tables({}): {}", databaseDto.getName(), databaseDto.getTables().size(), databaseDto.getTables()); return processTables(databaseDto.getName(), tableNames); }).filter(NOT_NULL).collect(Collectors.toList()); processDatabaseFutures.add(processDatabaseFuture);
/** * Save all databases to index it in elastic search. * * @param catalogName catalog name * @param dtos database dtos * @return future */ private ListenableFuture<Void> indexDatabaseDtos(final QualifiedName catalogName, final List<DatabaseDto> dtos) { return esService.submit(() -> { final List<ElasticSearchDoc> docs = dtos.stream() .filter(dto -> dto != null) .map(dto -> new ElasticSearchDoc(dto.getName().toString(), dto, "admin", false, refreshMarkerText)) .collect(Collectors.toList()); log.info("Saving databases for catalog: {}", catalogName); elasticSearchUtil.save(ElasticSearchDoc.Type.database.name(), docs); return null; }); }
"updateDatabase", () -> { final DatabaseDto newDto = new DatabaseDto(); newDto.setName(name); newDto.setMetadata(databaseUpdateRequestDto.getMetadata()); newDto.setDefinitionMetadata(databaseUpdateRequestDto.getDefinitionMetadata()); this.databaseService.update(name, newDto); return null;
/** * Calls the right method of the event bus for the given qualified name. * * @param name name * @param metacatRequestContext context */ public void postPreDeleteEvent( final QualifiedName name, final MetacatRequestContext metacatRequestContext ) { if (name.isPartitionDefinition()) { final PartitionsSaveRequestDto partitionsSaveRequestDto = new PartitionsSaveRequestDto(); partitionsSaveRequestDto.setPartitionIdsForDeletes(Lists.newArrayList(name.getPartitionName())); this.eventBus.post( new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, partitionsSaveRequestDto) ); } else if (name.isViewDefinition()) { this.eventBus.post( new MetacatDeleteMViewPreEvent(name, metacatRequestContext, this) ); } else if (name.isTableDefinition()) { this.eventBus.post(new MetacatDeleteTablePreEvent(name, metacatRequestContext, this)); } else if (name.isDatabaseDefinition()) { final DatabaseDto dto = new DatabaseDto(); dto.setName(name); eventBus.post(new MetacatDeleteDatabasePreEvent(name, metacatRequestContext, this, dto)); } else { throw new IllegalArgumentException(String.format("Invalid name %s", name)); } }
@Override @SuppressWarnings("unchecked") public Database metacatToHiveDatabase(final DatabaseDto dto) { final Database database = new Database(); String name = ""; String description = ""; final QualifiedName databaseName = dto.getName(); if (databaseName != null) { name = databaseName.getDatabaseName(); // Since this is required setting it to the same as the DB name for now description = databaseName.getDatabaseName(); } database.setName(name); database.setDescription(description); String dbUri = dto.getUri(); if (Strings.isNullOrEmpty(dbUri)) { dbUri = ""; } database.setLocationUri(dbUri); Map<String, String> metadata = dto.getMetadata(); if (metadata == null) { metadata = Collections.EMPTY_MAP; } database.setParameters(metadata); return database; }
/** * Subscriber. * * @param event event */ @EventListener public void metacatDeleteDatabasePostEventHandler(final MetacatDeleteDatabasePostEvent event) { log.debug("Received DeleteDatabaseEvent {}", event); this.databaseDeleteEventsDelayTimer .record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS); this.databaseDeleteTimer.record(() -> { final DatabaseDto dto = event.getDatabase(); es.softDelete(ElasticSearchDoc.Type.database.name(), dto.getName().toString(), event.getRequestContext()); }); }
"createDatabase", () -> { final DatabaseDto newDto = new DatabaseDto(); newDto.setName(name); if (databaseCreateRequestDto != null) { newDto.setMetadata(databaseCreateRequestDto.getMetadata()); newDto.setDefinitionMetadata(databaseCreateRequestDto.getDefinitionMetadata());
/** * {@inheritDoc} */ @Override public TableDto copy(final TableDto tableDto, final QualifiedName targetName) { final QualifiedName databaseName = QualifiedName.ofDatabase(targetName.getCatalogName(), targetName.getDatabaseName()); if (!databaseService.exists(databaseName)) { final DatabaseDto databaseDto = new DatabaseDto(); databaseDto.setName(databaseName); databaseService.create(databaseName, databaseDto); } final TableDto targetTableDto = new TableDto(); targetTableDto.setName(targetName); targetTableDto.setFields(tableDto.getFields()); targetTableDto.setPartition_keys(tableDto.getPartition_keys()); final StorageDto storageDto = tableDto.getSerde(); if (storageDto != null) { final StorageDto targetStorageDto = new StorageDto(); targetStorageDto.setInputFormat(storageDto.getInputFormat()); targetStorageDto.setOwner(storageDto.getOwner()); targetStorageDto.setOutputFormat(storageDto.getOutputFormat()); targetStorageDto.setParameters(storageDto.getParameters()); targetStorageDto.setUri(storageDto.getUri()); targetStorageDto.setSerializationLib(storageDto.getSerializationLib()); targetTableDto.setSerde(targetStorageDto); } create(targetName, targetTableDto); return targetTableDto; }
/** * {@inheritDoc} */ @Override @SuppressWarnings("unchecked") public Database metacatToHiveDatabase(final DatabaseDto dto) { final Database database = new Database(); String name = ""; String description = ""; final QualifiedName databaseName = dto.getName(); if (databaseName != null) { name = databaseName.getDatabaseName(); // Since this is required setting it to the same as the DB name for now description = databaseName.getDatabaseName(); } database.setName(name); database.setDescription(description); String dbUri = dto.getUri(); if (Strings.isNullOrEmpty(dbUri)) { dbUri = ""; } database.setLocationUri(dbUri); Map<String, String> metadata = dto.getMetadata(); if (metadata == null) { metadata = Collections.EMPTY_MAP; } database.setParameters(metadata); return database; }
/** * {@inheritDoc} */ @Override public List<String> get_tables(final String dbName, final String hivePattern) throws TException { return requestWrapper("get_tables", new Object[]{dbName, hivePattern}, () -> { List<String> result = v1.getDatabase(catalogName, dbName, false, true).getTables(); if (hivePattern != null) { final Pattern pattern = PATTERNS.getUnchecked("(?i)" + hivePattern.replaceAll("\\*", ".*")); result = result.stream().filter(name -> pattern.matcher(name).matches()).collect(Collectors.toList()); } return result; }); }
private void writeObject(final ObjectOutputStream out) throws IOException { out.defaultWriteObject(); serializeObjectNode(out, definitionMetadata); } }
private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); definitionMetadata = deserializeObjectNode(in); }
/** * Subscriber. * * @param event event */ @EventListener public void metacatCreateDatabasePostEventHandler(final MetacatCreateDatabasePostEvent event) { log.debug("Received CreateDatabaseEvent {}", event); this.databaseCreateEventsDelayTimer .record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS); this.databaseCreateTimer.record(() -> { final DatabaseDto dto = event.getDatabase(); final ElasticSearchDoc doc = new ElasticSearchDoc(dto.getName().toString(), dto, event.getRequestContext().getUserName(), false); es.save(ElasticSearchDoc.Type.database.name(), doc.getId(), doc); }); }
/** * {@inheritDoc} */ @Override public List<String> get_table_names_by_filter(final String dbname, final String filter, final short maxTables) throws TException { return requestWrapper("get_table_names_by_filter", new Object[]{dbname, filter, maxTables}, () -> { final String databaseName = normalizeIdentifier(dbname); final List<String> tables = v1.getDatabase(catalogName, databaseName, false, true).getTables(); // TODO apply filter if (!"hive_filter_field_params__presto_view = \"true\"".equals(filter)) { throw new IllegalArgumentException("Unexpected filter: '" + filter + "'"); } if (maxTables <= 0 || maxTables >= tables.size()) { return tables; } else { return tables.subList(0, maxTables); } }); }
/** * {@inheritDoc} */ @Override public List<String> get_table_names_by_filter(final String dbname, final String filter, final short maxTables) throws TException { return requestWrapper("get_table_names_by_filter", new Object[]{dbname, filter, maxTables}, () -> { final String databaseName = normalizeIdentifier(dbname); final List<String> tables = v1.getDatabase(catalogName, databaseName, false, true).getTables(); // TODO apply filter if (!"hive_filter_field_params__presto_view = \"true\"".equals(filter)) { throw new IllegalArgumentException("Unexpected filter: '" + filter + "'"); } if (maxTables <= 0 || maxTables >= tables.size()) { return tables; } else { return tables.subList(0, maxTables); } }); }