diff --git a/.github/workflows/sqllab-frontend.yml b/.github/workflows/sqllab-frontend.yml new file mode 100644 index 00000000000..8a14407b6d3 --- /dev/null +++ b/.github/workflows/sqllab-frontend.yml @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: SQL Lab Frontend CI + +on: + push: + paths: + - 'exec/java-exec/src/main/resources/webapp/**' + pull_request: + paths: + - 'exec/java-exec/src/main/resources/webapp/**' + +defaults: + run: + working-directory: exec/java-exec/src/main/resources/webapp + +jobs: + build-and-test: + name: Build & Test + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + cache-dependency-path: exec/java-exec/src/main/resources/webapp/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: TypeScript type check + run: npx tsc --noEmit + + - name: Lint + run: npm run lint + + - name: Run tests + run: npx vitest run --passWithNoTests + + - name: Build + run: npm run build diff --git a/.gitignore b/.gitignore index 0304fc07c3e..5f638ad294f 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,8 @@ target/ tools/venv/ venv/ .vscode/* +/exec/java-exec/src/main/resources/python/.venv/ +__pycache__/ +*.pyc +.pytest_cache/ +.coverage diff --git a/build-frontend.sh b/build-frontend.sh new file mode 100755 index 00000000000..dcadc01dc35 --- /dev/null +++ b/build-frontend.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Build the SQL Lab frontend and update the distribution. +# Usage: ./build-frontend.sh + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +WEBAPP_DIR="$SCRIPT_DIR/exec/java-exec/src/main/resources/webapp" +DIST_BASE="$SCRIPT_DIR/distribution/target" + +echo "=== Building SQL Lab frontend ===" +cd "$WEBAPP_DIR" +npm run build + +echo "" +echo "=== Building java-exec module ===" +cd "$SCRIPT_DIR" +mvn package -pl exec/java-exec -DskipTests -Dcheckstyle.skip=true -q + +# Find the distribution directory and copy the jar +DIST_DIR=$(find "$DIST_BASE" -name "jars" -type d 2>/dev/null | head -1) +if [ -n "$DIST_DIR" ]; then + echo "" + echo "=== Copying jar to distribution ===" + cp "$SCRIPT_DIR/exec/java-exec/target/drill-java-exec-"*"-SNAPSHOT.jar" "$DIST_DIR/" + echo "Updated: $DIST_DIR/" +else + echo "" + echo "WARNING: Distribution directory not found under $DIST_BASE" + echo "Run a full 'mvn package' first to create the distribution, or copy the jar manually:" + echo " cp exec/java-exec/target/drill-java-exec-*-SNAPSHOT.jar /jars/" +fi + +echo "" +echo "=== Done! Restart Drill and hard-refresh your browser (Cmd+Shift+R) ===" diff --git a/contrib/format-xml/src/main/java/org/apache/drill/exec/store/xml/XMLReader.java b/contrib/format-xml/src/main/java/org/apache/drill/exec/store/xml/XMLReader.java index d985cd69d89..a33d45204ba 100644 --- a/contrib/format-xml/src/main/java/org/apache/drill/exec/store/xml/XMLReader.java +++ b/contrib/format-xml/src/main/java/org/apache/drill/exec/store/xml/XMLReader.java @@ -161,8 +161,8 @@ public void close() { /** * This function processes the XML elements. This function stops reading when the - * limit (if any) which came from the query has been reached or the Iterator runs out of - * elements. + * limit (if any) which came from the query has been reached, a complete row has been + * read, or the Iterator runs out of elements. * @return True if there are more elements to parse, false if not */ private boolean processElements() { @@ -197,6 +197,13 @@ private boolean processElements() { // Process the event processEvent(currentEvent, lastEvent, reader.peek()); + + // After completing a row, return to let next() check batch capacity. + // This prevents batch overflow errors that occur when rows accumulate + // beyond what the batch can hold without the isFull() check running. + if (currentState == xmlState.ROW_ENDED) { + return true; + } } catch (XMLStreamException e) { throw UserException .dataReadError(e) diff --git a/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/TestCassandraSuite.java b/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/TestCassandraSuite.java index 1adc023b806..f3e1aa0b590 100644 --- a/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/TestCassandraSuite.java +++ b/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/TestCassandraSuite.java @@ -27,6 +27,7 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Suite; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.CassandraContainer; @Category(SlowTest.class) @@ -42,6 +43,10 @@ public class TestCassandraSuite extends BaseTest { @BeforeClass public static void initCassandra() { + org.junit.Assume.assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); synchronized (TestCassandraSuite.class) { if (initCount.get() == 0) { startCassandra(); diff --git a/contrib/storage-elasticsearch/src/test/java/org/apache/drill/exec/store/elasticsearch/TestElasticsearchSuite.java b/contrib/storage-elasticsearch/src/test/java/org/apache/drill/exec/store/elasticsearch/TestElasticsearchSuite.java index 258034e9fde..9e3d63f7094 100644 --- a/contrib/storage-elasticsearch/src/test/java/org/apache/drill/exec/store/elasticsearch/TestElasticsearchSuite.java +++ b/contrib/storage-elasticsearch/src/test/java/org/apache/drill/exec/store/elasticsearch/TestElasticsearchSuite.java @@ -39,6 +39,7 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Suite; +import org.testcontainers.DockerClientFactory; import org.testcontainers.elasticsearch.ElasticsearchContainer; import com.google.api.client.util.SslUtils; @@ -54,6 +55,8 @@ import java.time.Duration; import java.util.concurrent.atomic.AtomicInteger; +import static org.junit.Assume.assumeTrue; + @Category(SlowTest.class) @RunWith(Suite.class) @@ -76,6 +79,10 @@ public class TestElasticsearchSuite extends BaseTest { @BeforeClass public static void initElasticsearch() throws IOException, GeneralSecurityException { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); synchronized (TestElasticsearchSuite.class) { if (initCount.get() == 0) { startElasticsearch(); diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpSchemaFactory.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpSchemaFactory.java index ef6368d63cc..0fb293b47f7 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpSchemaFactory.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpSchemaFactory.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; @@ -112,6 +113,16 @@ public Table getTable(String name) { } } + @Override + public Set getTableNames() { + return tables.keySet(); + } + + @Override + public Set getSubSchemaNames() { + return subSchemas.keySet(); + } + @Override public String getTypeName() { return HttpStoragePluginConfig.NAME; diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithMySQL.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithMySQL.java index dadcf0b5bba..d3b481f1b70 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithMySQL.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithMySQL.java @@ -28,6 +28,7 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.ext.ScriptUtils; @@ -41,6 +42,7 @@ import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; @Category(JdbcStorageTest.class) public class TestJdbcInsertWithMySQL extends ClusterTest { @@ -51,6 +53,10 @@ public class TestJdbcInsertWithMySQL extends ClusterTest { @BeforeClass public static void initMysql() throws Exception { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); startCluster(ClusterFixture.builder(dirTestWatcher)); dirTestWatcher.copyResourceToRoot(Paths.get("")); diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithPostgres.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithPostgres.java index fcb66599480..f43090b4f4e 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithPostgres.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithPostgres.java @@ -26,6 +26,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; @@ -37,6 +38,7 @@ import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; @Category(JdbcStorageTest.class) public class TestJdbcInsertWithPostgres extends ClusterTest { @@ -46,6 +48,10 @@ public class TestJdbcInsertWithPostgres extends ClusterTest { @BeforeClass public static void initPostgres() throws Exception { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); startCluster(ClusterFixture.builder(dirTestWatcher)); dirTestWatcher.copyResourceToRoot(Paths.get("")); diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithClickhouse.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithClickhouse.java index 8b8f520615a..963bcab44a1 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithClickhouse.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithClickhouse.java @@ -33,6 +33,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.ClickHouseContainer; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.utility.DockerImageName; @@ -42,6 +43,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; /** * JDBC storage plugin tests against Clickhouse. @@ -56,6 +58,10 @@ public class TestJdbcPluginWithClickhouse extends ClusterTest { @BeforeClass public static void initClickhouse() throws Exception { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); startCluster(ClusterFixture.builder(dirTestWatcher)); String osName = System.getProperty("os.name").toLowerCase(); DockerImageName imageName; diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java index cd1e1b30e09..ed90733383b 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java @@ -35,6 +35,7 @@ import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.utility.DockerImageName; @@ -55,6 +56,10 @@ public class TestJdbcPluginWithMSSQL extends ClusterTest { @BeforeClass public static void initMSSQL() throws Exception { + Assume.assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); Assume.assumeTrue(System.getProperty("os.arch").matches("(amd64|x86_64)")); startCluster(ClusterFixture.builder(dirTestWatcher)); diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java index 11f5c4e64a1..f36b1f5a68c 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java @@ -34,6 +34,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.ext.ScriptUtils; @@ -58,6 +59,10 @@ public class TestJdbcPluginWithMySQLIT extends ClusterTest { @BeforeClass public static void initMysql() throws Exception { + Assume.assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); startCluster(ClusterFixture.builder(dirTestWatcher)); String osName = System.getProperty("os.name").toLowerCase(); String mysqlDBName = "drill_mysql_test"; diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithPostgres.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithPostgres.java index cfdd65899b2..2ad4c24e430 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithPostgres.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithPostgres.java @@ -32,6 +32,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.utility.DockerImageName; @@ -43,6 +44,7 @@ import java.util.TimeZone; import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; /** * JDBC storage plugin tests against Postgres. @@ -56,6 +58,10 @@ public class TestJdbcPluginWithPostgres extends ClusterTest { @BeforeClass public static void initPostgres() throws Exception { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); startCluster(ClusterFixture.builder(dirTestWatcher)); String postgresDBName = "drill_postgres_test"; TimeZone.setDefault(TimeZone.getTimeZone("UTC")); diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcUserTranslation.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcUserTranslation.java index a1086d9a733..4f43e526ece 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcUserTranslation.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcUserTranslation.java @@ -37,6 +37,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.ext.ScriptUtils; @@ -54,6 +55,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; @Category(JdbcStorageTest.class) public class TestJdbcUserTranslation extends ClusterTest { @@ -65,6 +67,10 @@ public class TestJdbcUserTranslation extends ClusterTest { @BeforeClass public static void initMysql() throws Exception { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); ClusterFixtureBuilder builder = new ClusterFixtureBuilder(dirTestWatcher) .configProperty(ExecConstants.HTTP_ENABLE, true) .configProperty(ExecConstants.HTTP_PORT_HUNT, true) diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcWriterWithMySQL.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcWriterWithMySQL.java index 95158989a19..aa03a05c7bb 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcWriterWithMySQL.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcWriterWithMySQL.java @@ -42,6 +42,7 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.ext.ScriptUtils; @@ -60,6 +61,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; @Category(JdbcStorageTest.class) public class TestJdbcWriterWithMySQL extends ClusterTest { @@ -70,6 +72,10 @@ public class TestJdbcWriterWithMySQL extends ClusterTest { @BeforeClass public static void initMysql() throws Exception { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); startCluster(ClusterFixture.builder(dirTestWatcher)); dirTestWatcher.copyResourceToRoot(Paths.get("")); diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcWriterWithPostgres.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcWriterWithPostgres.java index 43ca9a99284..c8a84dfaab1 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcWriterWithPostgres.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcWriterWithPostgres.java @@ -40,6 +40,7 @@ import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; @@ -56,6 +57,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; @Category(JdbcStorageTest.class) public class TestJdbcWriterWithPostgres extends ClusterTest { @@ -65,6 +67,10 @@ public class TestJdbcWriterWithPostgres extends ClusterTest { @BeforeClass public static void initPostgres() throws Exception { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); startCluster(ClusterFixture.builder(dirTestWatcher)); dirTestWatcher.copyResourceToRoot(Paths.get("")); diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuite.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuite.java index 543cc370b7a..bbde12d561e 100644 --- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuite.java +++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuite.java @@ -39,6 +39,7 @@ import org.junit.runners.Suite; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; @@ -54,6 +55,7 @@ import java.util.stream.Stream; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assume.assumeTrue; @RunWith(Suite.class) @Suite.SuiteClasses({ @@ -229,6 +231,10 @@ public String setup() throws IOException { @BeforeClass public static void initMongo() throws Exception { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); synchronized (MongoTestSuite.class) { if (initCount.get() == 0) { if (distMode) { diff --git a/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkTestSuite.java b/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkTestSuite.java index 8530270c1c5..c5446b2e8e8 100644 --- a/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkTestSuite.java +++ b/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkTestSuite.java @@ -28,12 +28,12 @@ import org.apache.drill.test.ClusterTest; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.ClassRule; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Suite; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; @@ -42,6 +42,7 @@ import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_1; import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_2; +import static org.junit.Assume.assumeTrue; @RunWith(Suite.class) @@ -107,23 +108,14 @@ private static java.io.File createDefaultYmlFile() { } } - @ClassRule - public static GenericContainer splunk = new GenericContainer<>( - DockerImageName.parse("splunk/splunk:9.3") - ) - .withExposedPorts(8089, 8089) - .withEnv("SPLUNK_START_ARGS", "--accept-license") - .withEnv("SPLUNK_PASSWORD", SPLUNK_PASS) - .withEnv("SPLUNKD_SSL_ENABLE", "false") - .withCopyFileToContainer( - org.testcontainers.utility.MountableFile.forHostPath( - createDefaultYmlFile().toPath() - ), - "/tmp/defaults/default.yml" - ); + private static GenericContainer splunk; @BeforeClass public static void initSplunk() throws Exception { + assumeTrue( + "Docker is not available, skipping container tests", + DockerClientFactory.instance().isDockerAvailable() + ); synchronized (SplunkTestSuite.class) { if (initCount.get() == 0) { ClusterFixtureBuilder builder = new ClusterFixtureBuilder(dirTestWatcher) @@ -132,6 +124,19 @@ public static void initSplunk() throws Exception { .configProperty(ExecConstants.IMPERSONATION_ENABLED, true); startCluster(builder); + splunk = new GenericContainer<>( + DockerImageName.parse("splunk/splunk:9.3") + ) + .withExposedPorts(8089, 8089) + .withEnv("SPLUNK_START_ARGS", "--accept-license") + .withEnv("SPLUNK_PASSWORD", SPLUNK_PASS) + .withEnv("SPLUNKD_SSL_ENABLE", "false") + .withCopyFileToContainer( + org.testcontainers.utility.MountableFile.forHostPath( + createDefaultYmlFile().toPath() + ), + "/tmp/defaults/default.yml" + ); splunk.start(); // Wait for Splunk to start and apply configuration from default.yml @@ -203,6 +208,9 @@ public static void initSplunk() throws Exception { * This should be called between test classes to prevent disk space exhaustion. */ public static void cleanDispatchDirectory() { + if (splunk == null) { + return; + } try { logger.info("Cleaning up Splunk dispatch directory..."); splunk.execInContainer("sh", "-c", "rm -rf /opt/splunk/var/run/splunk/dispatch/*"); @@ -215,7 +223,7 @@ public static void cleanDispatchDirectory() { @AfterClass public static void tearDownCluster() { synchronized (SplunkTestSuite.class) { - if (initCount.decrementAndGet() == 0) { + if (initCount.decrementAndGet() == 0 && splunk != null) { // Clean up Splunk dispatch files to free disk space before shutdown cleanDispatchDirectory(); splunk.close(); diff --git a/docs/dev/PROSPECTOR.md b/docs/dev/PROSPECTOR.md new file mode 100644 index 00000000000..5aa4964e19a --- /dev/null +++ b/docs/dev/PROSPECTOR.md @@ -0,0 +1,242 @@ +# Prospector - AI Assistant for Apache Drill SQL Lab + +## Overview + +Prospector is an integrated chat-based AI assistant for SQL Lab that helps users explore data, generate SQL queries, create visualizations, and build dashboards using natural language. It connects to LLM providers (OpenAI, Anthropic Claude, Ollama, etc.) through a backend proxy that keeps API keys secure. + +## Architecture + +``` +Browser (React) Drill Backend (Java) LLM Provider ++-------------------+ +----------------------+ +------------+ +| ProspectorDrawer |--POST SSE-->| ProspectorResources |--HTTP--> | OpenAI | +| useProspector hook| | LlmProviderRegistry | | Anthropic | +| Tool Executor | | OpenAiCompatProvider | | Ollama | +| Quick Actions | | AnthropicProvider | +------------+ ++-------------------+ | AiConfigResources | + | PersistentStore | + +----------------------+ +``` + +**Key design decisions:** +- Backend proxy holds API keys; the frontend never sees them +- SSE streaming via JAX-RS `StreamingOutput` for real-time responses +- Tool calls are returned to the frontend for execution using existing API functions +- No new Maven or npm dependencies (uses OkHttp, Jackson, react-markdown already in the project) +- Configuration stored in `PersistentStoreProvider` (same pattern as visualizations/dashboards) + +## Setup & Configuration + +### 1. Access Prospector Settings + +Click the robot icon in the top navigation bar to open the Prospector Settings modal. + +### 2. Configure a Provider + +| Provider | API Endpoint | API Key Required | Example Models | +|----------|-------------|-----------------|----------------| +| **OpenAI Compatible** | `https://api.openai.com/v1` (default) | Yes | `gpt-4o`, `gpt-4o-mini` | +| **Anthropic Claude** | `https://api.anthropic.com` (default) | Yes | `claude-sonnet-4-20250514`, `claude-haiku-4-5-20251001` | +| **Ollama (local)** | `http://localhost:11434/v1` | No | `llama3`, `mistral`, `codellama` | +| **Azure OpenAI** | Your Azure endpoint | Yes | Your deployed model name | + +### 3. Required Settings + +- **Provider**: Select from the dropdown +- **API Key**: Enter your API key (stored securely on the server, never sent to the browser) +- **Model**: Enter the model name +- **Enable Prospector**: Toggle on + +### 4. Optional Settings + +- **API Endpoint**: Override the default endpoint (useful for Ollama, Azure, or custom proxies) +- **Max Tokens**: Maximum response length (default: 4096) +- **Temperature**: Controls randomness (0 = deterministic, 2 = creative; default: 0.7) +- **Custom System Prompt**: Additional instructions appended to the default system prompt + +### 5. Test Connection + +Click "Test Connection" to validate your configuration before saving. + +## Usage + +### Opening Prospector + +Click the floating AI button in the bottom-right corner of SQL Lab. The Prospector drawer opens on the right side. + +### Asking Questions + +Type a question or request in the chat input and press Enter (or click Send). Examples: + +- "What tables are available in the dfs.tmp schema?" +- "Write a query to find the top 10 customers by revenue" +- "Explain this SQL query" +- "Create a bar chart showing sales by month" +- "Fix the error in my query" + +### Quick Actions + +Quick action buttons appear above the chat input: + +- **Generate SQL**: Ask Prospector to generate a SQL query +- **Explain Query**: Explain the SQL currently in the editor (shown when editor has SQL) +- **Fix Error**: Fix the current query error (shown when there's an error) +- **Suggest Chart**: Suggest a visualization for the current results (shown when results exist) + +### Tool Capabilities + +Prospector has access to the following tools: + +| Tool | Description | +|------|-------------| +| `execute_sql` | Execute SQL queries against Drill | +| `get_schema_info` | Browse schemas, tables, and columns | +| `create_visualization` | Create chart visualizations | +| `create_dashboard` | Create dashboards | +| `save_query` | Save SQL queries | +| `get_available_functions` | List Drill SQL functions | + +When Prospector uses a tool, you'll see a collapsible panel showing the tool name, arguments, and results. Tool calls are executed automatically and Prospector uses the results to continue the conversation. + +### Conversation Context + +Prospector automatically receives context about your current state: +- The SQL in the editor +- The selected schema +- Available schemas +- Current query error (if any) +- Query result summary (row count, column names and types) + +## REST API Reference + +### Status Endpoint + +``` +GET /api/v1/ai/status +``` + +Returns whether Prospector is enabled and configured. + +**Response:** +```json +{ + "enabled": true, + "configured": true +} +``` + +### Chat Endpoint + +``` +POST /api/v1/ai/chat +Content-Type: application/json +``` + +Streams AI responses via Server-Sent Events (SSE). + +**Request body:** +```json +{ + "messages": [ + { "role": "user", "content": "What tables are in dfs.tmp?" } + ], + "tools": [...], + "context": { + "currentSql": "SELECT * FROM ...", + "currentSchema": "dfs.tmp", + "availableSchemas": ["dfs", "dfs.tmp", "sys"], + "error": null, + "resultSummary": { "rowCount": 10, "columns": ["id", "name"], "columnTypes": ["INT", "VARCHAR"] } + } +} +``` + +**SSE Events:** +``` +event: delta +data: {"type":"content","content":"The tables in dfs.tmp are..."} + +event: delta +data: {"type":"tool_call_start","id":"call_1","name":"get_schema_info"} + +event: delta +data: {"type":"tool_call_delta","id":"call_1","arguments":"{\"schema\":\"dfs.tmp\"}"} + +event: delta +data: {"type":"tool_call_end","id":"call_1"} + +event: done +data: {"finish_reason":"tool_calls"} + +event: error +data: {"message":"API key invalid"} +``` + +### Configuration Endpoints (Admin Only) + +``` +GET /api/v1/ai/config - Get config (API key redacted) +PUT /api/v1/ai/config - Update config (partial updates supported) +POST /api/v1/ai/config/test - Test configuration +GET /api/v1/ai/config/providers - List available providers +``` + +## File Structure + +### Backend (Java) + +``` +exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ + ai/ + LlmProvider.java - Provider interface + LlmConfig.java - Config POJO (PersistentStore) + ChatMessage.java - Normalized chat message + ToolDefinition.java - Tool schema definition + ToolCall.java - Tool call from LLM + ValidationResult.java - Config validation result + OpenAiCompatibleProvider.java - OpenAI/Ollama provider + AnthropicProvider.java - Anthropic Claude provider + LlmProviderRegistry.java - Provider registry + ProspectorResources.java - Chat SSE endpoint + AiConfigResources.java - Admin config endpoint +``` + +### Frontend (TypeScript/React) + +``` +webapp/src/ + types/ai.ts - TypeScript type definitions + api/ai.ts - API client functions + hooks/useProspector.ts - Core chat hook with tool execution + components/prospector/ + ProspectorDrawer.tsx - Main drawer component + ChatMessageList.tsx - Scrollable message list + ChatMessageBubble.tsx - Individual message with markdown + ToolCallDisplay.tsx - Collapsible tool call panels + QuickActionBar.tsx - Quick action buttons + ChatInput.tsx - Text input with send/stop + ProspectorButton.tsx - Floating action button + ProspectorSettingsModal.tsx - Admin settings dialog + index.ts - Barrel exports +``` + +## Troubleshooting + +### Prospector button is disabled +- Check that Prospector is enabled in settings +- Verify an API key and model are configured +- Check the Drill logs for configuration errors + +### "Prospector is not enabled" error +- Open Prospector Settings and ensure "Enable Prospector" is toggled on +- Make sure you've saved the configuration after enabling + +### Streaming stops or errors +- Check that the API key is valid +- For Ollama, verify the server is running and the model is pulled +- Check Drill server logs for detailed error messages + +### Tool calls fail +- Ensure the Drill cluster is running and accessible +- Check that schemas referenced in queries exist +- Verify the user has permissions for the operations diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index fe2c229a9a0..58f7d026798 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -681,6 +681,18 @@ ${testcontainers.version} test + + + com.gtkcyber.sqlglot + sqlglot-core + 1.0.1 + + + com.gtkcyber.sqlglot + sqlglot-dialects + 1.0.1 + + io.swagger.core.v3 swagger-jaxrs2-jakarta @@ -849,6 +861,54 @@ org.apache.maven.plugins maven-jar-plugin + + + + com.github.eirslett + frontend-maven-plugin + 1.15.0 + + ${project.basedir}/src/main/resources/webapp + ${project.build.directory} + + + + + install-node-and-npm + generate-resources + + install-node-and-npm + + + v20.10.0 + 10.2.3 + + + + + npm-install + generate-resources + + npm + + + ci --legacy-peer-deps + + + + + npm-build + generate-resources + + npm + + + run build + + + + + maven-resources-plugin @@ -866,6 +926,20 @@ + + copy-webapp-dist + process-resources + copy-resources + + ${project.build.outputDirectory}/webapp/dist + + + ${project.basedir}/src/main/resources/webapp/dist + false + + + + diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/AiConfigResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/AiConfigResources.java new file mode 100644 index 00000000000..b138bfdb45f --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/AiConfigResources.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.exec.exception.StoreException; +import org.apache.drill.exec.server.rest.ai.LlmConfig; +import org.apache.drill.exec.server.rest.ai.LlmProvider; +import org.apache.drill.exec.server.rest.ai.LlmProviderRegistry; +import org.apache.drill.exec.server.rest.ai.ValidationResult; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; +import org.apache.drill.exec.store.sys.PersistentStore; +import org.apache.drill.exec.store.sys.PersistentStoreConfig; +import org.apache.drill.exec.store.sys.PersistentStoreProvider; +import org.apache.drill.exec.work.WorkManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.annotation.security.RolesAllowed; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.List; + +/** + * REST resource for managing Prospector AI configuration. + * Admin-only access. API keys are stored securely and redacted in GET responses. + */ +@Path("/api/v1/ai/config") +@Tag(name = "Prospector Configuration", description = "Admin configuration for Prospector AI") +@RolesAllowed(DrillUserPrincipal.ADMIN_ROLE) +public class AiConfigResources { + + private static final Logger logger = LoggerFactory.getLogger(AiConfigResources.class); + private static final String CONFIG_STORE_NAME = "drill.sqllab.ai_config"; + private static final String CONFIG_KEY = "default"; + + @Inject + WorkManager workManager; + + @Inject + PersistentStoreProvider storeProvider; + + private static volatile PersistentStore cachedStore; + + // ==================== Response Models ==================== + + public static class ConfigResponse { + @JsonProperty + public String provider; + + @JsonProperty + public String apiEndpoint; + + @JsonProperty + public boolean apiKeySet; + + @JsonProperty + public String model; + + @JsonProperty + public int maxTokens; + + @JsonProperty + public double temperature; + + @JsonProperty + public boolean enabled; + + @JsonProperty + public String systemPrompt; + + @JsonProperty + public boolean sendDataToAi; + + public ConfigResponse() { + } + + public ConfigResponse(LlmConfig config) { + this.provider = config.getProvider(); + this.apiEndpoint = config.getApiEndpoint(); + this.apiKeySet = config.getApiKey() != null && !config.getApiKey().isEmpty(); + this.model = config.getModel(); + this.maxTokens = config.getMaxTokens(); + this.temperature = config.getTemperature(); + this.enabled = config.isEnabled(); + this.systemPrompt = config.getSystemPrompt(); + this.sendDataToAi = config.isSendDataToAi(); + } + } + + public static class UpdateConfigRequest { + @JsonProperty + public String provider; + + @JsonProperty + public String apiEndpoint; + + @JsonProperty + public String apiKey; + + @JsonProperty + public String model; + + @JsonProperty + public Integer maxTokens; + + @JsonProperty + public Double temperature; + + @JsonProperty + public Boolean enabled; + + @JsonProperty + public String systemPrompt; + + @JsonProperty + public Boolean sendDataToAi; + + public UpdateConfigRequest() { + } + + @JsonCreator + public UpdateConfigRequest( + @JsonProperty("provider") String provider, + @JsonProperty("apiEndpoint") String apiEndpoint, + @JsonProperty("apiKey") String apiKey, + @JsonProperty("model") String model, + @JsonProperty("maxTokens") Integer maxTokens, + @JsonProperty("temperature") Double temperature, + @JsonProperty("enabled") Boolean enabled, + @JsonProperty("systemPrompt") String systemPrompt, + @JsonProperty("sendDataToAi") Boolean sendDataToAi) { + this.provider = provider; + this.apiEndpoint = apiEndpoint; + this.apiKey = apiKey; + this.model = model; + this.maxTokens = maxTokens; + this.temperature = temperature; + this.enabled = enabled; + this.systemPrompt = systemPrompt; + this.sendDataToAi = sendDataToAi; + } + } + + public static class ProviderInfo { + @JsonProperty + public String id; + + @JsonProperty + public String displayName; + + public ProviderInfo(String id, String displayName) { + this.id = id; + this.displayName = displayName; + } + } + + public static class ProvidersResponse { + @JsonProperty + public List providers; + + public ProvidersResponse(List providers) { + this.providers = providers; + } + } + + public static class MessageResponse { + @JsonProperty + public String message; + + public MessageResponse(String message) { + this.message = message; + } + } + + // ==================== Endpoints ==================== + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get AI configuration", + description = "Returns AI configuration with API key redacted") + public Response getConfig() { + logger.debug("Getting AI configuration"); + try { + PersistentStore store = getStore(); + LlmConfig config = store.get(CONFIG_KEY); + + if (config == null) { + return Response.ok(new ConfigResponse(new LlmConfig())).build(); + } + + return Response.ok(new ConfigResponse(config)).build(); + } catch (Exception e) { + logger.error("Error reading AI config", e); + throw new DrillRuntimeException("Failed to read AI configuration: " + e.getMessage(), e); + } + } + + @PUT + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Update AI configuration", + description = "Updates AI configuration. Supports partial updates.") + public Response updateConfig(UpdateConfigRequest request) { + logger.debug("Updating AI configuration"); + + try { + PersistentStore store = getStore(); + LlmConfig existing = store.get(CONFIG_KEY); + if (existing == null) { + existing = new LlmConfig(); + } + + // Apply partial updates + if (request.provider != null) { + existing.setProvider(request.provider); + } + if (request.apiEndpoint != null) { + existing.setApiEndpoint(request.apiEndpoint); + } + // Only update API key if provided and non-empty + if (request.apiKey != null && !request.apiKey.isEmpty()) { + existing.setApiKey(request.apiKey); + } + if (request.model != null) { + existing.setModel(request.model); + } + if (request.maxTokens != null) { + existing.setMaxTokens(request.maxTokens); + } + if (request.temperature != null) { + existing.setTemperature(request.temperature); + } + if (request.enabled != null) { + existing.setEnabled(request.enabled); + } + if (request.systemPrompt != null) { + existing.setSystemPrompt(request.systemPrompt); + } + if (request.sendDataToAi != null) { + existing.setSendDataToAi(request.sendDataToAi); + } + + store.put(CONFIG_KEY, existing); + + return Response.ok(new ConfigResponse(existing)).build(); + } catch (Exception e) { + logger.error("Error updating AI config", e); + throw new DrillRuntimeException("Failed to update AI configuration: " + e.getMessage(), e); + } + } + + @POST + @Path("/test") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Test AI configuration", + description = "Validates the configuration and tests connectivity to the LLM provider") + public Response testConfig(UpdateConfigRequest request) { + logger.debug("Testing AI configuration"); + + try { + // Build a config for testing + PersistentStore store = getStore(); + LlmConfig existing = store.get(CONFIG_KEY); + + LlmConfig testConfig = new LlmConfig(); + testConfig.setProvider(request.provider != null ? request.provider + : (existing != null ? existing.getProvider() : "openai")); + testConfig.setApiEndpoint(request.apiEndpoint != null ? request.apiEndpoint + : (existing != null ? existing.getApiEndpoint() : null)); + testConfig.setModel(request.model != null ? request.model + : (existing != null ? existing.getModel() : null)); + testConfig.setMaxTokens(request.maxTokens != null ? request.maxTokens : 100); + testConfig.setTemperature(request.temperature != null ? request.temperature : 0.7); + + // Use provided API key, or fall back to existing + if (request.apiKey != null && !request.apiKey.isEmpty()) { + testConfig.setApiKey(request.apiKey); + } else if (existing != null) { + testConfig.setApiKey(existing.getApiKey()); + } + + LlmProvider provider = LlmProviderRegistry.get(testConfig.getProvider()); + if (provider == null) { + return Response.ok(ValidationResult.error("Unknown provider: " + testConfig.getProvider())).build(); + } + + ValidationResult result = provider.validateConfig(testConfig); + return Response.ok(result).build(); + } catch (Exception e) { + logger.error("Error testing AI config", e); + return Response.ok(ValidationResult.error("Test failed: " + e.getMessage())).build(); + } + } + + @GET + @Path("/providers") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List available LLM providers", + description = "Returns the list of supported LLM providers") + public ProvidersResponse getProviders() { + List providers = new ArrayList<>(); + for (LlmProvider provider : LlmProviderRegistry.getAll()) { + providers.add(new ProviderInfo(provider.getId(), provider.getDisplayName())); + } + return new ProvidersResponse(providers); + } + + // ==================== Helper Methods ==================== + + private PersistentStore getStore() { + if (cachedStore == null) { + synchronized (AiConfigResources.class) { + if (cachedStore == null) { + try { + cachedStore = storeProvider.getOrCreateStore( + PersistentStoreConfig.newJacksonBuilder( + workManager.getContext().getLpPersistence().getMapper(), + LlmConfig.class + ) + .name(CONFIG_STORE_NAME) + .build() + ); + } catch (StoreException e) { + throw new DrillRuntimeException("Failed to access AI config store", e); + } + } + } + } + return cachedStore; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DashboardResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DashboardResources.java new file mode 100644 index 00000000000..905616123e3 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DashboardResources.java @@ -0,0 +1,1141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.exception.StoreException; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; +import org.apache.drill.exec.store.sys.PersistentStore; +import org.apache.drill.exec.store.sys.PersistentStoreConfig; +import org.apache.drill.exec.store.sys.PersistentStoreProvider; +import org.apache.drill.exec.work.WorkManager; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.annotation.security.RolesAllowed; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +/** + * REST API for managing dashboards. + * Dashboards combine multiple visualizations into interactive drag-and-drop layouts. + */ +@Path("/api/v1/dashboards") +@Tag(name = "Dashboards", description = "APIs for managing interactive dashboards") +@RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE) +public class DashboardResources { + private static final Logger logger = LoggerFactory.getLogger(DashboardResources.class); + private static final String STORE_NAME = "drill.sqllab.dashboards"; + private static final String FAVORITES_STORE_NAME = "drill.sqllab.dashboard_favorites"; + private static final String UPLOAD_DIR_NAME = "dashboard-images"; + private static final long MAX_FILE_SIZE = 5 * 1024 * 1024; // 5 MB + private static final Set ALLOWED_EXTENSIONS = new HashSet<>( + Arrays.asList("jpg", "jpeg", "png", "gif", "svg", "webp")); + + @Inject + WorkManager workManager; + + @Inject + DrillUserPrincipal principal; + + @Inject + PersistentStoreProvider storeProvider; + + private static volatile PersistentStore cachedStore; + private static volatile PersistentStore cachedFavoritesStore; + private static volatile File cachedUploadDir; + + // ==================== Model Classes ==================== + + /** + * Theme configuration for a dashboard's visual appearance. + */ + public static class DashboardTheme { + @JsonProperty + private String mode; + @JsonProperty + private String fontFamily; + @JsonProperty + private String backgroundColor; + @JsonProperty + private String fontColor; + @JsonProperty + private String panelBackground; + @JsonProperty + private String panelBorderColor; + @JsonProperty + private String panelBorderRadius; + @JsonProperty + private String accentColor; + @JsonProperty + private String headerColor; + + public DashboardTheme() { + } + + @JsonCreator + public DashboardTheme( + @JsonProperty("mode") String mode, + @JsonProperty("fontFamily") String fontFamily, + @JsonProperty("backgroundColor") String backgroundColor, + @JsonProperty("fontColor") String fontColor, + @JsonProperty("panelBackground") String panelBackground, + @JsonProperty("panelBorderColor") String panelBorderColor, + @JsonProperty("panelBorderRadius") String panelBorderRadius, + @JsonProperty("accentColor") String accentColor, + @JsonProperty("headerColor") String headerColor) { + this.mode = mode; + this.fontFamily = fontFamily; + this.backgroundColor = backgroundColor; + this.fontColor = fontColor; + this.panelBackground = panelBackground; + this.panelBorderColor = panelBorderColor; + this.panelBorderRadius = panelBorderRadius; + this.accentColor = accentColor; + this.headerColor = headerColor; + } + + public String getMode() { + return mode; + } + + public String getFontFamily() { + return fontFamily; + } + + public String getBackgroundColor() { + return backgroundColor; + } + + public String getFontColor() { + return fontColor; + } + + public String getPanelBackground() { + return panelBackground; + } + + public String getPanelBorderColor() { + return panelBorderColor; + } + + public String getPanelBorderRadius() { + return panelBorderRadius; + } + + public String getAccentColor() { + return accentColor; + } + + public String getHeaderColor() { + return headerColor; + } + + public void setMode(String mode) { + this.mode = mode; + } + + public void setFontFamily(String fontFamily) { + this.fontFamily = fontFamily; + } + + public void setBackgroundColor(String backgroundColor) { + this.backgroundColor = backgroundColor; + } + + public void setFontColor(String fontColor) { + this.fontColor = fontColor; + } + + public void setPanelBackground(String panelBackground) { + this.panelBackground = panelBackground; + } + + public void setPanelBorderColor(String panelBorderColor) { + this.panelBorderColor = panelBorderColor; + } + + public void setPanelBorderRadius(String panelBorderRadius) { + this.panelBorderRadius = panelBorderRadius; + } + + public void setAccentColor(String accentColor) { + this.accentColor = accentColor; + } + + public void setHeaderColor(String headerColor) { + this.headerColor = headerColor; + } + } + + /** + * Stores a user's list of favorited dashboard IDs. + */ + public static class UserFavorites { + @JsonProperty + private List dashboardIds; + + public UserFavorites() { + this.dashboardIds = new ArrayList<>(); + } + + @JsonCreator + public UserFavorites( + @JsonProperty("dashboardIds") List dashboardIds) { + this.dashboardIds = dashboardIds != null ? dashboardIds : new ArrayList<>(); + } + + public List getDashboardIds() { + return dashboardIds; + } + + public void setDashboardIds(List dashboardIds) { + this.dashboardIds = dashboardIds; + } + } + + /** + * Response for favorite toggle operations. + */ + public static class FavoriteResponse { + @JsonProperty + public boolean favorited; + @JsonProperty + public String message; + + public FavoriteResponse(boolean favorited, String message) { + this.favorited = favorited; + this.message = message; + } + } + + /** + * Response containing a list of favorited dashboard IDs. + */ + public static class FavoritesListResponse { + @JsonProperty + public List dashboardIds; + + public FavoritesListResponse(List dashboardIds) { + this.dashboardIds = dashboardIds; + } + } + + /** + * A panel within a dashboard, referencing a visualization or content and its layout position. + */ + public static class DashboardPanel { + @JsonProperty + private String id; + @JsonProperty + private String type; + @JsonProperty + private String visualizationId; + @JsonProperty + private String content; + @JsonProperty + private Map config; + @JsonProperty + private String tabId; + @JsonProperty + private int x; + @JsonProperty + private int y; + @JsonProperty + private int width; + @JsonProperty + private int height; + + public DashboardPanel() { + } + + @JsonCreator + public DashboardPanel( + @JsonProperty("id") String id, + @JsonProperty("type") String type, + @JsonProperty("visualizationId") String visualizationId, + @JsonProperty("content") String content, + @JsonProperty("config") Map config, + @JsonProperty("tabId") String tabId, + @JsonProperty("x") int x, + @JsonProperty("y") int y, + @JsonProperty("width") int width, + @JsonProperty("height") int height) { + this.id = id; + this.type = type != null ? type : "visualization"; + this.visualizationId = visualizationId; + this.content = content; + this.config = config; + this.tabId = tabId; + this.x = x; + this.y = y; + this.width = width; + this.height = height; + } + + public String getId() { + return id; + } + + public String getType() { + return type; + } + + public String getVisualizationId() { + return visualizationId; + } + + public String getContent() { + return content; + } + + public Map getConfig() { + return config; + } + + public String getTabId() { + return tabId; + } + + public int getX() { + return x; + } + + public int getY() { + return y; + } + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public void setId(String id) { + this.id = id; + } + + public void setType(String type) { + this.type = type; + } + + public void setVisualizationId(String visualizationId) { + this.visualizationId = visualizationId; + } + + public void setContent(String content) { + this.content = content; + } + + public void setConfig(Map config) { + this.config = config; + } + + public void setTabId(String tabId) { + this.tabId = tabId; + } + + public void setX(int x) { + this.x = x; + } + + public void setY(int y) { + this.y = y; + } + + public void setWidth(int width) { + this.width = width; + } + + public void setHeight(int height) { + this.height = height; + } + } + + /** + * A tab within a dashboard for organizing panels into groups. + */ + public static class DashboardTab { + @JsonProperty + private String id; + @JsonProperty + private String name; + @JsonProperty + private int order; + + public DashboardTab() { + } + + @JsonCreator + public DashboardTab( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("order") int order) { + this.id = id; + this.name = name; + this.order = order; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public int getOrder() { + return order; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setOrder(int order) { + this.order = order; + } + } + + /** + * Dashboard model for persistence. + */ + public static class Dashboard { + @JsonProperty + private String id; + @JsonProperty + private String name; + @JsonProperty + private String description; + @JsonProperty + private List panels; + @JsonProperty + private List tabs; + @JsonProperty + private DashboardTheme theme; + @JsonProperty + private String owner; + @JsonProperty + private long createdAt; + @JsonProperty + private long updatedAt; + @JsonProperty + private int refreshInterval; + @JsonProperty + private boolean isPublic; + + public Dashboard() { + } + + @JsonCreator + public Dashboard( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("description") String description, + @JsonProperty("panels") List panels, + @JsonProperty("tabs") List tabs, + @JsonProperty("theme") DashboardTheme theme, + @JsonProperty("owner") String owner, + @JsonProperty("createdAt") long createdAt, + @JsonProperty("updatedAt") long updatedAt, + @JsonProperty("refreshInterval") int refreshInterval, + @JsonProperty("isPublic") boolean isPublic) { + this.id = id; + this.name = name; + this.description = description; + this.panels = panels; + this.tabs = tabs; + this.theme = theme; + this.owner = owner; + this.createdAt = createdAt; + this.updatedAt = updatedAt; + this.refreshInterval = refreshInterval; + this.isPublic = isPublic; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public List getPanels() { + return panels; + } + + public List getTabs() { + return tabs; + } + + public DashboardTheme getTheme() { + return theme; + } + + public String getOwner() { + return owner; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getUpdatedAt() { + return updatedAt; + } + + public int getRefreshInterval() { + return refreshInterval; + } + + public boolean isPublic() { + return isPublic; + } + + public void setName(String name) { + this.name = name; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setPanels(List panels) { + this.panels = panels; + } + + public void setTabs(List tabs) { + this.tabs = tabs; + } + + public void setTheme(DashboardTheme theme) { + this.theme = theme; + } + + public void setUpdatedAt(long updatedAt) { + this.updatedAt = updatedAt; + } + + public void setRefreshInterval(int refreshInterval) { + this.refreshInterval = refreshInterval; + } + + public void setPublic(boolean isPublic) { + this.isPublic = isPublic; + } + } + + /** + * Request body for creating a new dashboard. + */ + public static class CreateDashboardRequest { + @JsonProperty + public String name; + @JsonProperty + public String description; + @JsonProperty + public List panels; + @JsonProperty + public List tabs; + @JsonProperty + public DashboardTheme theme; + @JsonProperty + public int refreshInterval; + @JsonProperty + public boolean isPublic; + } + + /** + * Request body for updating a dashboard. + */ + public static class UpdateDashboardRequest { + @JsonProperty + public String name; + @JsonProperty + public String description; + @JsonProperty + public List panels; + @JsonProperty + public List tabs; + @JsonProperty + public DashboardTheme theme; + @JsonProperty + public Integer refreshInterval; + @JsonProperty + public Boolean isPublic; + } + + /** + * Response containing a list of dashboards. + */ + public static class DashboardsResponse { + @JsonProperty + public List dashboards; + + public DashboardsResponse(List dashboards) { + this.dashboards = dashboards; + } + } + + /** + * Simple message response. + */ + public static class MessageResponse { + @JsonProperty + public String message; + + public MessageResponse(String message) { + this.message = message; + } + } + + /** + * Response for image upload operations. + */ + public static class ImageUploadResponse { + @JsonProperty + public String url; + @JsonProperty + public String filename; + + public ImageUploadResponse(String url, String filename) { + this.url = url; + this.filename = filename; + } + } + + // ==================== API Endpoints ==================== + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List dashboards", description = "Returns all dashboards accessible by the current user") + public DashboardsResponse listDashboards() { + logger.debug("Listing dashboards for user: {}", getCurrentUser()); + + List dashboards = new ArrayList<>(); + String currentUser = getCurrentUser(); + + try { + PersistentStore store = getStore(); + Iterator> iterator = store.getAll(); + + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + Dashboard dashboard = entry.getValue(); + + // Return dashboards owned by user or public dashboards + if (dashboard.getOwner().equals(currentUser) || dashboard.isPublic()) { + dashboards.add(dashboard); + } + } + } catch (Exception e) { + logger.error("Error listing dashboards", e); + throw new DrillRuntimeException("Failed to list dashboards: " + e.getMessage(), e); + } + + return new DashboardsResponse(dashboards); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Create dashboard", description = "Creates a new dashboard") + public Response createDashboard(CreateDashboardRequest request) { + logger.debug("Creating dashboard: {}", request.name); + + if (request.name == null || request.name.trim().isEmpty()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("Dashboard name is required")) + .build(); + } + + String id = UUID.randomUUID().toString(); + long now = Instant.now().toEpochMilli(); + + Dashboard dashboard = new Dashboard( + id, + request.name.trim(), + request.description, + request.panels != null ? request.panels : new ArrayList<>(), + request.tabs, + request.theme, + getCurrentUser(), + now, + now, + request.refreshInterval, + request.isPublic + ); + + try { + PersistentStore store = getStore(); + store.put(id, dashboard); + } catch (Exception e) { + logger.error("Error creating dashboard", e); + throw new DrillRuntimeException("Failed to create dashboard: " + e.getMessage(), e); + } + + return Response.status(Response.Status.CREATED).entity(dashboard).build(); + } + + @GET + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get dashboard", description = "Returns a dashboard by ID") + public Response getDashboard( + @Parameter(description = "Dashboard ID") @PathParam("id") String id) { + logger.debug("Getting dashboard: {}", id); + + try { + PersistentStore store = getStore(); + Dashboard dashboard = store.get(id); + + if (dashboard == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Dashboard not found")) + .build(); + } + + // Check access permissions + if (!dashboard.getOwner().equals(getCurrentUser()) && !dashboard.isPublic()) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Access denied")) + .build(); + } + + return Response.ok(dashboard).build(); + } catch (Exception e) { + logger.error("Error getting dashboard", e); + throw new DrillRuntimeException("Failed to get dashboard: " + e.getMessage(), e); + } + } + + @PUT + @Path("/{id}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Update dashboard", description = "Updates an existing dashboard") + public Response updateDashboard( + @Parameter(description = "Dashboard ID") @PathParam("id") String id, + UpdateDashboardRequest request) { + logger.debug("Updating dashboard: {}", id); + + try { + PersistentStore store = getStore(); + Dashboard dashboard = store.get(id); + + if (dashboard == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Dashboard not found")) + .build(); + } + + // Only owner can update + if (!dashboard.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can update this dashboard")) + .build(); + } + + // Update fields if provided + if (request.name != null) { + dashboard.setName(request.name.trim()); + } + if (request.description != null) { + dashboard.setDescription(request.description); + } + if (request.panels != null) { + dashboard.setPanels(request.panels); + } + if (request.tabs != null) { + dashboard.setTabs(request.tabs); + } + if (request.theme != null) { + dashboard.setTheme(request.theme); + } + if (request.refreshInterval != null) { + dashboard.setRefreshInterval(request.refreshInterval); + } + if (request.isPublic != null) { + dashboard.setPublic(request.isPublic); + } + + dashboard.setUpdatedAt(Instant.now().toEpochMilli()); + + store.put(id, dashboard); + + return Response.ok(dashboard).build(); + } catch (Exception e) { + logger.error("Error updating dashboard", e); + throw new DrillRuntimeException("Failed to update dashboard: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Delete dashboard", description = "Deletes a dashboard") + public Response deleteDashboard( + @Parameter(description = "Dashboard ID") @PathParam("id") String id) { + logger.debug("Deleting dashboard: {}", id); + + try { + PersistentStore store = getStore(); + Dashboard dashboard = store.get(id); + + if (dashboard == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Dashboard not found")) + .build(); + } + + // Only owner can delete + if (!dashboard.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can delete this dashboard")) + .build(); + } + + store.delete(id); + + return Response.ok(new MessageResponse("Dashboard deleted successfully")).build(); + } catch (Exception e) { + logger.error("Error deleting dashboard", e); + throw new DrillRuntimeException("Failed to delete dashboard: " + e.getMessage(), e); + } + } + + // ==================== Image Upload Endpoints ==================== + + @POST + @Path("/upload-image") + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Upload image", description = "Uploads an image file for use in dashboard panels") + public Response uploadImage( + @FormDataParam("file") InputStream fileInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail) { + + if (fileInputStream == null || fileDetail == null || fileDetail.getFileName() == null) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("No file provided")) + .build(); + } + + String originalFilename = fileDetail.getFileName(); + String ext = getFileExtension(originalFilename); + + if (!ALLOWED_EXTENSIONS.contains(ext)) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse( + "Invalid file type. Allowed: " + String.join(", ", ALLOWED_EXTENSIONS))) + .build(); + } + + String storedFilename = UUID.randomUUID() + "." + ext; + File uploadDir = getUploadDir(); + File targetFile = new File(uploadDir, storedFilename); + + try { + long totalBytes = 0; + byte[] buffer = new byte[8192]; + int bytesRead; + + try (FileOutputStream fos = new FileOutputStream(targetFile)) { + while ((bytesRead = fileInputStream.read(buffer)) != -1) { + totalBytes += bytesRead; + if (totalBytes > MAX_FILE_SIZE) { + fos.close(); + if (!targetFile.delete()) { + logger.warn("Failed to delete oversized upload: {}", targetFile); + } + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("File exceeds maximum size of 5 MB")) + .build(); + } + fos.write(buffer, 0, bytesRead); + } + } + } catch (IOException e) { + if (targetFile.exists() && !targetFile.delete()) { + logger.warn("Failed to clean up partial upload: {}", targetFile); + } + logger.error("Error uploading image", e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(new MessageResponse("Failed to upload image: " + e.getMessage())) + .build(); + } + + String url = "/api/v1/dashboards/images/" + storedFilename; + return Response.status(Response.Status.CREATED) + .entity(new ImageUploadResponse(url, originalFilename)) + .build(); + } + + @GET + @Path("/images/{filename}") + @Operation(summary = "Get uploaded image", description = "Serves a previously uploaded dashboard image") + public Response getImage( + @Parameter(description = "Image filename") + @PathParam("filename") String filename) { + + // Strict validation: UUID + allowed extension only + if (!filename.matches("[a-f0-9\\-]+\\.(jpg|jpeg|png|gif|svg|webp)")) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("Invalid filename")) + .build(); + } + + // Use java.nio.file.Path normalization to prevent path traversal + File uploadDir = getUploadDir(); + java.nio.file.Path basePath = uploadDir.toPath(); + java.nio.file.Path resolvedPath = basePath.resolve(filename).normalize(); + if (!resolvedPath.startsWith(basePath)) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("Invalid filename")) + .build(); + } + File imageFile = resolvedPath.toFile(); + + if (!imageFile.exists()) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Image not found")) + .build(); + } + + String ext = getFileExtension(filename); + String contentType; + switch (ext) { + case "jpg": + case "jpeg": + contentType = "image/jpeg"; + break; + case "png": + contentType = "image/png"; + break; + case "gif": + contentType = "image/gif"; + break; + case "svg": + contentType = "image/svg+xml"; + break; + case "webp": + contentType = "image/webp"; + break; + default: + contentType = "application/octet-stream"; + break; + } + + return Response.ok(imageFile, contentType) + .header("Cache-Control", "public, max-age=86400") + .header("Content-Disposition", "inline") + .header("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'") + .header("X-Content-Type-Options", "nosniff") + .build(); + } + + // ==================== Favorites Endpoints ==================== + + @GET + @Path("/favorites") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get favorites", description = "Returns the current user's favorited dashboard IDs") + public FavoritesListResponse getFavorites() { + logger.debug("Getting favorites for user: {}", getCurrentUser()); + + try { + PersistentStore store = getFavoritesStore(); + UserFavorites favorites = store.get(getCurrentUser()); + + if (favorites == null) { + return new FavoritesListResponse(new ArrayList<>()); + } + + return new FavoritesListResponse(favorites.getDashboardIds()); + } catch (Exception e) { + logger.error("Error getting favorites", e); + throw new DrillRuntimeException("Failed to get favorites: " + e.getMessage(), e); + } + } + + @POST + @Path("/{id}/favorite") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Toggle favorite", description = "Toggles a dashboard as favorite for the current user") + public Response toggleFavorite( + @Parameter(description = "Dashboard ID") @PathParam("id") String id) { + logger.debug("Toggling favorite for dashboard: {} by user: {}", id, getCurrentUser()); + + try { + PersistentStore store = getFavoritesStore(); + String user = getCurrentUser(); + UserFavorites favorites = store.get(user); + + if (favorites == null) { + favorites = new UserFavorites(); + } + + boolean nowFavorited; + if (favorites.getDashboardIds().contains(id)) { + favorites.getDashboardIds().remove(id); + nowFavorited = false; + } else { + favorites.getDashboardIds().add(id); + nowFavorited = true; + } + + store.put(user, favorites); + + String msg = nowFavorited ? "Dashboard added to favorites" : "Dashboard removed from favorites"; + return Response.ok(new FavoriteResponse(nowFavorited, msg)).build(); + } catch (Exception e) { + logger.error("Error toggling favorite", e); + throw new DrillRuntimeException("Failed to toggle favorite: " + e.getMessage(), e); + } + } + + // ==================== Helper Methods ==================== + + private PersistentStore getStore() { + if (cachedStore == null) { + synchronized (DashboardResources.class) { + if (cachedStore == null) { + try { + cachedStore = storeProvider.getOrCreateStore( + PersistentStoreConfig.newJacksonBuilder( + workManager.getContext().getLpPersistence().getMapper(), + Dashboard.class + ) + .name(STORE_NAME) + .build() + ); + } catch (StoreException e) { + throw new DrillRuntimeException("Failed to access dashboards store", e); + } + } + } + } + return cachedStore; + } + + private PersistentStore getFavoritesStore() { + if (cachedFavoritesStore == null) { + synchronized (DashboardResources.class) { + if (cachedFavoritesStore == null) { + try { + cachedFavoritesStore = storeProvider.getOrCreateStore( + PersistentStoreConfig.newJacksonBuilder( + workManager.getContext().getLpPersistence().getMapper(), + UserFavorites.class + ) + .name(FAVORITES_STORE_NAME) + .build() + ); + } catch (StoreException e) { + throw new DrillRuntimeException("Failed to access favorites store", e); + } + } + } + } + return cachedFavoritesStore; + } + + private File getUploadDir() { + if (cachedUploadDir == null) { + synchronized (DashboardResources.class) { + if (cachedUploadDir == null) { + // 1. Prefer DRILL_LOG_DIR (set by Drill startup scripts) + String basePath = System.getenv("DRILL_LOG_DIR"); + // 2. Fall back to Drill's persistent store path (same dir as system tables) + if (basePath == null) { + try { + basePath = workManager.getContext().getConfig() + .getString(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH); + } catch (Exception e) { + logger.debug("Could not read sys.store.provider.local.path", e); + } + } + // 3. Last resort: user home directory + if (basePath == null) { + basePath = System.getProperty("user.home"); + } + File dir = new File(basePath, UPLOAD_DIR_NAME); + if (!dir.exists() && !dir.mkdirs()) { + throw new DrillRuntimeException("Failed to create upload directory: " + dir); + } + logger.info("Dashboard image upload directory: {}", dir.getAbsolutePath()); + cachedUploadDir = dir; + } + } + } + return cachedUploadDir; + } + + private static String getFileExtension(String filename) { + int dotIndex = filename.lastIndexOf('.'); + if (dotIndex < 0 || dotIndex == filename.length() - 1) { + return ""; + } + return filename.substring(dotIndex + 1).toLowerCase(); + } + + private String getCurrentUser() { + return principal.getName(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java index 239936ea8eb..f76fb6b5fb3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java @@ -98,8 +98,18 @@ public DrillRestServer(final WorkManager workManager, final ServletContext servl register(MetricsResources.class); register(ThreadsResources.class); register(LogsResources.class); - - logger.info("Registered {} resource classes", 9); + register(MetadataResources.class); + register(SavedQueryResources.class); + register(VisualizationResources.class); + register(DashboardResources.class); + register(ProspectorResources.class); + register(AiConfigResources.class); + register(ProjectResources.class); + register(TestConnectionResources.class); + register(TranspileResources.class); + register(SharedQueryApiResources.class); + + logger.info("Registered {} resource classes", 19); property(FreemarkerMvcFeature.TEMPLATE_OBJECT_FACTORY, getFreemarkerConfiguration(servletContext)); register(FreemarkerMvcFeature.class); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/MetadataResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/MetadataResources.java new file mode 100644 index 00000000000..10dfdcb6e80 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/MetadataResources.java @@ -0,0 +1,914 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.drill.exec.expr.fn.registry.FunctionHolder; +import org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry; +import org.apache.drill.exec.server.rest.RestQueryRunner.QueryResult; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.work.WorkManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.annotation.security.RolesAllowed; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * REST API for browsing database metadata (schemas, tables, columns). + * Used by the SQL Lab frontend for schema exploration and autocomplete. + */ +@Path("/api/v1/metadata") +@Tag(name = "Metadata", description = "Database metadata exploration APIs") +@RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE) +public class MetadataResources { + private static final Logger logger = LoggerFactory.getLogger(MetadataResources.class); + + @Inject + WorkManager workManager; + + @Inject + WebUserConnection webUserConnection; + + @Inject + StoragePluginRegistry storageRegistry; + + // Plugins to completely exclude from the UI + private static final Set EXCLUDED_PLUGINS = new HashSet<>(Arrays.asList( + "cp", // classpath plugin - internal use only + "sys", // system tables - internal use only + "information_schema" // INFORMATION_SCHEMA - metadata only + )); + + // Plugins that appear but cannot enumerate their tables + private static final Set NON_BROWSABLE_PLUGINS = new HashSet<>(Arrays.asList( + "http" // http plugin - shows endpoints but can't enumerate table schema + )); + + // ==================== Response Models ==================== + + public static class PluginInfo { + @JsonProperty + public String name; + @JsonProperty + public String type; + @JsonProperty + public boolean enabled; + @JsonProperty + public boolean browsable; + + public PluginInfo() {} + + public PluginInfo(String name, String type, boolean enabled, boolean browsable) { + this.name = name; + this.type = type; + this.enabled = enabled; + this.browsable = browsable; + } + } + + public static class PluginsResponse { + @JsonProperty + public List plugins; + + public PluginsResponse(List plugins) { + this.plugins = plugins; + } + } + + public static class SchemaInfo { + @JsonProperty + public String name; + @JsonProperty + public String type = "schema"; + @JsonProperty + public String plugin; + @JsonProperty + public boolean browsable = true; + + public SchemaInfo() {} + + public SchemaInfo(String name) { + this.name = name; + } + + public SchemaInfo(String name, String plugin, boolean browsable) { + this.name = name; + this.plugin = plugin; + this.browsable = browsable; + } + } + + public static class SchemasResponse { + @JsonProperty + public List schemas; + + public SchemasResponse(List schemas) { + this.schemas = schemas; + } + } + + public static class TableInfo { + @JsonProperty + public String name; + @JsonProperty + public String schema; + @JsonProperty + public String type; + + public TableInfo() {} + + public TableInfo(String name, String schema, String type) { + this.name = name; + this.schema = schema; + this.type = type; + } + } + + public static class TablesResponse { + @JsonProperty + public List tables; + + public TablesResponse(List tables) { + this.tables = tables; + } + } + + public static class ColumnInfo { + @JsonProperty + public String name; + @JsonProperty + public String type; + @JsonProperty + public boolean nullable; + @JsonProperty + public String schema; + @JsonProperty + public String table; + + public ColumnInfo() {} + + public ColumnInfo(String name, String type, boolean nullable, String schema, String table) { + this.name = name; + this.type = type; + this.nullable = nullable; + this.schema = schema; + this.table = table; + } + } + + public static class ColumnsResponse { + @JsonProperty + public List columns; + + public ColumnsResponse(List columns) { + this.columns = columns; + } + } + + public static class TablePreviewResponse { + @JsonProperty + public List columns; + @JsonProperty + public List> rows; + + public TablePreviewResponse(List columns, List> rows) { + this.columns = columns; + this.rows = rows; + } + } + + public static class FunctionsResponse { + @JsonProperty + public List functions; + + public FunctionsResponse(List functions) { + this.functions = functions; + } + } + + public static class SchemaTreeRequest { + @JsonProperty + public List schemas; + + public SchemaTreeRequest() { + } + } + + public static class SchemaTreeTable { + @JsonProperty + public String name; + @JsonProperty + public List columns; + + public SchemaTreeTable(String name, List columns) { + this.name = name; + this.columns = columns; + } + } + + public static class SchemaTreeEntry { + @JsonProperty + public String name; + @JsonProperty + public List tables; + + public SchemaTreeEntry(String name, List tables) { + this.name = name; + this.tables = tables; + } + } + + public static class SchemaTreeResponse { + @JsonProperty + public List schemas; + + public SchemaTreeResponse(List schemas) { + this.schemas = schemas; + } + } + + /** + * File or folder information from SHOW FILES command. + */ + public static class FileInfo { + @JsonProperty + public String name; + @JsonProperty + public boolean isDirectory; + @JsonProperty + public boolean isFile; + @JsonProperty + public long length; + @JsonProperty + public String owner; + @JsonProperty + public String group; + @JsonProperty + public String permissions; + @JsonProperty + public String modificationTime; + + public FileInfo() {} + + public FileInfo(String name, boolean isDirectory, boolean isFile, long length, + String owner, String group, String permissions, String modificationTime) { + this.name = name; + this.isDirectory = isDirectory; + this.isFile = isFile; + this.length = length; + this.owner = owner; + this.group = group; + this.permissions = permissions; + this.modificationTime = modificationTime; + } + } + + public static class FilesResponse { + @JsonProperty + public List files; + @JsonProperty + public String path; + + public FilesResponse(List files, String path) { + this.files = files; + this.path = path; + } + } + + // ==================== API Endpoints ==================== + + @GET + @Path("/plugins") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List storage plugins", description = "Returns a list of enabled storage plugins") + public PluginsResponse getPlugins() { + logger.debug("Fetching storage plugins"); + + List plugins = new ArrayList<>(); + + try { + // Get all enabled storage plugins + Map enabledPlugins = storageRegistry.enabledConfigs(); + for (Map.Entry entry : enabledPlugins.entrySet()) { + String pluginName = entry.getKey(); + + // Skip excluded plugins (cp, sys, information_schema) + if (EXCLUDED_PLUGINS.contains(pluginName.toLowerCase())) { + continue; + } + + StoragePluginConfig pluginConfig = entry.getValue(); + String pluginType = pluginConfig != null ? pluginConfig.getClass().getSimpleName() : "unknown"; + // Remove "Config" suffix if present for cleaner display + if (pluginType.endsWith("Config")) { + pluginType = pluginType.substring(0, pluginType.length() - 6); + } + boolean isBrowsable = !NON_BROWSABLE_PLUGINS.contains(pluginName.toLowerCase()); + plugins.add(new PluginInfo(pluginName, pluginType.toLowerCase(), true, isBrowsable)); + } + } catch (Exception e) { + logger.error("Error fetching plugins", e); + throw new RuntimeException("Failed to fetch plugins: " + e.getMessage(), e); + } + + return new PluginsResponse(plugins); + } + + @GET + @Path("/plugins/{plugin}/schemas") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List schemas for a plugin", description = "Returns a list of schemas for the specified plugin") + public SchemasResponse getPluginSchemas( + @Parameter(description = "Plugin name") @PathParam("plugin") String plugin) { + logger.debug("Fetching schemas for plugin: {}", plugin); + + List schemas = new ArrayList<>(); + boolean isBrowsable = !NON_BROWSABLE_PLUGINS.contains(plugin.toLowerCase()); + + // For non-browsable plugins, just return the plugin name as the only schema + if (!isBrowsable) { + schemas.add(new SchemaInfo(plugin, plugin, false)); + return new SchemasResponse(schemas); + } + + // Try to get sub-schemas from INFORMATION_SCHEMA + String sql = String.format( + "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA " + + "WHERE SCHEMA_NAME = '%s' OR SCHEMA_NAME LIKE '%s.%%' " + + "ORDER BY SCHEMA_NAME", + escapeQuotes(plugin), escapeQuotes(plugin)); + + try { + QueryResult result = executeQuery(sql); + for (Map row : result.rows) { + String schemaName = row.get("SCHEMA_NAME"); + if (schemaName != null) { + schemas.add(new SchemaInfo(schemaName, plugin, isBrowsable)); + } + } + } catch (Exception e) { + logger.warn("Error fetching schemas for plugin: {}, returning plugin as sole schema", plugin, e); + // Return at least the plugin name as a schema + schemas.add(new SchemaInfo(plugin, plugin, isBrowsable)); + } + + // If no schemas found, return the plugin name itself + if (schemas.isEmpty()) { + schemas.add(new SchemaInfo(plugin, plugin, isBrowsable)); + } + + return new SchemasResponse(schemas); + } + + @GET + @Path("/schemas") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List all schemas", description = "Returns a list of all available schemas/databases") + public SchemasResponse getSchemas() { + logger.debug("Fetching all schemas"); + + List schemas = new ArrayList<>(); + + // First, add schemas from enabled storage plugins (fast, no connection needed) + try { + Map enabledPlugins = storageRegistry.enabledConfigs(); + for (String pluginName : enabledPlugins.keySet()) { + // Skip excluded plugins + if (EXCLUDED_PLUGINS.contains(pluginName.toLowerCase())) { + continue; + } + boolean isBrowsable = !NON_BROWSABLE_PLUGINS.contains(pluginName.toLowerCase()); + // Add the root plugin as a schema + schemas.add(new SchemaInfo(pluginName, pluginName, isBrowsable)); + } + } catch (Exception e) { + logger.warn("Error fetching plugins for schema list, falling back to INFORMATION_SCHEMA query", e); + } + + // If we got schemas from plugins, return them; otherwise fall back to SQL query + if (!schemas.isEmpty()) { + return new SchemasResponse(schemas); + } + + // Fallback: try INFORMATION_SCHEMA query (may fail if plugins have connection issues) + String sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA ORDER BY SCHEMA_NAME"; + try { + QueryResult result = executeQuery(sql); + for (Map row : result.rows) { + String schemaName = row.get("SCHEMA_NAME"); + if (schemaName != null) { + schemas.add(new SchemaInfo(schemaName)); + } + } + } catch (Exception e) { + logger.error("Error fetching schemas from INFORMATION_SCHEMA", e); + // Return empty list rather than failing completely + return new SchemasResponse(schemas); + } + + return new SchemasResponse(schemas); + } + + @GET + @Path("/schemas/{schema}/tables") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List tables in schema", description = "Returns a list of tables in the specified schema") + public TablesResponse getTables( + @Parameter(description = "Schema name") @PathParam("schema") String schema) { + logger.debug("Fetching tables for schema: {}", schema); + + String sql = String.format( + "SELECT TABLE_NAME, TABLE_TYPE FROM INFORMATION_SCHEMA.`TABLES` " + + "WHERE TABLE_SCHEMA = '%s' ORDER BY TABLE_NAME", + escapeQuotes(schema)); + + List tables = new ArrayList<>(); + + try { + QueryResult result = executeQuery(sql); + for (Map row : result.rows) { + String tableName = row.get("TABLE_NAME"); + String tableType = row.get("TABLE_TYPE"); + if (tableName != null) { + tables.add(new TableInfo(tableName, schema, tableType != null ? tableType : "TABLE")); + } + } + } catch (Exception e) { + logger.error("Error fetching tables for schema: {}", schema, e); + throw new RuntimeException("Failed to fetch tables: " + e.getMessage(), e); + } + + return new TablesResponse(tables); + } + + @GET + @Path("/schemas/{schema}/files") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List files in schema/workspace", + description = "Returns a list of files and folders in the specified schema (for file-based plugins like dfs)") + public FilesResponse getFiles( + @Parameter(description = "Schema name (e.g., dfs.tmp)") @PathParam("schema") String schema, + @Parameter(description = "Subdirectory path") @QueryParam("path") @DefaultValue("") String subPath) { + logger.debug("Fetching files for schema: {}, path: {}", schema, subPath); + + List files = new ArrayList<>(); + + // Build a properly-quoted compound schema path: + // dfs.tmp + myFolder → dfs.`tmp`.`myFolder` + String formattedPath; + if (subPath != null && !subPath.isEmpty()) { + formattedPath = formatSchemaPath(schema) + ".`" + escapeBackticks(subPath) + "`"; + } else { + formattedPath = formatSchemaPath(schema); + } + + // Use SHOW FILES command to list files + String sql = String.format("SHOW FILES IN %s", formattedPath); + + try { + QueryResult result = executeQuery(sql); + for (Map row : result.rows) { + String name = row.get("name"); + if (name == null) { + continue; + } + + boolean isDirectory = "true".equalsIgnoreCase(row.get("isDirectory")); + boolean isFile = "true".equalsIgnoreCase(row.get("isFile")); + long length = 0; + try { + String lenStr = row.get("length"); + if (lenStr != null) { + length = Long.parseLong(lenStr); + } + } catch (NumberFormatException e) { + // ignore + } + + files.add(new FileInfo( + name, + isDirectory, + isFile, + length, + row.get("owner"), + row.get("group"), + row.get("permissions"), + row.get("modificationTime") + )); + } + } catch (Exception e) { + logger.warn("Error fetching files for schema: {} - this may not be a file-based plugin", schema, e); + // Return empty list for non-file plugins rather than throwing + return new FilesResponse(files, subPath); + } + + return new FilesResponse(files, subPath); + } + + @GET + @Path("/schemas/{schema}/tables/{table}/columns") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List columns in table", description = "Returns a list of columns in the specified table") + public ColumnsResponse getColumns( + @Parameter(description = "Schema name") @PathParam("schema") String schema, + @Parameter(description = "Table name") @PathParam("table") String table) { + logger.debug("Fetching columns for table: {}.{}", schema, table); + + String sql = String.format( + "SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE " + + "FROM INFORMATION_SCHEMA.COLUMNS " + + "WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' " + + "ORDER BY ORDINAL_POSITION", + escapeQuotes(schema), escapeQuotes(table)); + + List columns = new ArrayList<>(); + + try { + QueryResult result = executeQuery(sql); + for (Map row : result.rows) { + String columnName = row.get("COLUMN_NAME"); + String dataType = row.get("DATA_TYPE"); + String isNullable = row.get("IS_NULLABLE"); + if (columnName != null) { + columns.add(new ColumnInfo( + columnName, + dataType != null ? dataType : "ANY", + "YES".equalsIgnoreCase(isNullable), + schema, + table + )); + } + } + } catch (Exception e) { + logger.error("Error fetching columns for table: {}.{}", schema, table, e); + throw new RuntimeException("Failed to fetch columns: " + e.getMessage(), e); + } + + return new ColumnsResponse(columns); + } + + @GET + @Path("/schemas/{schema}/tables/{table}/preview") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Preview table data", description = "Returns a preview of data from the specified table") + public TablePreviewResponse previewTable( + @Parameter(description = "Schema name") @PathParam("schema") String schema, + @Parameter(description = "Table name") @PathParam("table") String table, + @Parameter(description = "Maximum rows to return") @QueryParam("limit") @DefaultValue("100") int limit) { + logger.debug("Previewing table: {}.{} with limit {}", schema, table, limit); + + // Cap the limit to prevent excessive data retrieval + int safeLimit = Math.min(Math.max(1, limit), 1000); + + String sql = String.format( + "SELECT * FROM %s.`%s` LIMIT %d", + formatSchemaPath(schema), escapeBackticks(table), safeLimit); + + try { + QueryResult result = executeQuery(sql); + return new TablePreviewResponse(new ArrayList<>(result.columns), result.rows); + } catch (Exception e) { + logger.error("Error previewing table: {}.{}", schema, table, e); + throw new RuntimeException("Failed to preview table: " + e.getMessage(), e); + } + } + + @GET + @Path("/schemas/{schema}/files/columns") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get columns from a file", + description = "Returns column names and types by executing SELECT * LIMIT 1 on the file") + public ColumnsResponse getFileColumns( + @Parameter(description = "Schema name (e.g., dfs.tmp)") @PathParam("schema") String schema, + @Parameter(description = "File path within the schema") @QueryParam("path") String filePath) { + logger.debug("Fetching columns for file: {}/{}", schema, filePath); + + if (filePath == null || filePath.isEmpty()) { + throw new IllegalArgumentException("File path is required"); + } + + // Build the fully qualified path + // Plugin name stays unquoted, workspace parts are individually backtick-quoted, + // and the file path is backtick-quoted. e.g. dfs.`test`.`file.xml` + String fullPath = formatSchemaPath(schema) + ".`" + escapeBackticks(filePath) + "`"; + + String sql = String.format("SELECT * FROM %s LIMIT 1", fullPath); + + List columns = new ArrayList<>(); + + try { + QueryResult result = executeQuery(sql); + + // Use result.metadata for column types when available (preferred), + // fall back to value-based inference. + List columnNames = new ArrayList<>(result.columns); + for (int i = 0; i < columnNames.size(); i++) { + String columnName = columnNames.get(i); + String dataType = "ANY"; + if (result.metadata != null && i < result.metadata.size()) { + dataType = result.metadata.get(i); + } else if (!result.rows.isEmpty()) { + String value = result.rows.get(0).get(columnName); + dataType = inferDataType(value); + } + columns.add(new ColumnInfo(columnName, dataType, true, schema, filePath)); + } + } catch (Exception e) { + logger.error("Error fetching columns for file: {}/{}", schema, filePath, e); + throw new RuntimeException("Failed to get file columns: " + e.getMessage(), e); + } + + return new ColumnsResponse(columns); + } + + /** + * Infer the data type from a string value. + * This is a simple heuristic for display purposes. + */ + private String inferDataType(String value) { + if (value == null) { + return "ANY"; + } + + // Try integer + try { + Long.parseLong(value); + return "BIGINT"; + } catch (NumberFormatException e) { + // not an integer + } + + // Try floating point + try { + Double.parseDouble(value); + return "DOUBLE"; + } catch (NumberFormatException e) { + // not a number + } + + // Check for boolean + if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) { + return "BOOLEAN"; + } + + // Check for date/time patterns + if (value.matches("\\d{4}-\\d{2}-\\d{2}")) { + return "DATE"; + } + if (value.matches("\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}:\\d{2}.*")) { + return "TIMESTAMP"; + } + + // Default to VARCHAR + return "VARCHAR"; + } + + @GET + @Path("/functions") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List SQL functions", description = "Returns a list of available SQL functions for autocomplete") + public FunctionsResponse getFunctions() { + logger.debug("Fetching SQL functions"); + + TreeSet functionSet = new TreeSet<>(); + + try { + // Get built-in functions from the function registry + List builtInFunctions = workManager.getContext() + .getFunctionImplementationRegistry() + .getLocalFunctionRegistry() + .getAllJarsWithFunctionsHolders() + .get(LocalFunctionRegistry.BUILT_IN); + + if (builtInFunctions != null) { + for (FunctionHolder holder : builtInFunctions) { + String name = holder.getName(); + // Only include functions that start with a letter and don't contain spaces + if (name != null && !name.contains(" ") && name.matches("([a-z]|[A-Z])\\w+") + && !holder.getHolder().isInternal()) { + functionSet.add(name); + } + } + } + } catch (Exception e) { + logger.error("Error fetching functions", e); + // Return empty list on error rather than failing + } + + return new FunctionsResponse(new ArrayList<>(functionSet)); + } + + @POST + @Path("/schema-tree") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get schema tree", + description = "Returns schemas with their tables and columns in a single call") + public SchemaTreeResponse getSchemaTree(SchemaTreeRequest request) { + logger.debug("Fetching schema tree for {} schemas", + request.schemas != null ? request.schemas.size() : 0); + + List entries = new ArrayList<>(); + + if (request.schemas == null || request.schemas.isEmpty()) { + return new SchemaTreeResponse(entries); + } + + // Step 1: Resolve all sub-schemas in one query + StringBuilder whereClause = new StringBuilder(); + for (int i = 0; i < request.schemas.size(); i++) { + if (i > 0) { + whereClause.append(" OR "); + } + String name = escapeQuotes(request.schemas.get(i)); + whereClause.append(String.format( + "(SCHEMA_NAME = '%s' OR SCHEMA_NAME LIKE '%s.%%')", name, name)); + } + + List allSchemas = new ArrayList<>(); + try { + String schemaSql = "SELECT DISTINCT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE " + + whereClause + " ORDER BY SCHEMA_NAME"; + QueryResult result = executeQuery(schemaSql); + for (Map row : new ArrayList<>(result.rows)) { + String name = row.get("SCHEMA_NAME"); + if (name != null) { + allSchemas.add(name); + } + } + } catch (Exception e) { + logger.warn("Error resolving sub-schemas", e); + allSchemas.addAll(request.schemas); + } + + if (allSchemas.isEmpty()) { + return new SchemaTreeResponse(entries); + } + + // Step 2: Get all tables across all resolved schemas in one query + StringBuilder inClause = new StringBuilder(); + for (int i = 0; i < allSchemas.size(); i++) { + if (i > 0) { + inClause.append(", "); + } + inClause.append("'").append(escapeQuotes(allSchemas.get(i))).append("'"); + } + + // schema -> set of table names (preserving order, deduplicating) + Map> schemaToTables = new java.util.LinkedHashMap<>(); + try { + String tablesSql = "SELECT DISTINCT TABLE_SCHEMA, TABLE_NAME " + + "FROM INFORMATION_SCHEMA.`TABLES` " + + "WHERE TABLE_SCHEMA IN (" + inClause + ") " + + "ORDER BY TABLE_SCHEMA, TABLE_NAME"; + QueryResult result = executeQuery(tablesSql); + for (Map row : new ArrayList<>(result.rows)) { + String schema = row.get("TABLE_SCHEMA"); + String table = row.get("TABLE_NAME"); + if (schema != null && table != null) { + schemaToTables.computeIfAbsent(schema, k -> new LinkedHashSet<>()).add(table); + } + } + } catch (Exception e) { + logger.warn("Error fetching tables for schema tree", e); + } + + // Step 3: Get all columns across all resolved schemas in one query + // Key: "schema|table" -> ordered set of column names + Map> tableToColumns = new java.util.LinkedHashMap<>(); + try { + String colsSql = "SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME " + + "FROM INFORMATION_SCHEMA.COLUMNS " + + "WHERE TABLE_SCHEMA IN (" + inClause + ") " + + "ORDER BY TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION"; + QueryResult result = executeQuery(colsSql); + for (Map row : new ArrayList<>(result.rows)) { + String schema = row.get("TABLE_SCHEMA"); + String table = row.get("TABLE_NAME"); + String col = row.get("COLUMN_NAME"); + if (schema != null && table != null && col != null) { + String key = schema + "|" + table; + tableToColumns.computeIfAbsent(key, k -> new LinkedHashSet<>()).add(col); + } + } + } catch (Exception e) { + logger.warn("Error fetching columns for schema tree", e); + } + + // Step 4: Assemble the tree + for (String schema : allSchemas) { + Set tableNames = schemaToTables.get(schema); + if (tableNames == null || tableNames.isEmpty()) { + continue; + } + + List tables = new ArrayList<>(); + for (String tableName : tableNames) { + String key = schema + "|" + tableName; + Set cols = tableToColumns.get(key); + List columns = cols != null ? new ArrayList<>(cols) : new ArrayList<>(); + tables.add(new SchemaTreeTable(tableName, columns)); + } + entries.add(new SchemaTreeEntry(schema, tables)); + } + + return new SchemaTreeResponse(entries); + } + + // ==================== Helper Methods ==================== + + /** + * Execute a SQL query and return the results. + * Clears previous results from the shared WebUserConnection + * to prevent accumulation across multiple queries in one request. + */ + private QueryResult executeQuery(String sql) throws Exception { + webUserConnection.results.clear(); + webUserConnection.columns.clear(); + webUserConnection.metadata.clear(); + + QueryWrapper wrapper = new QueryWrapper.RestQueryBuilder() + .query(sql) + .queryType("SQL") + .rowLimit("10000") // Reasonable limit for metadata queries + .build(); + + return new RestQueryRunner(wrapper, workManager, webUserConnection).run(); + } + + /** + * Escape single quotes in SQL strings + */ + private String escapeQuotes(String value) { + if (value == null) { + return ""; + } + return value.replace("'", "''"); + } + + /** + * Escape backticks in SQL identifiers + */ + private String escapeBackticks(String value) { + if (value == null) { + return ""; + } + return value.replace("`", "``"); + } + + /** + * Format a compound schema name for SQL queries. + * Plugin name stays unquoted; workspace parts are individually backtick-quoted. + * e.g. "dfs.test" → "dfs.`test`", "dfs" → "dfs" + */ + private String formatSchemaPath(String schema) { + if (schema == null || !schema.contains(".")) { + return schema; + } + String[] parts = schema.split("\\.", 2); + String[] workspaceParts = parts[1].split("\\."); + StringBuilder sb = new StringBuilder(parts[0]); + for (String wp : workspaceParts) { + sb.append(".`").append(escapeBackticks(wp)).append("`"); + } + return sb.toString(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProjectResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProjectResources.java new file mode 100644 index 00000000000..f92e6645e11 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProjectResources.java @@ -0,0 +1,1269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.exec.exception.StoreException; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; +import org.apache.drill.exec.store.sys.PersistentStore; +import org.apache.drill.exec.store.sys.PersistentStoreConfig; +import org.apache.drill.exec.store.sys.PersistentStoreProvider; +import org.apache.drill.exec.work.WorkManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.annotation.security.RolesAllowed; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * REST API for managing projects. + * Projects are the top-level organizational unit containing datasets, + * saved queries, visualizations, dashboards, and wiki documentation. + */ +@Path("/api/v1/projects") +@Tag(name = "Projects", description = "APIs for managing data analytics projects") +@RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE) +public class ProjectResources { + private static final Logger logger = LoggerFactory.getLogger(ProjectResources.class); + private static final String STORE_NAME = "drill.sqllab.projects"; + private static final String FAVORITES_STORE_NAME = "drill.sqllab.project_favorites"; + + @Inject + WorkManager workManager; + + @Inject + DrillUserPrincipal principal; + + @Inject + PersistentStoreProvider storeProvider; + + private static volatile PersistentStore cachedStore; + private static volatile PersistentStore cachedFavoritesStore; + + // ==================== Model Classes ==================== + + /** + * A reference to a dataset (table or saved query) within a project. + */ + public static class DatasetRef { + @JsonProperty + private String id; + @JsonProperty + private String type; + @JsonProperty + private String schema; + @JsonProperty + private String table; + @JsonProperty + private String savedQueryId; + @JsonProperty + private String label; + + public DatasetRef() { + } + + @JsonCreator + public DatasetRef( + @JsonProperty("id") String id, + @JsonProperty("type") String type, + @JsonProperty("schema") String schema, + @JsonProperty("table") String table, + @JsonProperty("savedQueryId") String savedQueryId, + @JsonProperty("label") String label) { + this.id = id; + this.type = type != null ? type : "table"; + this.schema = schema; + this.table = table; + this.savedQueryId = savedQueryId; + this.label = label; + } + + public String getId() { return id; } + public String getType() { return type; } + public String getSchema() { return schema; } + public String getTable() { return table; } + public String getSavedQueryId() { return savedQueryId; } + public String getLabel() { return label; } + + public void setId(String id) { this.id = id; } + public void setType(String type) { this.type = type; } + public void setSchema(String schema) { this.schema = schema; } + public void setTable(String table) { this.table = table; } + public void setSavedQueryId(String savedQueryId) { this.savedQueryId = savedQueryId; } + public void setLabel(String label) { this.label = label; } + } + + /** + * A wiki page within a project for documentation. + */ + public static class WikiPage { + @JsonProperty + private String id; + @JsonProperty + private String title; + @JsonProperty + private String content; + @JsonProperty + private int order; + @JsonProperty + private long createdAt; + @JsonProperty + private long updatedAt; + + public WikiPage() { + } + + @JsonCreator + public WikiPage( + @JsonProperty("id") String id, + @JsonProperty("title") String title, + @JsonProperty("content") String content, + @JsonProperty("order") int order, + @JsonProperty("createdAt") long createdAt, + @JsonProperty("updatedAt") long updatedAt) { + this.id = id; + this.title = title; + this.content = content; + this.order = order; + this.createdAt = createdAt; + this.updatedAt = updatedAt; + } + + public String getId() { return id; } + public String getTitle() { return title; } + public String getContent() { return content; } + public int getOrder() { return order; } + public long getCreatedAt() { return createdAt; } + public long getUpdatedAt() { return updatedAt; } + + public void setId(String id) { this.id = id; } + public void setTitle(String title) { this.title = title; } + public void setContent(String content) { this.content = content; } + public void setOrder(int order) { this.order = order; } + public void setCreatedAt(long createdAt) { this.createdAt = createdAt; } + public void setUpdatedAt(long updatedAt) { this.updatedAt = updatedAt; } + } + + /** + * Project model for persistence. + */ + public static class Project { + @JsonProperty + private String id; + @JsonProperty + private String name; + @JsonProperty + private String description; + @JsonProperty + private List tags; + @JsonProperty + private String owner; + @JsonProperty + private boolean isPublic; + @JsonProperty + private List sharedWith; + @JsonProperty + private List datasets; + @JsonProperty + private List savedQueryIds; + @JsonProperty + private List visualizationIds; + @JsonProperty + private List dashboardIds; + @JsonProperty + private List wikiPages; + @JsonProperty + private long createdAt; + @JsonProperty + private long updatedAt; + + public Project() { + } + + @JsonCreator + public Project( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("description") String description, + @JsonProperty("tags") List tags, + @JsonProperty("owner") String owner, + @JsonProperty("isPublic") boolean isPublic, + @JsonProperty("sharedWith") List sharedWith, + @JsonProperty("datasets") List datasets, + @JsonProperty("savedQueryIds") List savedQueryIds, + @JsonProperty("visualizationIds") List visualizationIds, + @JsonProperty("dashboardIds") List dashboardIds, + @JsonProperty("wikiPages") List wikiPages, + @JsonProperty("createdAt") long createdAt, + @JsonProperty("updatedAt") long updatedAt) { + this.id = id; + this.name = name; + this.description = description; + this.tags = tags != null ? tags : new ArrayList<>(); + this.owner = owner; + this.isPublic = isPublic; + this.sharedWith = sharedWith != null ? sharedWith : new ArrayList<>(); + this.datasets = datasets != null ? datasets : new ArrayList<>(); + this.savedQueryIds = savedQueryIds != null ? savedQueryIds : new ArrayList<>(); + this.visualizationIds = visualizationIds != null ? visualizationIds : new ArrayList<>(); + this.dashboardIds = dashboardIds != null ? dashboardIds : new ArrayList<>(); + this.wikiPages = wikiPages != null ? wikiPages : new ArrayList<>(); + this.createdAt = createdAt; + this.updatedAt = updatedAt; + } + + public String getId() { return id; } + public String getName() { return name; } + public String getDescription() { return description; } + public List getTags() { return tags; } + public String getOwner() { return owner; } + public boolean isPublic() { return isPublic; } + public List getSharedWith() { return sharedWith; } + public List getDatasets() { return datasets; } + public List getSavedQueryIds() { return savedQueryIds; } + public List getVisualizationIds() { return visualizationIds; } + public List getDashboardIds() { return dashboardIds; } + public List getWikiPages() { return wikiPages; } + public long getCreatedAt() { return createdAt; } + public long getUpdatedAt() { return updatedAt; } + + public void setName(String name) { this.name = name; } + public void setDescription(String description) { this.description = description; } + public void setTags(List tags) { this.tags = tags; } + public void setPublic(boolean isPublic) { this.isPublic = isPublic; } + public void setSharedWith(List sharedWith) { this.sharedWith = sharedWith; } + public void setDatasets(List datasets) { this.datasets = datasets; } + public void setSavedQueryIds(List savedQueryIds) { this.savedQueryIds = savedQueryIds; } + public void setVisualizationIds(List visualizationIds) { this.visualizationIds = visualizationIds; } + public void setDashboardIds(List dashboardIds) { this.dashboardIds = dashboardIds; } + public void setWikiPages(List wikiPages) { this.wikiPages = wikiPages; } + public void setUpdatedAt(long updatedAt) { this.updatedAt = updatedAt; } + } + + /** + * Stores a user's list of favorited project IDs. + */ + public static class UserFavorites { + @JsonProperty + private List projectIds; + + public UserFavorites() { + this.projectIds = new ArrayList<>(); + } + + @JsonCreator + public UserFavorites( + @JsonProperty("projectIds") List projectIds) { + this.projectIds = projectIds != null ? projectIds : new ArrayList<>(); + } + + public List getProjectIds() { return projectIds; } + public void setProjectIds(List projectIds) { this.projectIds = projectIds; } + } + + // ==================== Request/Response Classes ==================== + + /** + * Request body for creating a new project. + */ + public static class CreateProjectRequest { + @JsonProperty + public String name; + @JsonProperty + public String description; + @JsonProperty + public List tags; + @JsonProperty + public boolean isPublic; + } + + /** + * Request body for updating a project. + */ + public static class UpdateProjectRequest { + @JsonProperty + public String name; + @JsonProperty + public String description; + @JsonProperty + public List tags; + @JsonProperty + public Boolean isPublic; + @JsonProperty + public List sharedWith; + } + + /** + * Request body for creating/updating a wiki page. + */ + public static class WikiPageRequest { + @JsonProperty + public String title; + @JsonProperty + public String content; + @JsonProperty + public Integer order; + } + + /** + * Response containing a list of projects. + */ + public static class ProjectsResponse { + @JsonProperty + public List projects; + + public ProjectsResponse(List projects) { + this.projects = projects; + } + } + + /** + * Simple message response. + */ + public static class MessageResponse { + @JsonProperty + public String message; + + public MessageResponse(String message) { + this.message = message; + } + } + + /** + * Response for favorite toggle operations. + */ + public static class FavoriteResponse { + @JsonProperty + public boolean favorited; + @JsonProperty + public String message; + + public FavoriteResponse(boolean favorited, String message) { + this.favorited = favorited; + this.message = message; + } + } + + /** + * Response containing a list of favorited project IDs. + */ + public static class FavoritesListResponse { + @JsonProperty + public List projectIds; + + public FavoritesListResponse(List projectIds) { + this.projectIds = projectIds; + } + } + + // ==================== CRUD Endpoints ==================== + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List projects", + description = "Returns all projects accessible by the current user") + public ProjectsResponse listProjects() { + logger.debug("Listing projects for user: {}", getCurrentUser()); + + List projects = new ArrayList<>(); + String currentUser = getCurrentUser(); + + try { + PersistentStore store = getStore(); + Iterator> iterator = store.getAll(); + + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + Project project = entry.getValue(); + + if (canRead(project, currentUser)) { + projects.add(project); + } + } + } catch (Exception e) { + logger.error("Error listing projects", e); + throw new DrillRuntimeException("Failed to list projects: " + e.getMessage(), e); + } + + return new ProjectsResponse(projects); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Create project", description = "Creates a new project") + public Response createProject(CreateProjectRequest request) { + logger.debug("Creating project: {}", request.name); + + if (request.name == null || request.name.trim().isEmpty()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("Project name is required")) + .build(); + } + + String id = UUID.randomUUID().toString(); + long now = Instant.now().toEpochMilli(); + + Project project = new Project( + id, + request.name.trim(), + request.description, + request.tags, + getCurrentUser(), + request.isPublic, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + now, + now + ); + + try { + PersistentStore store = getStore(); + store.put(id, project); + } catch (Exception e) { + logger.error("Error creating project", e); + throw new DrillRuntimeException("Failed to create project: " + e.getMessage(), e); + } + + return Response.status(Response.Status.CREATED).entity(project).build(); + } + + @GET + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get project", description = "Returns a project by ID") + public Response getProject( + @Parameter(description = "Project ID") @PathParam("id") String id) { + logger.debug("Getting project: {}", id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!canRead(project, getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Access denied")) + .build(); + } + + return Response.ok(project).build(); + } catch (Exception e) { + logger.error("Error getting project", e); + throw new DrillRuntimeException("Failed to get project: " + e.getMessage(), e); + } + } + + @PUT + @Path("/{id}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Update project", description = "Updates an existing project") + public Response updateProject( + @Parameter(description = "Project ID") @PathParam("id") String id, + UpdateProjectRequest request) { + logger.debug("Updating project: {}", id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can update this project")) + .build(); + } + + if (request.name != null) { + project.setName(request.name.trim()); + } + if (request.description != null) { + project.setDescription(request.description); + } + if (request.tags != null) { + project.setTags(request.tags); + } + if (request.isPublic != null) { + project.setPublic(request.isPublic); + } + if (request.sharedWith != null) { + project.setSharedWith(request.sharedWith); + } + + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(project).build(); + } catch (Exception e) { + logger.error("Error updating project", e); + throw new DrillRuntimeException("Failed to update project: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Delete project", description = "Deletes a project") + public Response deleteProject( + @Parameter(description = "Project ID") @PathParam("id") String id) { + logger.debug("Deleting project: {}", id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can delete this project")) + .build(); + } + + store.delete(id); + + return Response.ok(new MessageResponse("Project deleted successfully")).build(); + } catch (Exception e) { + logger.error("Error deleting project", e); + throw new DrillRuntimeException("Failed to delete project: " + e.getMessage(), e); + } + } + + // ==================== Dataset Endpoints ==================== + + @POST + @Path("/{id}/datasets") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Add dataset", description = "Adds a dataset reference to a project") + public Response addDataset( + @Parameter(description = "Project ID") @PathParam("id") String id, + DatasetRef datasetRef) { + logger.debug("Adding dataset to project: {}", id); + + try { + PersistentStore store = getStore(); + + // Synchronize on the interned project ID to prevent concurrent + // read-modify-write races on the same project. + synchronized (id.intern()) { + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + if (datasetRef.getId() == null || datasetRef.getId().isEmpty()) { + datasetRef.setId(UUID.randomUUID().toString()); + } + + project.getDatasets().add(datasetRef); + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(project).build(); + } + } catch (Exception e) { + logger.error("Error adding dataset to project", e); + throw new DrillRuntimeException("Failed to add dataset: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}/datasets/{datasetId}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Remove dataset", description = "Removes a dataset reference from a project") + public Response removeDataset( + @Parameter(description = "Project ID") @PathParam("id") String id, + @Parameter(description = "Dataset ID") @PathParam("datasetId") String datasetId) { + logger.debug("Removing dataset {} from project: {}", datasetId, id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + project.getDatasets().removeIf(d -> d.getId().equals(datasetId)); + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(project).build(); + } catch (Exception e) { + logger.error("Error removing dataset from project", e); + throw new DrillRuntimeException("Failed to remove dataset: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/cleanup/plugin/{pluginName}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Clean up plugin datasets", + description = "Removes all dataset references for a deleted plugin from all projects") + public Response cleanupPluginDatasets( + @Parameter(description = "Plugin name") @PathParam("pluginName") String pluginName) { + logger.debug("Cleaning up datasets for deleted plugin: {}", pluginName); + + try { + PersistentStore store = getStore(); + Iterator> iterator = store.getAll(); + int removedCount = 0; + + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + Project project = entry.getValue(); + List datasets = project.getDatasets(); + + int before = datasets.size(); + datasets.removeIf(d -> { + if (d.getSchema() == null) { + return false; + } + String dsPlugin = d.getSchema().split("\\.")[0]; + return dsPlugin.equals(pluginName); + }); + + if (datasets.size() < before) { + removedCount += before - datasets.size(); + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(entry.getKey(), project); + } + } + + return Response.ok(new MessageResponse( + "Removed " + removedCount + " dataset reference(s) for plugin " + pluginName + )).build(); + } catch (Exception e) { + logger.error("Error cleaning up plugin datasets", e); + throw new DrillRuntimeException("Failed to clean up plugin datasets: " + e.getMessage(), e); + } + } + + // ==================== Saved Query Endpoints ==================== + + @POST + @Path("/{id}/saved-queries/{queryId}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Add saved query", + description = "Adds a saved query to a project, removing it from any other project") + public Response addSavedQuery( + @Parameter(description = "Project ID") @PathParam("id") String id, + @Parameter(description = "Saved Query ID") @PathParam("queryId") String queryId) { + logger.debug("Adding saved query {} to project: {}", queryId, id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + // Remove from all other projects first + removeItemFromAllProjects(store, "savedQuery", queryId, id); + + if (!project.getSavedQueryIds().contains(queryId)) { + project.getSavedQueryIds().add(queryId); + } + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(project).build(); + } catch (Exception e) { + logger.error("Error adding saved query to project", e); + throw new DrillRuntimeException("Failed to add saved query: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}/saved-queries/{queryId}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Remove saved query", + description = "Removes a saved query from a project") + public Response removeSavedQuery( + @Parameter(description = "Project ID") @PathParam("id") String id, + @Parameter(description = "Saved Query ID") @PathParam("queryId") String queryId) { + logger.debug("Removing saved query {} from project: {}", queryId, id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + project.getSavedQueryIds().remove(queryId); + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(project).build(); + } catch (Exception e) { + logger.error("Error removing saved query from project", e); + throw new DrillRuntimeException("Failed to remove saved query: " + e.getMessage(), e); + } + } + + // ==================== Visualization Endpoints ==================== + + @POST + @Path("/{id}/visualizations/{vizId}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Add visualization", + description = "Adds a visualization to a project, removing it from any other project") + public Response addVisualization( + @Parameter(description = "Project ID") @PathParam("id") String id, + @Parameter(description = "Visualization ID") @PathParam("vizId") String vizId) { + logger.debug("Adding visualization {} to project: {}", vizId, id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + removeItemFromAllProjects(store, "visualization", vizId, id); + + if (!project.getVisualizationIds().contains(vizId)) { + project.getVisualizationIds().add(vizId); + } + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(project).build(); + } catch (Exception e) { + logger.error("Error adding visualization to project", e); + throw new DrillRuntimeException("Failed to add visualization: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}/visualizations/{vizId}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Remove visualization", + description = "Removes a visualization from a project") + public Response removeVisualization( + @Parameter(description = "Project ID") @PathParam("id") String id, + @Parameter(description = "Visualization ID") @PathParam("vizId") String vizId) { + logger.debug("Removing visualization {} from project: {}", vizId, id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + project.getVisualizationIds().remove(vizId); + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(project).build(); + } catch (Exception e) { + logger.error("Error removing visualization from project", e); + throw new DrillRuntimeException("Failed to remove visualization: " + e.getMessage(), e); + } + } + + // ==================== Dashboard Endpoints ==================== + + @POST + @Path("/{id}/dashboards/{dashId}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Add dashboard", + description = "Adds a dashboard to a project, removing it from any other project") + public Response addDashboard( + @Parameter(description = "Project ID") @PathParam("id") String id, + @Parameter(description = "Dashboard ID") @PathParam("dashId") String dashId) { + logger.debug("Adding dashboard {} to project: {}", dashId, id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + removeItemFromAllProjects(store, "dashboard", dashId, id); + + if (!project.getDashboardIds().contains(dashId)) { + project.getDashboardIds().add(dashId); + } + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(project).build(); + } catch (Exception e) { + logger.error("Error adding dashboard to project", e); + throw new DrillRuntimeException("Failed to add dashboard: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}/dashboards/{dashId}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Remove dashboard", + description = "Removes a dashboard from a project") + public Response removeDashboard( + @Parameter(description = "Project ID") @PathParam("id") String id, + @Parameter(description = "Dashboard ID") @PathParam("dashId") String dashId) { + logger.debug("Removing dashboard {} from project: {}", dashId, id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + project.getDashboardIds().remove(dashId); + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(project).build(); + } catch (Exception e) { + logger.error("Error removing dashboard from project", e); + throw new DrillRuntimeException("Failed to remove dashboard: " + e.getMessage(), e); + } + } + + // ==================== Wiki Endpoints ==================== + + @POST + @Path("/{id}/wiki") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Create wiki page", + description = "Creates a new wiki page in a project") + public Response createWikiPage( + @Parameter(description = "Project ID") @PathParam("id") String id, + WikiPageRequest request) { + logger.debug("Creating wiki page in project: {}", id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + if (request.title == null || request.title.trim().isEmpty()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("Wiki page title is required")) + .build(); + } + + long now = Instant.now().toEpochMilli(); + WikiPage page = new WikiPage( + UUID.randomUUID().toString(), + request.title.trim(), + request.content != null ? request.content : "", + request.order != null ? request.order : project.getWikiPages().size(), + now, + now + ); + + project.getWikiPages().add(page); + project.setUpdatedAt(now); + store.put(id, project); + + return Response.status(Response.Status.CREATED).entity(page).build(); + } catch (Exception e) { + logger.error("Error creating wiki page", e); + throw new DrillRuntimeException("Failed to create wiki page: " + e.getMessage(), e); + } + } + + @PUT + @Path("/{id}/wiki/{pageId}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Update wiki page", + description = "Updates a wiki page in a project") + public Response updateWikiPage( + @Parameter(description = "Project ID") @PathParam("id") String id, + @Parameter(description = "Wiki Page ID") @PathParam("pageId") String pageId, + WikiPageRequest request) { + logger.debug("Updating wiki page {} in project: {}", pageId, id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + WikiPage target = null; + for (WikiPage page : project.getWikiPages()) { + if (page.getId().equals(pageId)) { + target = page; + break; + } + } + + if (target == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Wiki page not found")) + .build(); + } + + if (request.title != null) { + target.setTitle(request.title.trim()); + } + if (request.content != null) { + target.setContent(request.content); + } + if (request.order != null) { + target.setOrder(request.order); + } + + long now = Instant.now().toEpochMilli(); + target.setUpdatedAt(now); + project.setUpdatedAt(now); + store.put(id, project); + + return Response.ok(target).build(); + } catch (Exception e) { + logger.error("Error updating wiki page", e); + throw new DrillRuntimeException("Failed to update wiki page: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}/wiki/{pageId}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Delete wiki page", + description = "Deletes a wiki page from a project") + public Response deleteWikiPage( + @Parameter(description = "Project ID") @PathParam("id") String id, + @Parameter(description = "Wiki Page ID") @PathParam("pageId") String pageId) { + logger.debug("Deleting wiki page {} from project: {}", pageId, id); + + try { + PersistentStore store = getStore(); + Project project = store.get(id); + + if (project == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Project not found")) + .build(); + } + + if (!project.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can modify this project")) + .build(); + } + + project.getWikiPages().removeIf(p -> p.getId().equals(pageId)); + project.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(id, project); + + return Response.ok(new MessageResponse("Wiki page deleted successfully")).build(); + } catch (Exception e) { + logger.error("Error deleting wiki page", e); + throw new DrillRuntimeException("Failed to delete wiki page: " + e.getMessage(), e); + } + } + + // ==================== Favorites Endpoints ==================== + + @GET + @Path("/favorites") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get favorites", + description = "Returns the current user's favorited project IDs") + public FavoritesListResponse getFavorites() { + logger.debug("Getting favorites for user: {}", getCurrentUser()); + + try { + PersistentStore store = getFavoritesStore(); + UserFavorites favorites = store.get(getCurrentUser()); + + if (favorites == null) { + return new FavoritesListResponse(new ArrayList<>()); + } + + return new FavoritesListResponse(favorites.getProjectIds()); + } catch (Exception e) { + logger.error("Error getting favorites", e); + throw new DrillRuntimeException("Failed to get favorites: " + e.getMessage(), e); + } + } + + @POST + @Path("/{id}/favorite") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Toggle favorite", + description = "Toggles a project as favorite for the current user") + public Response toggleFavorite( + @Parameter(description = "Project ID") @PathParam("id") String id) { + logger.debug("Toggling favorite for project: {} by user: {}", id, getCurrentUser()); + + try { + PersistentStore store = getFavoritesStore(); + String user = getCurrentUser(); + UserFavorites favorites = store.get(user); + + if (favorites == null) { + favorites = new UserFavorites(); + } + + boolean nowFavorited; + if (favorites.getProjectIds().contains(id)) { + favorites.getProjectIds().remove(id); + nowFavorited = false; + } else { + favorites.getProjectIds().add(id); + nowFavorited = true; + } + + store.put(user, favorites); + + String msg = nowFavorited + ? "Project added to favorites" + : "Project removed from favorites"; + return Response.ok(new FavoriteResponse(nowFavorited, msg)).build(); + } catch (Exception e) { + logger.error("Error toggling favorite", e); + throw new DrillRuntimeException("Failed to toggle favorite: " + e.getMessage(), e); + } + } + + // ==================== Helper Methods ==================== + + private boolean canRead(Project project, String user) { + return project.getOwner().equals(user) + || project.isPublic() + || project.getSharedWith().contains(user); + } + + /** + * Removes an item ID from all projects except the target project. + * This ensures single-project ownership for saved queries, visualizations, and dashboards. + */ + private void removeItemFromAllProjects( + PersistentStore store, String itemType, String itemId, String excludeProjectId) { + Iterator> iterator = store.getAll(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + Project p = entry.getValue(); + + if (p.getId().equals(excludeProjectId)) { + continue; + } + + boolean modified = false; + switch (itemType) { + case "savedQuery": + modified = p.getSavedQueryIds().remove(itemId); + break; + case "visualization": + modified = p.getVisualizationIds().remove(itemId); + break; + case "dashboard": + modified = p.getDashboardIds().remove(itemId); + break; + default: + break; + } + + if (modified) { + p.setUpdatedAt(Instant.now().toEpochMilli()); + store.put(entry.getKey(), p); + } + } + } + + private PersistentStore getStore() { + if (cachedStore == null) { + synchronized (ProjectResources.class) { + if (cachedStore == null) { + try { + cachedStore = storeProvider.getOrCreateStore( + PersistentStoreConfig.newJacksonBuilder( + workManager.getContext().getLpPersistence().getMapper(), + Project.class + ) + .name(STORE_NAME) + .build() + ); + } catch (StoreException e) { + throw new DrillRuntimeException("Failed to access projects store", e); + } + } + } + } + return cachedStore; + } + + private PersistentStore getFavoritesStore() { + if (cachedFavoritesStore == null) { + synchronized (ProjectResources.class) { + if (cachedFavoritesStore == null) { + try { + cachedFavoritesStore = storeProvider.getOrCreateStore( + PersistentStoreConfig.newJacksonBuilder( + workManager.getContext().getLpPersistence().getMapper(), + UserFavorites.class + ) + .name(FAVORITES_STORE_NAME) + .build() + ); + } catch (StoreException e) { + throw new DrillRuntimeException("Failed to access favorites store", e); + } + } + } + } + return cachedFavoritesStore; + } + + private String getCurrentUser() { + return principal.getName(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProspectorResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProspectorResources.java new file mode 100644 index 00000000000..de5b5058072 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProspectorResources.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.drill.exec.exception.StoreException; +import org.apache.drill.exec.server.rest.ai.ChatMessage; +import org.apache.drill.exec.server.rest.ai.LlmConfig; +import org.apache.drill.exec.server.rest.ai.LlmProvider; +import org.apache.drill.exec.server.rest.ai.LlmProviderRegistry; +import org.apache.drill.exec.server.rest.ai.ToolDefinition; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; +import org.apache.drill.exec.store.sys.PersistentStore; +import org.apache.drill.exec.store.sys.PersistentStoreConfig; +import org.apache.drill.exec.store.sys.PersistentStoreProvider; +import org.apache.drill.exec.work.WorkManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.annotation.security.RolesAllowed; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.StreamingOutput; +import java.util.ArrayList; +import java.util.List; + +/** + * REST resource for the Prospector chat endpoint. + * Provides streaming SSE responses by proxying to the configured LLM provider. + */ +@Path("/api/v1/ai") +@Tag(name = "Prospector", description = "AI assistant for Apache Drill SQL Lab") +@RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE) +public class ProspectorResources { + + private static final Logger logger = LoggerFactory.getLogger(ProspectorResources.class); + private static final String CONFIG_STORE_NAME = "drill.sqllab.ai_config"; + private static final String CONFIG_KEY = "default"; + + @Inject + WorkManager workManager; + + @Inject + PersistentStoreProvider storeProvider; + + private static volatile PersistentStore cachedStore; + + // ==================== Request/Response Models ==================== + + public static class AiStatusResponse { + @JsonProperty + public boolean enabled; + + @JsonProperty + public boolean configured; + + public AiStatusResponse(boolean enabled, boolean configured) { + this.enabled = enabled; + this.configured = configured; + } + } + + public static class ChatRequest { + @JsonProperty + public List messages; + + @JsonProperty + public List tools; + + @JsonProperty + public ChatContext context; + + public ChatRequest() { + } + + @JsonCreator + public ChatRequest( + @JsonProperty("messages") List messages, + @JsonProperty("tools") List tools, + @JsonProperty("context") ChatContext context) { + this.messages = messages; + this.tools = tools; + this.context = context; + } + } + + public static class ChatContext { + @JsonProperty + public String currentSql; + + @JsonProperty + public String currentSchema; + + @JsonProperty + public List availableSchemas; + + @JsonProperty + public String error; + + @JsonProperty + public ResultSummary resultSummary; + + public ChatContext() { + } + + @JsonCreator + public ChatContext( + @JsonProperty("currentSql") String currentSql, + @JsonProperty("currentSchema") String currentSchema, + @JsonProperty("availableSchemas") List availableSchemas, + @JsonProperty("error") String error, + @JsonProperty("resultSummary") ResultSummary resultSummary) { + this.currentSql = currentSql; + this.currentSchema = currentSchema; + this.availableSchemas = availableSchemas; + this.error = error; + this.resultSummary = resultSummary; + } + } + + public static class ResultSummary { + @JsonProperty + public int rowCount; + + @JsonProperty + public List columns; + + @JsonProperty + public List columnTypes; + + public ResultSummary() { + } + + @JsonCreator + public ResultSummary( + @JsonProperty("rowCount") int rowCount, + @JsonProperty("columns") List columns, + @JsonProperty("columnTypes") List columnTypes) { + this.rowCount = rowCount; + this.columns = columns; + this.columnTypes = columnTypes; + } + } + + public static class ErrorResponse { + @JsonProperty + public String message; + + public ErrorResponse(String message) { + this.message = message; + } + } + + // ==================== Endpoints ==================== + + @GET + @Path("/status") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get Prospector status", + description = "Returns whether Prospector is enabled and configured") + public AiStatusResponse getStatus() { + try { + LlmConfig config = getConfig(); + if (config == null) { + return new AiStatusResponse(false, false); + } + boolean configured = config.getApiKey() != null && !config.getApiKey().isEmpty() + && config.getModel() != null && !config.getModel().isEmpty(); + return new AiStatusResponse(config.isEnabled() && configured, configured); + } catch (Exception e) { + logger.error("Error checking AI status", e); + return new AiStatusResponse(false, false); + } + } + + @POST + @Path("/chat") + @Consumes(MediaType.APPLICATION_JSON) + @Produces("text/event-stream") + @Operation(summary = "Stream AI chat completion", + description = "Sends messages to the LLM and streams back SSE events") + public Response chat(ChatRequest request) { + try { + LlmConfig config = getConfig(); + if (config == null || !config.isEnabled()) { + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity(new ErrorResponse("Prospector is not enabled")) + .type(MediaType.APPLICATION_JSON) + .build(); + } + + LlmProvider provider = LlmProviderRegistry.get(config.getProvider()); + if (provider == null) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new ErrorResponse("Unknown LLM provider: " + config.getProvider())) + .type(MediaType.APPLICATION_JSON) + .build(); + } + + // Build the full message list with system prompt + List fullMessages = buildMessages(config, request); + List tools = request.tools != null ? request.tools : new ArrayList<>(); + + StreamingOutput stream = out -> { + try { + provider.streamChatCompletion(config, fullMessages, tools, out); + } catch (Exception e) { + logger.error("Error during AI chat streaming", e); + try { + ObjectMapper mapper = new ObjectMapper(); + String errorData = "{\"message\":" + mapper.writeValueAsString(e.getMessage()) + "}"; + String sse = "event: error\ndata: " + errorData + "\n\n"; + out.write(sse.getBytes("UTF-8")); + out.flush(); + } catch (Exception writeErr) { + logger.error("Error writing error event", writeErr); + } + } + }; + + return Response.ok(stream) + .header("Content-Type", "text/event-stream") + .header("Cache-Control", "no-cache") + .header("Connection", "keep-alive") + .header("X-Accel-Buffering", "no") + .build(); + } catch (Exception e) { + logger.error("Error initiating AI chat", e); + return Response.serverError() + .entity(new ErrorResponse("Failed to initiate chat: " + e.getMessage())) + .type(MediaType.APPLICATION_JSON) + .build(); + } + } + + // ==================== Helper Methods ==================== + + private List buildMessages(LlmConfig config, ChatRequest request) { + List messages = new ArrayList<>(); + + // Build system prompt + StringBuilder systemPrompt = new StringBuilder(); + systemPrompt.append("You are an AI assistant for Apache Drill SQL Lab. "); + systemPrompt.append("Apache Drill is a schema-free SQL query engine that supports "); + systemPrompt.append("querying various data sources (JSON, CSV, Parquet, databases, APIs) "); + systemPrompt.append("using ANSI SQL.\n\n"); + + // Add context + if (request.context != null) { + ChatContext ctx = request.context; + + if (ctx.currentSchema != null && !ctx.currentSchema.isEmpty()) { + systemPrompt.append("Current schema: ").append(ctx.currentSchema).append("\n"); + } + + if (ctx.availableSchemas != null && !ctx.availableSchemas.isEmpty()) { + systemPrompt.append("Available schemas: ") + .append(String.join(", ", ctx.availableSchemas)).append("\n"); + } + + if (ctx.currentSql != null && !ctx.currentSql.isEmpty()) { + systemPrompt.append("\nCurrent SQL in editor:\n```sql\n") + .append(ctx.currentSql).append("\n```\n"); + } + + if (ctx.error != null && !ctx.error.isEmpty()) { + systemPrompt.append("\nCurrent error: ").append(ctx.error).append("\n"); + } + + if (ctx.resultSummary != null) { + ResultSummary rs = ctx.resultSummary; + systemPrompt.append("\nQuery results: ").append(rs.rowCount).append(" rows"); + if (rs.columns != null && !rs.columns.isEmpty()) { + systemPrompt.append(", columns: "); + for (int i = 0; i < rs.columns.size(); i++) { + if (i > 0) { + systemPrompt.append(", "); + } + systemPrompt.append(rs.columns.get(i)); + if (rs.columnTypes != null && i < rs.columnTypes.size()) { + systemPrompt.append(" (").append(rs.columnTypes.get(i)).append(")"); + } + } + } + systemPrompt.append("\n"); + } + } + + systemPrompt.append("\nWhen generating SQL, use Apache Drill SQL syntax. "); + systemPrompt.append("Use backtick quoting for identifiers with special characters. "); + systemPrompt.append("Use `LIMIT` for row limiting.\n\n"); + systemPrompt.append("You have tools available to execute SQL, explore schemas, "); + systemPrompt.append("create visualizations and dashboards. Use them proactively to help the user."); + + // Append custom system prompt if configured + if (config.getSystemPrompt() != null && !config.getSystemPrompt().isEmpty()) { + systemPrompt.append("\n\n").append(config.getSystemPrompt()); + } + + messages.add(ChatMessage.system(systemPrompt.toString())); + + // Add user messages + if (request.messages != null) { + messages.addAll(request.messages); + } + + return messages; + } + + private LlmConfig getConfig() { + try { + PersistentStore store = getStore(); + return store.get(CONFIG_KEY); + } catch (Exception e) { + logger.error("Error reading AI config", e); + return null; + } + } + + private PersistentStore getStore() { + if (cachedStore == null) { + synchronized (ProspectorResources.class) { + if (cachedStore == null) { + try { + cachedStore = storeProvider.getOrCreateStore( + PersistentStoreConfig.newJacksonBuilder( + workManager.getContext().getLpPersistence().getMapper(), + LlmConfig.class + ) + .name(CONFIG_STORE_NAME) + .build() + ); + } catch (StoreException e) { + throw new RuntimeException("Failed to access AI config store", e); + } + } + } + } + return cachedStore; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SavedQueryResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SavedQueryResources.java new file mode 100644 index 00000000000..c0e145a7de7 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SavedQueryResources.java @@ -0,0 +1,492 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.exec.exception.StoreException; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; +import org.apache.drill.exec.store.sys.PersistentStore; +import org.apache.drill.exec.store.sys.PersistentStoreConfig; +import org.apache.drill.exec.store.sys.PersistentStoreProvider; +import org.apache.drill.exec.work.WorkManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.annotation.security.RolesAllowed; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * REST API for managing saved SQL queries. + * Queries are persisted using Drill's PersistentStore mechanism. + */ +@Path("/api/v1/saved-queries") +@Tag(name = "Saved Queries", description = "APIs for managing saved SQL queries") +@RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE) +public class SavedQueryResources { + private static final Logger logger = LoggerFactory.getLogger(SavedQueryResources.class); + private static final String STORE_NAME = "drill.sqllab.saved_queries"; + + @Inject + WorkManager workManager; + + @Inject + DrillUserPrincipal principal; + + @Inject + PersistentStoreProvider storeProvider; + + private static volatile PersistentStore cachedStore; + + // ==================== Model Classes ==================== + + /** + * Saved query model for persistence. + */ + public static class SavedQuery { + @JsonProperty + private String id; + @JsonProperty + private String name; + @JsonProperty + private String description; + @JsonProperty + private String sql; + @JsonProperty + private String defaultSchema; + @JsonProperty + private String owner; + @JsonProperty + private long createdAt; + @JsonProperty + private long updatedAt; + @JsonProperty + private Map tags; + @JsonProperty + private boolean isPublic; + + // Default constructor for Jackson + public SavedQuery() { + } + + @JsonCreator + public SavedQuery( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("description") String description, + @JsonProperty("sql") String sql, + @JsonProperty("defaultSchema") String defaultSchema, + @JsonProperty("owner") String owner, + @JsonProperty("createdAt") long createdAt, + @JsonProperty("updatedAt") long updatedAt, + @JsonProperty("tags") Map tags, + @JsonProperty("isPublic") boolean isPublic) { + this.id = id; + this.name = name; + this.description = description; + this.sql = sql; + this.defaultSchema = defaultSchema; + this.owner = owner; + this.createdAt = createdAt; + this.updatedAt = updatedAt; + this.tags = tags; + this.isPublic = isPublic; + } + + // Getters + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getSql() { + return sql; + } + + public String getDefaultSchema() { + return defaultSchema; + } + + public String getOwner() { + return owner; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getUpdatedAt() { + return updatedAt; + } + + public Map getTags() { + return tags; + } + + public boolean isPublic() { + return isPublic; + } + + // Setters for updates + public void setName(String name) { + this.name = name; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public void setDefaultSchema(String defaultSchema) { + this.defaultSchema = defaultSchema; + } + + public void setUpdatedAt(long updatedAt) { + this.updatedAt = updatedAt; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public void setPublic(boolean isPublic) { + this.isPublic = isPublic; + } + } + + /** + * Request body for creating a new saved query. + */ + public static class CreateSavedQueryRequest { + @JsonProperty + public String name; + @JsonProperty + public String description; + @JsonProperty + public String sql; + @JsonProperty + public String defaultSchema; + @JsonProperty + public Map tags; + @JsonProperty + public boolean isPublic; + } + + /** + * Request body for updating a saved query. + */ + public static class UpdateSavedQueryRequest { + @JsonProperty + public String name; + @JsonProperty + public String description; + @JsonProperty + public String sql; + @JsonProperty + public String defaultSchema; + @JsonProperty + public Map tags; + @JsonProperty + public Boolean isPublic; + } + + /** + * Response containing a list of saved queries. + */ + public static class SavedQueriesResponse { + @JsonProperty + public List queries; + + public SavedQueriesResponse(List queries) { + this.queries = queries; + } + } + + /** + * Simple message response. + */ + public static class MessageResponse { + @JsonProperty + public String message; + + public MessageResponse(String message) { + this.message = message; + } + } + + // ==================== API Endpoints ==================== + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List saved queries", description = "Returns all saved queries accessible by the current user") + public SavedQueriesResponse listSavedQueries() { + logger.debug("Listing saved queries for user: {}", getCurrentUser()); + + List queries = new ArrayList<>(); + String currentUser = getCurrentUser(); + + try { + PersistentStore store = getStore(); + Iterator> iterator = store.getAll(); + + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + SavedQuery query = entry.getValue(); + + // Return queries owned by user or public queries + if (query.getOwner().equals(currentUser) || query.isPublic()) { + queries.add(query); + } + } + } catch (Exception e) { + logger.error("Error listing saved queries", e); + throw new DrillRuntimeException("Failed to list saved queries: " + e.getMessage(), e); + } + + return new SavedQueriesResponse(queries); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Create saved query", description = "Creates a new saved query") + public Response createSavedQuery(CreateSavedQueryRequest request) { + logger.debug("Creating saved query: {}", request.name); + + if (request.name == null || request.name.trim().isEmpty()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("Query name is required")) + .build(); + } + + if (request.sql == null || request.sql.trim().isEmpty()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("SQL is required")) + .build(); + } + + String id = UUID.randomUUID().toString(); + long now = Instant.now().toEpochMilli(); + + SavedQuery query = new SavedQuery( + id, + request.name.trim(), + request.description, + request.sql, + request.defaultSchema, + getCurrentUser(), + now, + now, + request.tags != null ? request.tags : new HashMap<>(), + request.isPublic + ); + + try { + PersistentStore store = getStore(); + store.put(id, query); + } catch (Exception e) { + logger.error("Error creating saved query", e); + throw new DrillRuntimeException("Failed to create saved query: " + e.getMessage(), e); + } + + return Response.status(Response.Status.CREATED).entity(query).build(); + } + + @GET + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get saved query", description = "Returns a saved query by ID") + public Response getSavedQuery( + @Parameter(description = "Query ID") @PathParam("id") String id) { + logger.debug("Getting saved query: {}", id); + + try { + PersistentStore store = getStore(); + SavedQuery query = store.get(id); + + if (query == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Query not found")) + .build(); + } + + // Check access permissions + if (!query.getOwner().equals(getCurrentUser()) && !query.isPublic()) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Access denied")) + .build(); + } + + return Response.ok(query).build(); + } catch (Exception e) { + logger.error("Error getting saved query", e); + throw new DrillRuntimeException("Failed to get saved query: " + e.getMessage(), e); + } + } + + @PUT + @Path("/{id}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Update saved query", description = "Updates an existing saved query") + public Response updateSavedQuery( + @Parameter(description = "Query ID") @PathParam("id") String id, + UpdateSavedQueryRequest request) { + logger.debug("Updating saved query: {}", id); + + try { + PersistentStore store = getStore(); + SavedQuery query = store.get(id); + + if (query == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Query not found")) + .build(); + } + + // Only owner can update + if (!query.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can update this query")) + .build(); + } + + // Update fields if provided + if (request.name != null) { + query.setName(request.name.trim()); + } + if (request.description != null) { + query.setDescription(request.description); + } + if (request.sql != null) { + query.setSql(request.sql); + } + if (request.defaultSchema != null) { + query.setDefaultSchema(request.defaultSchema); + } + if (request.tags != null) { + query.setTags(request.tags); + } + if (request.isPublic != null) { + query.setPublic(request.isPublic); + } + + query.setUpdatedAt(Instant.now().toEpochMilli()); + + store.put(id, query); + + return Response.ok(query).build(); + } catch (Exception e) { + logger.error("Error updating saved query", e); + throw new DrillRuntimeException("Failed to update saved query: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Delete saved query", description = "Deletes a saved query") + public Response deleteSavedQuery( + @Parameter(description = "Query ID") @PathParam("id") String id) { + logger.debug("Deleting saved query: {}", id); + + try { + PersistentStore store = getStore(); + SavedQuery query = store.get(id); + + if (query == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Query not found")) + .build(); + } + + // Only owner can delete + if (!query.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can delete this query")) + .build(); + } + + store.delete(id); + + return Response.ok(new MessageResponse("Query deleted successfully")).build(); + } catch (Exception e) { + logger.error("Error deleting saved query", e); + throw new DrillRuntimeException("Failed to delete saved query: " + e.getMessage(), e); + } + } + + // ==================== Helper Methods ==================== + + private PersistentStore getStore() { + if (cachedStore == null) { + synchronized (SavedQueryResources.class) { + if (cachedStore == null) { + try { + cachedStore = storeProvider.getOrCreateStore( + PersistentStoreConfig.newJacksonBuilder( + workManager.getContext().getLpPersistence().getMapper(), + SavedQuery.class + ) + .name(STORE_NAME) + .build() + ); + } catch (StoreException e) { + throw new DrillRuntimeException("Failed to access saved queries store", e); + } + } + } + } + return cachedStore; + } + + private String getCurrentUser() { + return principal.getName(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SharedQueryApiResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SharedQueryApiResources.java new file mode 100644 index 00000000000..89e6277493e --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SharedQueryApiResources.java @@ -0,0 +1,507 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.exec.exception.StoreException; +import org.apache.drill.exec.server.rest.QueryWrapper.RestQueryBuilder; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; +import org.apache.drill.exec.server.rest.stream.QueryRunner; +import org.apache.drill.exec.store.sys.PersistentStore; +import org.apache.drill.exec.store.sys.PersistentStoreConfig; +import org.apache.drill.exec.store.sys.PersistentStoreProvider; +import org.apache.drill.exec.work.WorkManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.annotation.security.PermitAll; +import jakarta.annotation.security.RolesAllowed; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.StreamingOutput; +import java.io.IOException; +import java.io.OutputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * REST API for managing shared query APIs. + * Allows users to expose query results as public REST endpoints. + */ +@Path("/api/v1/shared-queries") +@Tag(name = "Shared Query APIs", description = "APIs for sharing query results as REST endpoints") +@RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE) +public class SharedQueryApiResources { + private static final Logger logger = LoggerFactory.getLogger(SharedQueryApiResources.class); + private static final String STORE_NAME = "drill.sqllab.shared_query_apis"; + + @Inject + WorkManager workManager; + + @Inject + DrillUserPrincipal principal; + + @Inject + PersistentStoreProvider storeProvider; + + @Inject + WebUserConnection webUserConnection; + + private static volatile PersistentStore cachedStore; + + // ==================== Model Classes ==================== + + /** + * Shared query API model for persistence. + */ + public static class SharedQueryApi { + @JsonProperty + private String id; + @JsonProperty + private String name; + @JsonProperty + private String sql; + @JsonProperty + private String defaultSchema; + @JsonProperty + private String owner; + @JsonProperty + private long createdAt; + @JsonProperty + private long updatedAt; + @JsonProperty + private boolean apiEnabled; + + // Default constructor for Jackson + public SharedQueryApi() { + } + + @JsonCreator + public SharedQueryApi( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("sql") String sql, + @JsonProperty("defaultSchema") String defaultSchema, + @JsonProperty("owner") String owner, + @JsonProperty("createdAt") long createdAt, + @JsonProperty("updatedAt") long updatedAt, + @JsonProperty("apiEnabled") boolean apiEnabled) { + this.id = id; + this.name = name; + this.sql = sql; + this.defaultSchema = defaultSchema; + this.owner = owner; + this.createdAt = createdAt; + this.updatedAt = updatedAt; + this.apiEnabled = apiEnabled; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getSql() { + return sql; + } + + public String getDefaultSchema() { + return defaultSchema; + } + + public String getOwner() { + return owner; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getUpdatedAt() { + return updatedAt; + } + + public boolean isApiEnabled() { + return apiEnabled; + } + + public void setName(String name) { + this.name = name; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public void setDefaultSchema(String defaultSchema) { + this.defaultSchema = defaultSchema; + } + + public void setUpdatedAt(long updatedAt) { + this.updatedAt = updatedAt; + } + + public void setApiEnabled(boolean apiEnabled) { + this.apiEnabled = apiEnabled; + } + } + + /** + * Request body for creating a shared query API. + */ + public static class CreateRequest { + @JsonProperty + public String name; + @JsonProperty + public String sql; + @JsonProperty + public String defaultSchema; + @JsonProperty + public boolean apiEnabled; + } + + /** + * Request body for updating a shared query API. + */ + public static class UpdateRequest { + @JsonProperty + public String name; + @JsonProperty + public String sql; + @JsonProperty + public String defaultSchema; + @JsonProperty + public Boolean apiEnabled; + } + + /** + * Response containing a list of shared query APIs. + */ + public static class SharedQueryApisResponse { + @JsonProperty + public List queries; + + public SharedQueryApisResponse(List queries) { + this.queries = queries; + } + } + + /** + * Simple message response. + */ + public static class MessageResponse { + @JsonProperty + public String message; + + public MessageResponse(String message) { + this.message = message; + } + } + + // ==================== CRUD Endpoints ==================== + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List shared query APIs", description = "Returns all shared query APIs owned by the current user") + public SharedQueryApisResponse listSharedQueryApis() { + logger.debug("Listing shared query APIs for user: {}", getCurrentUser()); + + List queries = new ArrayList<>(); + String currentUser = getCurrentUser(); + + try { + PersistentStore store = getStore(); + Iterator> iterator = store.getAll(); + + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + SharedQueryApi query = entry.getValue(); + + if (query.getOwner().equals(currentUser)) { + queries.add(query); + } + } + } catch (Exception e) { + logger.error("Error listing shared query APIs", e); + throw new DrillRuntimeException("Failed to list shared query APIs: " + e.getMessage(), e); + } + + return new SharedQueryApisResponse(queries); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Create shared query API", description = "Creates a new shared query API endpoint") + public Response createSharedQueryApi(CreateRequest request) { + logger.debug("Creating shared query API: {}", request.name); + + if (request.sql == null || request.sql.trim().isEmpty()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("SQL is required")) + .build(); + } + + String id = UUID.randomUUID().toString(); + long now = Instant.now().toEpochMilli(); + + SharedQueryApi query = new SharedQueryApi( + id, + request.name != null ? request.name.trim() : "Shared Query API", + request.sql, + request.defaultSchema, + getCurrentUser(), + now, + now, + request.apiEnabled + ); + + try { + PersistentStore store = getStore(); + store.put(id, query); + } catch (Exception e) { + logger.error("Error creating shared query API", e); + throw new DrillRuntimeException("Failed to create shared query API: " + e.getMessage(), e); + } + + return Response.status(Response.Status.CREATED).entity(query).build(); + } + + @GET + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get shared query API", description = "Returns a shared query API by ID") + public Response getSharedQueryApi( + @Parameter(description = "Shared query API ID") @PathParam("id") String id) { + logger.debug("Getting shared query API: {}", id); + + try { + PersistentStore store = getStore(); + SharedQueryApi query = store.get(id); + + if (query == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Shared query API not found")) + .build(); + } + + if (!query.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Access denied")) + .build(); + } + + return Response.ok(query).build(); + } catch (Exception e) { + logger.error("Error getting shared query API", e); + throw new DrillRuntimeException("Failed to get shared query API: " + e.getMessage(), e); + } + } + + @PUT + @Path("/{id}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Update shared query API", description = "Updates an existing shared query API") + public Response updateSharedQueryApi( + @Parameter(description = "Shared query API ID") @PathParam("id") String id, + UpdateRequest request) { + logger.debug("Updating shared query API: {}", id); + + try { + PersistentStore store = getStore(); + SharedQueryApi query = store.get(id); + + if (query == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Shared query API not found")) + .build(); + } + + if (!query.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can update this shared query API")) + .build(); + } + + if (request.name != null) { + query.setName(request.name.trim()); + } + if (request.sql != null) { + query.setSql(request.sql); + } + if (request.defaultSchema != null) { + query.setDefaultSchema(request.defaultSchema); + } + if (request.apiEnabled != null) { + query.setApiEnabled(request.apiEnabled); + } + + query.setUpdatedAt(Instant.now().toEpochMilli()); + + store.put(id, query); + + return Response.ok(query).build(); + } catch (Exception e) { + logger.error("Error updating shared query API", e); + throw new DrillRuntimeException("Failed to update shared query API: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Delete shared query API", description = "Deletes a shared query API") + public Response deleteSharedQueryApi( + @Parameter(description = "Shared query API ID") @PathParam("id") String id) { + logger.debug("Deleting shared query API: {}", id); + + try { + PersistentStore store = getStore(); + SharedQueryApi query = store.get(id); + + if (query == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Shared query API not found")) + .build(); + } + + if (!query.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can delete this shared query API")) + .build(); + } + + store.delete(id); + + return Response.ok(new MessageResponse("Shared query API deleted successfully")).build(); + } catch (Exception e) { + logger.error("Error deleting shared query API", e); + throw new DrillRuntimeException("Failed to delete shared query API: " + e.getMessage(), e); + } + } + + // ==================== Public Data Endpoint ==================== + + @GET + @Path("/{id}/data") + @Produces(MediaType.APPLICATION_JSON) + @PermitAll + @Operation(summary = "Get shared query data", description = "Executes the shared query and returns results as JSON") + public Response getSharedQueryData( + @Parameter(description = "Shared query API ID") @PathParam("id") String id) { + logger.debug("Fetching data for shared query API: {}", id); + + SharedQueryApi query; + try { + PersistentStore store = getStore(); + query = store.get(id); + } catch (Exception e) { + logger.error("Error accessing shared query API store", e); + throw new DrillRuntimeException("Failed to access shared query API: " + e.getMessage(), e); + } + + if (query == null || !query.isApiEnabled()) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Shared query API not found")) + .build(); + } + + QueryWrapper queryWrapper = new RestQueryBuilder() + .query(query.getSql()) + .queryType("SQL") + .rowLimit(10000) + .defaultSchema(query.getDefaultSchema()) + .build(); + + QueryRunner runner = new QueryRunner(workManager, webUserConnection); + try { + runner.start(queryWrapper); + } catch (Exception e) { + throw new WebApplicationException("Query execution failed", e); + } + + StreamingOutput streamingOutput = new StreamingOutput() { + @Override + public void write(OutputStream output) + throws IOException, WebApplicationException { + try { + runner.sendResults(output); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new WebApplicationException("Query execution failed", e); + } + } + }; + + return Response.ok(streamingOutput) + .header("Access-Control-Allow-Origin", "*") + .build(); + } + + // ==================== Helper Methods ==================== + + private PersistentStore getStore() { + if (cachedStore == null) { + synchronized (SharedQueryApiResources.class) { + if (cachedStore == null) { + try { + cachedStore = storeProvider.getOrCreateStore( + PersistentStoreConfig.newJacksonBuilder( + workManager.getContext().getLpPersistence().getMapper(), + SharedQueryApi.class + ) + .name(STORE_NAME) + .build() + ); + } catch (StoreException e) { + throw new DrillRuntimeException("Failed to access shared query APIs store", e); + } + } + } + } + return cachedStore; + } + + private String getCurrentUser() { + return principal.getName(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SqlLabSpaServlet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SqlLabSpaServlet.java new file mode 100644 index 00000000000..168dd8487c7 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SqlLabSpaServlet.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Servlet for serving the SQL Lab React SPA. + * Handles client-side routing by serving index.html for all non-static routes. + */ +public class SqlLabSpaServlet extends HttpServlet { + private static final Logger logger = LoggerFactory.getLogger(SqlLabSpaServlet.class); + private static final String INDEX_HTML = "index.html"; + private static final Path WEBAPP_BASE = Paths.get("webapp"); + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String pathInfo = request.getPathInfo(); + if (pathInfo == null || pathInfo.equals("/")) { + pathInfo = "/" + INDEX_HTML; + } + + // Check if this is a static asset request (has file extension) + boolean isStaticAsset = pathInfo.lastIndexOf('.') > pathInfo.lastIndexOf('/'); + + // Try to serve from built assets first + String subPath = isStaticAsset ? "dist" + pathInfo : "dist/" + INDEX_HTML; + URL resource = resolveResource(subPath); + + // Fallback to development mode (source files) + if (resource == null && isStaticAsset) { + resource = resolveResource(pathInfo.substring(1)); + } + + // For SPA routing, always serve index.html for non-static requests + if (resource == null && !isStaticAsset) { + resource = getClass().getResource("/webapp/dist/" + INDEX_HTML); + + // Fallback to source index.html + if (resource == null) { + resource = getClass().getResource("/webapp/" + INDEX_HTML); + } + } + + if (resource == null) { + logger.debug("Resource not found: {}", pathInfo); + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + + // Set content type based on file extension. + // For SPA fallback routes (non-static assets), we're serving index.html + // so use ".html" for content type detection instead of the original path. + String contentTypePath = isStaticAsset ? pathInfo : INDEX_HTML; + String contentType = getContentType(contentTypePath); + if (contentType != null) { + response.setContentType(contentType); + } + + // Serve the resource + try (InputStream in = resource.openStream(); + OutputStream out = response.getOutputStream()) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } + + /** + * Resolve a classpath resource within the webapp directory, using + * java.nio.file.Path normalization to prevent path traversal. + * + * @param subPath the relative path within webapp (e.g. "dist/assets/index.js") + * @return the resource URL, or null if invalid or not found + */ + private URL resolveResource(String subPath) { + try { + Path resolved = WEBAPP_BASE.resolve(subPath).normalize(); + if (!resolved.startsWith(WEBAPP_BASE)) { + logger.debug("Blocked path traversal attempt: {}", subPath); + return null; + } + String resourcePath = "/" + resolved.toString().replace('\\', '/'); + return getClass().getResource(resourcePath); + } catch (InvalidPathException e) { + return null; + } + } + + private String getContentType(String path) { + if (path.endsWith(".html")) { + return "text/html; charset=UTF-8"; + } else if (path.endsWith(".js")) { + return "application/javascript; charset=UTF-8"; + } else if (path.endsWith(".css")) { + return "text/css; charset=UTF-8"; + } else if (path.endsWith(".json")) { + return "application/json; charset=UTF-8"; + } else if (path.endsWith(".png")) { + return "image/png"; + } else if (path.endsWith(".jpg") || path.endsWith(".jpeg")) { + return "image/jpeg"; + } else if (path.endsWith(".svg")) { + return "image/svg+xml"; + } else if (path.endsWith(".ico")) { + return "image/x-icon"; + } else if (path.endsWith(".woff")) { + return "font/woff"; + } else if (path.endsWith(".woff2")) { + return "font/woff2"; + } else if (path.endsWith(".ttf")) { + return "font/ttf"; + } else if (path.endsWith(".eot")) { + return "application/vnd.ms-fontobject"; + } + return null; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SqlTranspiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SqlTranspiler.java new file mode 100644 index 00000000000..f1660602fb2 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/SqlTranspiler.java @@ -0,0 +1,596 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.gtkcyber.sqlglot.SqlGlot; +import com.gtkcyber.sqlglot.dialect.Dialect; +import com.gtkcyber.sqlglot.expressions.Expression; +import com.gtkcyber.sqlglot.expressions.Nodes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Singleton service that provides SQL dialect transpilation using Java sqlglot. + * Thread-safe: each call creates independent AST objects with no shared mutable state. + * + *

Provides SQL transpilation, formatting, data type conversion, and time grain + * manipulation using the pure Java sqlglot library.

+ */ +public final class SqlTranspiler { + + private static final Logger logger = LoggerFactory.getLogger(SqlTranspiler.class); + private static final SqlTranspiler INSTANCE = new SqlTranspiler(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final Dialect DRILL = Dialect.of("DRILL"); + + /** + * Mapping of characters that are illegal in SQL identifiers to token placeholders. + */ + private static final Map CHAR_MAPPING; + + static { + Map map = new HashMap<>(); + map.put(" ", "__SPACE__"); + map.put("-", "__DASH__"); + map.put(",", "__COMMA__"); + map.put(":", "__COLON__"); + map.put(";", "__SEMICOLON__"); + map.put("(", "__RPAREN__"); + map.put(")", "__LPAREN__"); + map.put("1", "__ONE__"); + map.put("2", "__TWO__"); + map.put("3", "__THREE__"); + map.put("4", "__FOUR__"); + map.put("5", "__FIVE__"); + map.put("6", "__SIX__"); + map.put("7", "__SEVEN__"); + map.put("8", "__EIGHT__"); + map.put("9", "__NINE__"); + map.put("0", "__ZERO__"); + CHAR_MAPPING = Collections.unmodifiableMap(map); + } + + /** + * Known file extensions in Drill (used for table name resolution). + */ + private static final Set EXTENSIONS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + "3g2", "3gp", "accdb", "mdb", "mdx", "ai", "arw", + "avi", "avro", "bmp", "cr2", "crw", "csv", "csvh", + "dng", "eps", "epsf", "epsi", "gif", "h5", "httpd", + "ico", "jpe", "jpeg", "jpg", "json", "log", "ltsv", + "m4a", "m4b", "m4p", "m4r", "m4v", "mov", "mp4", + "nef", "orf", "parquet", "pcap", "pcapng", "pcx", + "pdf", "png", "psd", "raf", "rw2", "rwl", "sav", + "sas7bdat", "seq", "shp", "srw", "syslog", "tbl", + "tif", "tiff", "tsv", "wav", "wave", "webp", "x3f", + "xlsx", "xml", "zip" + ))); + + /** + * Returns the singleton instance. + */ + public static SqlTranspiler getInstance() { + return INSTANCE; + } + + private SqlTranspiler() { + logger.info("SqlTranspiler initialized with Java sqlglot"); + } + + /** + * Transpile SQL from one dialect to another (without schema metadata). + */ + public String transpile(String sql, String sourceDialect, String targetDialect) { + return transpile(sql, sourceDialect, targetDialect, "[]"); + } + + /** + * Transpile SQL from one dialect to another. + * + * @param sql the SQL string to transpile + * @param sourceDialect the source dialect (e.g. "mysql") + * @param targetDialect the target dialect (e.g. "drill") + * @param schemasJson JSON string of schema metadata for table name resolution + * @return the transpiled SQL, or the original SQL if transpilation fails + */ + public String transpile(String sql, String sourceDialect, + String targetDialect, String schemasJson) { + if (sql == null || sql.isEmpty()) { + return sql; + } + try { + String src = sourceDialect != null ? sourceDialect.toUpperCase() : "ANSI"; + String tgt = targetDialect != null ? targetDialect.toUpperCase() : "DRILL"; + String result = SqlGlot.transpile(sql, src, tgt); + + if (schemasJson != null && !schemasJson.equals("[]") && !schemasJson.isEmpty()) { + result = ensureFullTableNames(result, schemasJson); + } + + return result; + } catch (Exception e) { + logger.debug("SQL transpilation failed, returning original SQL: {}", e.getMessage()); + return sql; + } + } + + /** + * Convert a column's data type in a SQL query using sqlglot AST manipulation. + * + * @param sql the SQL query + * @param columnName the column to convert + * @param dataType the target SQL data type (e.g. "INTEGER", "VARCHAR") + * @param columnsJson optional JSON object mapping column names to types (for star queries) + * @return the transformed SQL, or null if conversion fails + */ + public String convertDataType(String sql, String columnName, + String dataType, String columnsJson) { + try { + Optional parsed = DRILL.parseOne(sql); + if (!parsed.isPresent()) { + return null; + } + Expression query = parsed.get(); + + // Handle star queries by expanding to explicit columns + if (isStarQuery(query)) { + if (columnsJson == null || columnsJson.isEmpty()) { + return sql; + } + Map columnMap = MAPPER.readValue(columnsJson, + new TypeReference>() { }); + query = replaceStarWithColumns(query, columnMap); + } + + if (!(query instanceof Nodes.Select)) { + return sql; + } + + Nodes.Select select = (Nodes.Select) query; + List newExprs = new ArrayList<>(); + for (Expression expr : select.getExpressions()) { + newExprs.add(wrapWithCast(expr, columnName, dataType)); + } + + Expression result = new Nodes.Select(newExprs, select.isDistinct(), + select.getFrom(), select.getJoins(), select.getWhere(), + select.getGroupBy(), select.getHaving(), select.getOrderBy(), + select.getLimit(), select.getOffset()); + return DRILL.format(SqlGlot.generate(result, "DRILL")); + } catch (Exception e) { + logger.warn("Data type conversion failed: {}", e.getMessage(), e); + return null; + } + } + + /** + * Pretty-print a SQL string using sqlglot. + * + * @param sql the SQL string to format + * @return the formatted SQL, or the original if formatting fails + */ + public String formatSql(String sql) { + if (sql == null || sql.isEmpty()) { + return sql; + } + try { + return DRILL.format(sql); + } catch (Exception e) { + logger.debug("SQL formatting failed, returning original: {}", e.getMessage()); + return sql; + } + } + + /** + * Change the time grain of a temporal column in a SQL query using sqlglot AST manipulation. + * + * @param sql the SQL query + * @param columnName the temporal column to transform + * @param timeGrain the time grain (e.g. "MONTH", "YEAR") + * @param columnsJson optional JSON array string of column names (for star queries) + * @return the transformed SQL, or the original SQL if transformation fails + */ + public String changeTimeGrain(String sql, String columnName, + String timeGrain, String columnsJson) { + if (sql == null || sql.isEmpty()) { + return sql; + } + try { + String grain = timeGrain.toUpperCase(); + Optional parsed = DRILL.parseOne(sql); + if (!parsed.isPresent()) { + return sql; + } + Expression query = parsed.get(); + + // Handle star queries + if (isStarQuery(query)) { + if (columnsJson != null && !columnsJson.isEmpty()) { + List columnList = MAPPER.readValue(columnsJson, + new TypeReference>() { }); + Map columnMap = new HashMap<>(); + for (String col : columnList) { + columnMap.put(col, "VARCHAR"); + } + query = replaceStarWithColumns(query, columnMap); + } + } + + if (!(query instanceof Nodes.Select)) { + return sql; + } + + Nodes.Select select = (Nodes.Select) query; + List newExprs = new ArrayList<>(); + for (Expression expr : select.getExpressions()) { + newExprs.add(wrapWithDateTrunc(expr, columnName, grain)); + } + + Expression result = new Nodes.Select(newExprs, select.isDistinct(), + select.getFrom(), select.getJoins(), select.getWhere(), + select.getGroupBy(), select.getHaving(), select.getOrderBy(), + select.getLimit(), select.getOffset()); + return DRILL.format(SqlGlot.generate(result, "DRILL")); + } catch (Exception e) { + logger.warn("Time grain change failed: {}", e.getMessage(), e); + return sql; + } + } + + /** + * Returns whether the transpiler is available. Always true with Java sqlglot. + */ + public boolean isAvailable() { + return true; + } + + // ==================== Private Helpers ==================== + + /** + * Returns true if the query contains a star (*) in the outermost SELECT. + */ + private boolean isStarQuery(Expression query) { + List stars = query.findAll(Nodes.Star.class) + .collect(Collectors.toList()); + return !stars.isEmpty(); + } + + /** + * Expands a star query to use explicit column names. + */ + private Expression replaceStarWithColumns(Expression query, Map columnMap) { + if (!(query instanceof Nodes.Select)) { + return query; + } + Nodes.Select select = (Nodes.Select) query; + + List newExprs = new ArrayList<>(); + for (String col : columnMap.keySet()) { + newExprs.add(new Nodes.Identifier(col, true)); + } + + // Keep any non-star expressions from the original SELECT + for (Expression expr : select.getExpressions()) { + if (!(expr instanceof Nodes.Star)) { + newExprs.add(expr); + } + } + + return new Nodes.Select(newExprs, select.isDistinct(), + select.getFrom(), select.getJoins(), select.getWhere(), + select.getGroupBy(), select.getHaving(), select.getOrderBy(), + select.getLimit(), select.getOffset()); + } + + /** + * Wraps a select expression with CAST if it references the target column. + */ + private Expression wrapWithCast(Expression expr, String columnName, String dataType) { + if (expr instanceof Nodes.Alias) { + Nodes.Alias alias = (Nodes.Alias) expr; + Expression inner = alias.getExpression(); + if (hasIdentifierNamed(inner, columnName)) { + Nodes.Cast cast = new Nodes.Cast(inner, new Nodes.DataTypeExpr(dataType)); + return new Nodes.Alias(cast, alias.getAlias()); + } + } else if (expr instanceof Nodes.Identifier) { + if (((Nodes.Identifier) expr).getName().equalsIgnoreCase(columnName)) { + Nodes.Cast cast = new Nodes.Cast(expr, new Nodes.DataTypeExpr(dataType)); + return new Nodes.Alias(cast, columnName); + } + } else if (expr instanceof Nodes.Dot) { + if (hasIdentifierNamed(expr, columnName)) { + Nodes.Cast cast = new Nodes.Cast(expr, new Nodes.DataTypeExpr(dataType)); + return new Nodes.Alias(cast, columnName); + } + } else if (expr instanceof Nodes.Function) { + if (hasIdentifierNamed(expr, columnName)) { + Nodes.Cast cast = new Nodes.Cast(expr, new Nodes.DataTypeExpr(dataType)); + return new Nodes.Alias(cast, columnName); + } + } + return expr; + } + + /** + * Wraps a select expression with DATE_TRUNC if it references the target column. + */ + private Expression wrapWithDateTrunc(Expression expr, String columnName, String timeGrain) { + if (expr instanceof Nodes.Alias) { + Nodes.Alias alias = (Nodes.Alias) expr; + Expression inner = alias.getExpression(); + + // Case: already a DATE_TRUNC function — update the grain + if (inner instanceof Nodes.Function + && ((Nodes.Function) inner).getName().equalsIgnoreCase("DATE_TRUNC") + && hasIdentifierNamed(inner, columnName)) { + List args = ((Nodes.Function) inner).getArgs(); + List newArgs = new ArrayList<>(args); + newArgs.set(0, new Nodes.Literal(timeGrain, true)); + return new Nodes.Alias(new Nodes.Function("DATE_TRUNC", newArgs), alias.getAlias()); + } + + if (hasIdentifierNamed(inner, columnName)) { + Nodes.Function trunc = new Nodes.Function("DATE_TRUNC", + Arrays.asList(new Nodes.Literal(timeGrain, true), inner)); + return new Nodes.Alias(trunc, alias.getAlias()); + } + } else if (expr instanceof Nodes.Function + && ((Nodes.Function) expr).getName().equalsIgnoreCase("DATE_TRUNC") + && hasIdentifierNamed(expr, columnName)) { + // Naked DATE_TRUNC — update grain and add alias + List args = ((Nodes.Function) expr).getArgs(); + List newArgs = new ArrayList<>(args); + newArgs.set(0, new Nodes.Literal(timeGrain, true)); + return new Nodes.Alias(new Nodes.Function("DATE_TRUNC", newArgs), columnName); + } else if (expr instanceof Nodes.Identifier) { + if (((Nodes.Identifier) expr).getName().equalsIgnoreCase(columnName)) { + Nodes.Function trunc = new Nodes.Function("DATE_TRUNC", + Arrays.asList(new Nodes.Literal(timeGrain, true), expr)); + return new Nodes.Alias(trunc, columnName); + } + } else if (expr instanceof Nodes.Dot) { + if (hasIdentifierNamed(expr, columnName)) { + Nodes.Function trunc = new Nodes.Function("DATE_TRUNC", + Arrays.asList(new Nodes.Literal(timeGrain, true), expr)); + return new Nodes.Alias(trunc, columnName); + } + } else if (expr instanceof Nodes.Function) { + if (hasIdentifierNamed(expr, columnName)) { + Nodes.Function trunc = new Nodes.Function("DATE_TRUNC", + Arrays.asList(new Nodes.Literal(timeGrain, true), expr)); + return new Nodes.Alias(trunc, columnName); + } + } + return expr; + } + + /** + * Checks whether an expression tree contains an Identifier with the given name. + * In Java sqlglot, simple column references are parsed as Identifier nodes. + */ + private boolean hasIdentifierNamed(Expression expr, String columnName) { + if (expr instanceof Nodes.Identifier) { + return ((Nodes.Identifier) expr).getName().equalsIgnoreCase(columnName); + } + return expr.findAll(Nodes.Identifier.class) + .anyMatch(id -> ((Nodes.Identifier) id).getName().equalsIgnoreCase(columnName)); + } + + /** + * Resolves short table names to fully qualified names using schema metadata. + */ + private String ensureFullTableNames(String sql, String schemasJson) { + try { + List> schemas = MAPPER.readValue(schemasJson, + new TypeReference>>() { }); + if (schemas == null || schemas.isEmpty()) { + return sql; + } + + schemas = cleanupTableNames(schemas); + Optional parsed = DRILL.parseOne(sql); + if (!parsed.isPresent()) { + return sql; + } + Expression query = parsed.get(); + + // Count tables in query + List queryTables = query.findAll(Nodes.Table.class) + .collect(Collectors.toList()); + + // Build table-to-plugin mapping + Map tableToPluginMap = new HashMap<>(); + + for (Map schema : schemas) { + String pluginName = (String) schema.get("name"); + @SuppressWarnings("unchecked") + List> tables = (List>) schema.get("tables"); + if (tables == null) { + continue; + } + + for (Map table : tables) { + String tableName = (String) table.get("name"); + if (tableName == null) { + continue; + } + + if (tableName.contains(".")) { + String[] tableParts = tableName.split("\\.", 2); + if (startsWithKnownExtension(tableParts[1])) { + tableToPluginMap.put(tableName, pluginName + "." + tableName); + } + tableToPluginMap.put(tableParts[0], pluginName + "." + tableName); + String tableNameWithPlugin = pluginName + "." + tableParts[0]; + tableToPluginMap.put(tableNameWithPlugin, pluginName + "." + tableName); + } else if (tableName.isEmpty() && pluginName.contains(".")) { + String[] parts = pluginName.split("\\.", 2); + tableToPluginMap.put(parts[1], pluginName); + } else { + tableToPluginMap.put(tableName, pluginName + "." + tableName); + } + } + } + + // Apply table name replacements using transform + Expression result = query.transform(node -> { + if (node instanceof Nodes.Table) { + String tblName = ((Nodes.Table) node).getName(); + String fullName = tableToPluginMap.get(tblName); + if (fullName != null && fullName.contains(".")) { + String[] parts = fullName.split("\\.", 2); + return new Nodes.Table(parts[1], parts[0]); + } + } + return node; + }); + + // Fix aggregate query projection + result = fixAggregateQueryProjection(result); + + String resultSql = SqlGlot.generate(result, "DRILL"); + resultSql = removeTokens(resultSql); + return DRILL.format(resultSql); + } catch (Exception e) { + logger.debug("Table name resolution failed: {}", e.getMessage()); + return sql; + } + } + + /** + * Fixes aggregate queries where projected columns are missing from GROUP BY. + */ + private Expression fixAggregateQueryProjection(Expression query) { + if (!(query instanceof Nodes.Select)) { + return query; + } + Nodes.Select select = (Nodes.Select) query; + Nodes.GroupBy groupBy = select.getGroupBy(); + if (groupBy == null) { + return query; + } + + List groupExprs = new ArrayList<>(groupBy.getExpressions()); + boolean modified = false; + for (Expression expr : select.getExpressions()) { + if (expr instanceof Nodes.Identifier || expr instanceof Nodes.Column) { + boolean alreadyInGroup = groupExprs.stream() + .anyMatch(g -> g.toString().equals(expr.toString())); + if (!alreadyInGroup) { + groupExprs.add(expr); + modified = true; + } + } + } + + if (modified) { + return new Nodes.Select(select.getExpressions(), select.isDistinct(), + select.getFrom(), select.getJoins(), select.getWhere(), + new Nodes.GroupBy(groupExprs), select.getHaving(), + select.getOrderBy(), select.getLimit(), select.getOffset()); + } + return query; + } + + /** + * Removes token placeholders and restores original characters. + */ + private String removeTokens(String sql) { + for (Map.Entry entry : CHAR_MAPPING.entrySet()) { + if (sql.contains(entry.getValue())) { + sql = sql.replace(entry.getValue(), entry.getKey()); + } + } + return sql; + } + + /** + * Replaces illegal characters in a name with token placeholders. + */ + private String replaceIllegalCharacterWithToken(String name) { + for (Map.Entry entry : CHAR_MAPPING.entrySet()) { + if (name.contains(entry.getKey())) { + name = name.replace(entry.getKey(), entry.getValue()); + } + } + return name; + } + + /** + * Checks if a name starts with a known file extension. + */ + private boolean startsWithKnownExtension(String name) { + for (String ext : EXTENSIONS) { + if (name.startsWith(ext)) { + return true; + } + } + return false; + } + + /** + * Cleans up table names in schema metadata: + * removes .view.drill extensions and replaces illegal characters. + */ + private List> cleanupTableNames(List> schemas) { + for (Map schema : schemas) { + String name = (String) schema.get("name"); + if (name != null) { + schema.put("name", replaceIllegalCharacterWithToken(name)); + } + + @SuppressWarnings("unchecked") + List> tables = (List>) schema.get("tables"); + if (tables == null) { + continue; + } + + for (Map table : tables) { + String tableName = (String) table.get("name"); + if (tableName != null) { + tableName = tableName.replace(".view.drill", ""); + tableName = replaceIllegalCharacterWithToken(tableName); + table.put("name", tableName); + } + + @SuppressWarnings("unchecked") + List columns = (List) table.get("columns"); + if (columns != null) { + for (int j = 0; j < columns.size(); j++) { + columns.set(j, replaceIllegalCharacterWithToken(columns.get(j))); + } + } + } + } + return schemas; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java index c294f832d09..8102c97a7a4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java @@ -21,8 +21,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import jakarta.annotation.security.PermitAll; import jakarta.annotation.security.RolesAllowed; @@ -58,6 +60,13 @@ import org.apache.http.client.methods.HttpGet; import org.glassfish.jersey.server.mvc.Viewable; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Snapshot; +import com.codahale.metrics.Timer; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -104,6 +113,102 @@ public Viewable getStatus() { return ViewableWithPermissions.create(authEnabled.get(), "/rest/status.ftl", sc, getStatusJSON()); } + /** + * Returns the local Drillbit's metrics as JSON. + * Replaces the old Codahale MetricsServlet that was removed during + * the javax.servlet to jakarta.servlet migration. + */ + @GET + @Path(StatusResources.PATH_METRICS) + @Produces(MediaType.APPLICATION_JSON) + @Operation(externalDocs = @ExternalDocumentation(description = "Apache Drill REST API documentation:", url = "https://drill.apache.org/docs/rest-api-introduction/")) + public Map getLocalMetrics() { + MetricRegistry metrics = work.getContext().getMetrics(); + Map result = new LinkedHashMap<>(); + + // Gauges + Map gauges = new LinkedHashMap<>(); + for (Map.Entry entry : metrics.getGauges().entrySet()) { + Map g = new LinkedHashMap<>(); + g.put("value", entry.getValue().getValue()); + gauges.put(entry.getKey(), g); + } + result.put("gauges", gauges); + + // Counters + Map counters = new LinkedHashMap<>(); + for (Map.Entry entry : metrics.getCounters().entrySet()) { + Map c = new LinkedHashMap<>(); + c.put("count", entry.getValue().getCount()); + counters.put(entry.getKey(), c); + } + result.put("counters", counters); + + // Histograms + Map histograms = new LinkedHashMap<>(); + for (Map.Entry entry : metrics.getHistograms().entrySet()) { + Snapshot snap = entry.getValue().getSnapshot(); + Map h = new LinkedHashMap<>(); + h.put("count", entry.getValue().getCount()); + h.put("max", snap.getMax()); + h.put("mean", snap.getMean()); + h.put("min", snap.getMin()); + h.put("p50", snap.getMedian()); + h.put("p75", snap.get75thPercentile()); + h.put("p95", snap.get95thPercentile()); + h.put("p98", snap.get98thPercentile()); + h.put("p99", snap.get99thPercentile()); + h.put("p999", snap.get999thPercentile()); + h.put("stddev", snap.getStdDev()); + histograms.put(entry.getKey(), h); + } + result.put("histograms", histograms); + + // Meters + Map meters = new LinkedHashMap<>(); + for (Map.Entry entry : metrics.getMeters().entrySet()) { + Meter m = entry.getValue(); + Map mData = new LinkedHashMap<>(); + mData.put("count", m.getCount()); + mData.put("m1_rate", m.getOneMinuteRate()); + mData.put("m5_rate", m.getFiveMinuteRate()); + mData.put("m15_rate", m.getFifteenMinuteRate()); + mData.put("mean_rate", m.getMeanRate()); + mData.put("units", "events/second"); + meters.put(entry.getKey(), mData); + } + result.put("meters", meters); + + // Timers + Map timers = new LinkedHashMap<>(); + for (Map.Entry entry : metrics.getTimers().entrySet()) { + Timer t = entry.getValue(); + Snapshot snap = t.getSnapshot(); + Map tData = new LinkedHashMap<>(); + tData.put("count", t.getCount()); + tData.put("max", snap.getMax()); + tData.put("mean", snap.getMean()); + tData.put("min", snap.getMin()); + tData.put("p50", snap.getMedian()); + tData.put("p75", snap.get75thPercentile()); + tData.put("p95", snap.get95thPercentile()); + tData.put("p98", snap.get98thPercentile()); + tData.put("p99", snap.get99thPercentile()); + tData.put("p999", snap.get999thPercentile()); + tData.put("stddev", snap.getStdDev()); + tData.put("m1_rate", t.getOneMinuteRate()); + tData.put("m5_rate", t.getFiveMinuteRate()); + tData.put("m15_rate", t.getFifteenMinuteRate()); + tData.put("mean_rate", t.getMeanRate()); + tData.put("duration_units", "seconds"); + tData.put("rate_units", "calls/second"); + timers.put(entry.getKey(), tData); + } + result.put("timers", timers); + + return result; + } + @GET @Path(StatusResources.PATH_METRICS + "/{hostname}") @Produces(MediaType.APPLICATION_JSON) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/TestConnectionResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/TestConnectionResources.java new file mode 100644 index 00000000000..9ee887a61ec --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/TestConnectionResources.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import jakarta.annotation.security.RolesAllowed; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.logical.security.CredentialsProvider; +import org.apache.drill.exec.store.ClassPathFileSystem; +import org.apache.drill.exec.store.LocalSyncableFileSystem; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.dfs.BoxFileSystem; +import org.apache.drill.exec.store.dfs.DropboxFileSystem; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.exec.store.security.UsernamePasswordCredentials; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.sftp.SFTPFileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE; + +/** + * REST resource for testing storage plugin connections. + * Supports FileSystem and JDBC plugin types. + */ +@Path("/") +@RolesAllowed(ADMIN_ROLE) +public class TestConnectionResources { + private static final Logger logger = LoggerFactory.getLogger(TestConnectionResources.class); + + private static final int CONNECTION_TIMEOUT_SECONDS = 10; + + @Inject + StoragePluginRegistry storage; + + @POST + @Path("/storage/test-connection") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public TestConnectionResult testConnection(PluginConfigWrapper pluginWrapper) { + try { + StoragePluginConfig cfg = pluginWrapper.getConfig(); + if (cfg instanceof FileSystemConfig) { + return testFileSystemConnection((FileSystemConfig) cfg); + } else if (isJdbcConfig(cfg)) { + return testJdbcConnection(cfg); + } else { + return new TestConnectionResult(false, + "Test connection is not supported for this plugin type: " + + cfg.getClass().getSimpleName(), null); + } + } catch (Exception e) { + logger.error("Unexpected error testing connection", e); + return new TestConnectionResult(false, + "Unexpected error: " + e.getMessage(), + e.getClass().getName()); + } + } + + private TestConnectionResult testFileSystemConnection(FileSystemConfig config) { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Configuration conf = new Configuration(); + Optional.ofNullable(config.getConfig()) + .ifPresent(c -> c.forEach(conf::set)); + + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.getConnection()); + conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName()); + conf.set("fs.dropbox.impl", DropboxFileSystem.class.getName()); + conf.set("fs.sftp.impl", SFTPFileSystem.class.getName()); + conf.set("fs.box.impl", BoxFileSystem.class.getName()); + conf.set("fs.drill-local.impl", LocalSyncableFileSystem.class.getName()); + + CredentialsProvider credentialsProvider = config.getCredentialsProvider(); + if (credentialsProvider != null) { + credentialsProvider.getCredentials().forEach(conf::set); + } + + Future future = executor.submit(new FsTestCallable(conf)); + return future.get(CONNECTION_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (TimeoutException e) { + return new TestConnectionResult(false, + "Connection timed out after " + CONNECTION_TIMEOUT_SECONDS + + " seconds. The endpoint may be unreachable.", null); + } catch (Exception e) { + Throwable cause = e.getCause() != null ? e.getCause() : e; + return new TestConnectionResult(false, + cause.getMessage(), + cause.getClass().getName()); + } finally { + executor.shutdownNow(); + } + } + + /** + * Check if a config is a JDBC storage config without a compile-time dependency + * on the contrib/storage-jdbc module. + */ + private boolean isJdbcConfig(StoragePluginConfig cfg) { + return "JdbcStorageConfig".equals(cfg.getClass().getSimpleName()); + } + + /** + * Test a JDBC connection using reflection to access driver/url from the config, + * since JdbcStorageConfig lives in the contrib/storage-jdbc module. + */ + private TestConnectionResult testJdbcConnection(StoragePluginConfig config) { + String driver; + String url; + try { + Method getDriver = config.getClass().getMethod("getDriver"); + Method getUrl = config.getClass().getMethod("getUrl"); + driver = (String) getDriver.invoke(config); + url = (String) getUrl.invoke(config); + } catch (Exception e) { + return new TestConnectionResult(false, + "Unable to read JDBC config properties: " + e.getMessage(), + e.getClass().getName()); + } + + // Load the JDBC driver class + try { + Class.forName(driver); + } catch (ClassNotFoundException e) { + return new TestConnectionResult(false, + "JDBC driver class not found: " + driver + + ". Make sure the driver JAR is in Drill's classpath.", + e.getClass().getName()); + } + + // Get credentials + String username = null; + String password = null; + CredentialsProvider credentialsProvider = config.getCredentialsProvider(); + if (credentialsProvider != null) { + Optional creds = + new UsernamePasswordCredentials.Builder() + .setCredentialsProvider(credentialsProvider) + .build(); + if (creds.isPresent()) { + username = creds.get().getUsername(); + password = creds.get().getPassword(); + } + } + + // Attempt to connect + DriverManager.setLoginTimeout(CONNECTION_TIMEOUT_SECONDS); + Connection conn = null; + try { + if (username != null) { + conn = DriverManager.getConnection(url, username, password); + } else { + conn = DriverManager.getConnection(url); + } + return new TestConnectionResult(true, + "Successfully connected to " + url, null); + } catch (SQLException e) { + return new TestConnectionResult(false, + e.getMessage(), + e.getClass().getName()); + } finally { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.warn("Error closing test JDBC connection", e); + } + } + } + } + + /** + * Callable that performs the FileSystem connection test + * so it can be executed with a timeout. + */ + private static class FsTestCallable implements Callable { + private final Configuration conf; + + FsTestCallable(Configuration conf) { + this.conf = conf; + } + + @Override + public TestConnectionResult call() throws Exception { + FileSystem fs = null; + try { + fs = FileSystem.get(conf); + fs.exists(new org.apache.hadoop.fs.Path("/")); + return new TestConnectionResult(true, + "Successfully connected to " + conf.get(FileSystem.FS_DEFAULT_NAME_KEY), + null); + } finally { + if (fs != null) { + try { + fs.close(); + } catch (Exception e) { + logger.warn("Error closing test FileSystem", e); + } + } + } + } + + private static final Logger logger = LoggerFactory.getLogger(FsTestCallable.class); + } + + /** + * JSON response for test-connection endpoint. + */ + public static class TestConnectionResult { + private final boolean success; + private final String message; + private final String errorClass; + + @JsonCreator + public TestConnectionResult( + @JsonProperty("success") boolean success, + @JsonProperty("message") String message, + @JsonProperty("errorClass") String errorClass) { + this.success = success; + this.message = message; + this.errorClass = errorClass; + } + + public boolean isSuccess() { + return success; + } + + public String getMessage() { + return message; + } + + public String getErrorClass() { + return errorClass; + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/TranspileResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/TranspileResources.java new file mode 100644 index 00000000000..a329d79ace5 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/TranspileResources.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; + +import jakarta.annotation.security.RolesAllowed; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * REST resource for SQL dialect transpilation via Java sqlglot. + * Converts SQL from one dialect (e.g. MySQL) to another (e.g. Apache Drill). + */ +@Path("/api/v1/transpile") +@Tag(name = "Transpiler", description = "SQL dialect transpilation via sqlglot") +@RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE) +public class TranspileResources { + + // ==================== Request/Response Models ==================== + + public static class TranspileRequest { + @JsonProperty + public String sql; + + @JsonProperty + public String sourceDialect; + + @JsonProperty + public String targetDialect; + + @JsonProperty + public List> schemas; + + public TranspileRequest() { + } + + @JsonCreator + public TranspileRequest( + @JsonProperty("sql") String sql, + @JsonProperty("sourceDialect") String sourceDialect, + @JsonProperty("targetDialect") String targetDialect, + @JsonProperty("schemas") List> schemas) { + this.sql = sql; + this.sourceDialect = sourceDialect; + this.targetDialect = targetDialect; + this.schemas = schemas; + } + } + + public static class TranspileResponse { + @JsonProperty + public String sql; + + @JsonProperty + public boolean success; + + @JsonProperty + public String formattedOriginal; + + public TranspileResponse(String sql, boolean success) { + this.sql = sql; + this.success = success; + } + + public TranspileResponse(String sql, boolean success, String formattedOriginal) { + this.sql = sql; + this.success = success; + this.formattedOriginal = formattedOriginal; + } + } + + // ==================== Endpoints ==================== + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Transpile SQL", + description = "Transpiles SQL from one dialect to another using sqlglot") + public Response transpile(TranspileRequest request) { + if (request.sql == null || request.sql.trim().isEmpty()) { + return Response.ok(new TranspileResponse("", true)).build(); + } + + SqlTranspiler transpiler = SqlTranspiler.getInstance(); + String sourceDialect = request.sourceDialect != null ? request.sourceDialect : "mysql"; + String targetDialect = request.targetDialect != null ? request.targetDialect : "drill"; + + String schemasJson = "[]"; + if (request.schemas != null && !request.schemas.isEmpty()) { + try { + schemasJson = new ObjectMapper().writeValueAsString(request.schemas); + } catch (Exception e) { + schemasJson = "[]"; + } + } + + String result = transpiler.transpile(request.sql, sourceDialect, targetDialect, schemasJson); + return Response.ok(new TranspileResponse(result, true)).build(); + } + + // ==================== Convert Data Type ==================== + + public static class ConvertDataTypeRequest { + @JsonProperty + public String sql; + + @JsonProperty + public String columnName; + + @JsonProperty + public String dataType; + + @JsonProperty + public Map columns; + + public ConvertDataTypeRequest() { + } + + @JsonCreator + public ConvertDataTypeRequest( + @JsonProperty("sql") String sql, + @JsonProperty("columnName") String columnName, + @JsonProperty("dataType") String dataType, + @JsonProperty("columns") Map columns) { + this.sql = sql; + this.columnName = columnName; + this.dataType = dataType; + this.columns = columns; + } + } + + @POST + @Path("/convert-type") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Convert column data type", + description = "Wraps a column in a CAST expression using sqlglot AST manipulation") + public Response convertDataType(ConvertDataTypeRequest request) { + if (request.sql == null || request.sql.trim().isEmpty()) { + return Response.ok(new TranspileResponse("", true)).build(); + } + if (request.columnName == null || request.dataType == null) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new TranspileResponse(request.sql, false)).build(); + } + + SqlTranspiler transpiler = SqlTranspiler.getInstance(); + String columnsJson = null; + if (request.columns != null && !request.columns.isEmpty()) { + try { + columnsJson = new ObjectMapper().writeValueAsString(request.columns); + } catch (Exception e) { + columnsJson = null; + } + } + + String result = transpiler.convertDataType( + request.sql, request.columnName, request.dataType, columnsJson); + if (result == null) { + return Response.ok(new TranspileResponse(request.sql, false)).build(); + } + String formattedOriginal = transpiler.formatSql(request.sql); + return Response.ok(new TranspileResponse(result, true, formattedOriginal)).build(); + } + + // ==================== Change Time Grain ==================== + + public static class TimeGrainRequest { + @JsonProperty + public String sql; + + @JsonProperty + public String columnName; + + @JsonProperty + public String timeGrain; + + @JsonProperty + public List columns; + + public TimeGrainRequest() { + } + + @JsonCreator + public TimeGrainRequest( + @JsonProperty("sql") String sql, + @JsonProperty("columnName") String columnName, + @JsonProperty("timeGrain") String timeGrain, + @JsonProperty("columns") List columns) { + this.sql = sql; + this.columnName = columnName; + this.timeGrain = timeGrain; + this.columns = columns; + } + } + + @POST + @Path("/time-grain") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Change time grain", + description = "Wraps a temporal column with DATE_TRUNC using sqlglot AST manipulation") + public Response changeTimeGrain(TimeGrainRequest request) { + if (request.sql == null || request.sql.trim().isEmpty()) { + return Response.ok(new TranspileResponse("", true)).build(); + } + if (request.columnName == null || request.timeGrain == null) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new TranspileResponse(request.sql, false)).build(); + } + + SqlTranspiler transpiler = SqlTranspiler.getInstance(); + String columnsJson = null; + if (request.columns != null && !request.columns.isEmpty()) { + try { + columnsJson = new ObjectMapper().writeValueAsString(request.columns); + } catch (Exception e) { + columnsJson = null; + } + } + + String result = transpiler.changeTimeGrain( + request.sql, request.columnName, request.timeGrain, columnsJson); + return Response.ok(new TranspileResponse(result, true)).build(); + } + + // ==================== Format SQL ==================== + + public static class FormatRequest { + @JsonProperty + public String sql; + + public FormatRequest() { + } + + @JsonCreator + public FormatRequest(@JsonProperty("sql") String sql) { + this.sql = sql; + } + } + + @POST + @Path("/format") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Format SQL", + description = "Pretty-prints a SQL string using sqlglot") + public Response formatSql(FormatRequest request) { + if (request.sql == null || request.sql.trim().isEmpty()) { + return Response.ok(new TranspileResponse("", true)).build(); + } + String result = SqlTranspiler.getInstance().formatSql(request.sql); + return Response.ok(new TranspileResponse(result, true)).build(); + } + + @GET + @Path("/status") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get transpiler status", + description = "Returns whether sqlglot transpilation is available") + public Response status() { + Map status = Collections.singletonMap( + "available", SqlTranspiler.getInstance().isAvailable()); + return Response.ok(status).build(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/VisualizationResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/VisualizationResources.java new file mode 100644 index 00000000000..e35879e1eb2 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/VisualizationResources.java @@ -0,0 +1,615 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.exec.exception.StoreException; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; +import org.apache.drill.exec.store.sys.PersistentStore; +import org.apache.drill.exec.store.sys.PersistentStoreConfig; +import org.apache.drill.exec.store.sys.PersistentStoreProvider; +import org.apache.drill.exec.work.WorkManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.annotation.security.RolesAllowed; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * REST API for managing visualizations. + * Visualizations are charts created from saved query results. + */ +@Path("/api/v1/visualizations") +@Tag(name = "Visualizations", description = "APIs for managing chart visualizations") +@RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE) +public class VisualizationResources { + private static final Logger logger = LoggerFactory.getLogger(VisualizationResources.class); + private static final String STORE_NAME = "drill.sqllab.visualizations"; + + @Inject + WorkManager workManager; + + @Inject + DrillUserPrincipal principal; + + @Inject + PersistentStoreProvider storeProvider; + + private static volatile PersistentStore cachedStore; + + // ==================== Model Classes ==================== + + /** + * Configuration for visualization axes and metrics. + */ + public static class VisualizationConfig { + @JsonProperty + private String xAxis; + @JsonProperty + private String yAxis; + @JsonProperty + private List metrics; + @JsonProperty + private List dimensions; + @JsonProperty + private Map chartOptions; + @JsonProperty + private String colorScheme; + + public VisualizationConfig() { + } + + @JsonCreator + public VisualizationConfig( + @JsonProperty("xAxis") String xAxis, + @JsonProperty("yAxis") String yAxis, + @JsonProperty("metrics") List metrics, + @JsonProperty("dimensions") List dimensions, + @JsonProperty("chartOptions") Map chartOptions, + @JsonProperty("colorScheme") String colorScheme) { + this.xAxis = xAxis; + this.yAxis = yAxis; + this.metrics = metrics; + this.dimensions = dimensions; + this.chartOptions = chartOptions; + this.colorScheme = colorScheme; + } + + public String getXAxis() { + return xAxis; + } + + public String getYAxis() { + return yAxis; + } + + public List getMetrics() { + return metrics; + } + + public List getDimensions() { + return dimensions; + } + + public Map getChartOptions() { + return chartOptions; + } + + public String getColorScheme() { + return colorScheme; + } + + public void setXAxis(String xAxis) { + this.xAxis = xAxis; + } + + public void setYAxis(String yAxis) { + this.yAxis = yAxis; + } + + public void setMetrics(List metrics) { + this.metrics = metrics; + } + + public void setDimensions(List dimensions) { + this.dimensions = dimensions; + } + + public void setChartOptions(Map chartOptions) { + this.chartOptions = chartOptions; + } + + public void setColorScheme(String colorScheme) { + this.colorScheme = colorScheme; + } + } + + /** + * Visualization model for persistence. + */ + public static class Visualization { + @JsonProperty + private String id; + @JsonProperty + private String name; + @JsonProperty + private String description; + @JsonProperty + private String savedQueryId; + @JsonProperty + private String chartType; + @JsonProperty + private VisualizationConfig config; + @JsonProperty + private String owner; + @JsonProperty + private long createdAt; + @JsonProperty + private long updatedAt; + @JsonProperty + private boolean isPublic; + @JsonProperty + private String sql; + @JsonProperty + private String defaultSchema; + + public Visualization() { + } + + @JsonCreator + public Visualization( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("description") String description, + @JsonProperty("savedQueryId") String savedQueryId, + @JsonProperty("chartType") String chartType, + @JsonProperty("config") VisualizationConfig config, + @JsonProperty("owner") String owner, + @JsonProperty("createdAt") long createdAt, + @JsonProperty("updatedAt") long updatedAt, + @JsonProperty("isPublic") boolean isPublic, + @JsonProperty("sql") String sql, + @JsonProperty("defaultSchema") String defaultSchema) { + this.id = id; + this.name = name; + this.description = description; + this.savedQueryId = savedQueryId; + this.chartType = chartType; + this.config = config; + this.owner = owner; + this.createdAt = createdAt; + this.updatedAt = updatedAt; + this.isPublic = isPublic; + this.sql = sql; + this.defaultSchema = defaultSchema; + } + + // Getters + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getSavedQueryId() { + return savedQueryId; + } + + public String getChartType() { + return chartType; + } + + public VisualizationConfig getConfig() { + return config; + } + + public String getOwner() { + return owner; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getUpdatedAt() { + return updatedAt; + } + + public boolean isPublic() { + return isPublic; + } + + public String getSql() { + return sql; + } + + public String getDefaultSchema() { + return defaultSchema; + } + + // Setters for updates + public void setName(String name) { + this.name = name; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setSavedQueryId(String savedQueryId) { + this.savedQueryId = savedQueryId; + } + + public void setChartType(String chartType) { + this.chartType = chartType; + } + + public void setConfig(VisualizationConfig config) { + this.config = config; + } + + public void setUpdatedAt(long updatedAt) { + this.updatedAt = updatedAt; + } + + public void setPublic(boolean isPublic) { + this.isPublic = isPublic; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public void setDefaultSchema(String defaultSchema) { + this.defaultSchema = defaultSchema; + } + } + + /** + * Request body for creating a new visualization. + */ + public static class CreateVisualizationRequest { + @JsonProperty + public String name; + @JsonProperty + public String description; + @JsonProperty + public String savedQueryId; + @JsonProperty + public String chartType; + @JsonProperty + public VisualizationConfig config; + @JsonProperty + public boolean isPublic; + @JsonProperty + public String sql; + @JsonProperty + public String defaultSchema; + } + + /** + * Request body for updating a visualization. + */ + public static class UpdateVisualizationRequest { + @JsonProperty + public String name; + @JsonProperty + public String description; + @JsonProperty + public String savedQueryId; + @JsonProperty + public String chartType; + @JsonProperty + public VisualizationConfig config; + @JsonProperty + public Boolean isPublic; + @JsonProperty + public String sql; + @JsonProperty + public String defaultSchema; + } + + /** + * Response containing a list of visualizations. + */ + public static class VisualizationsResponse { + @JsonProperty + public List visualizations; + + public VisualizationsResponse(List visualizations) { + this.visualizations = visualizations; + } + } + + /** + * Simple message response. + */ + public static class MessageResponse { + @JsonProperty + public String message; + + public MessageResponse(String message) { + this.message = message; + } + } + + // ==================== API Endpoints ==================== + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "List visualizations", description = "Returns all visualizations accessible by the current user") + public VisualizationsResponse listVisualizations() { + logger.debug("Listing visualizations for user: {}", getCurrentUser()); + + List visualizations = new ArrayList<>(); + String currentUser = getCurrentUser(); + + try { + PersistentStore store = getStore(); + Iterator> iterator = store.getAll(); + + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + Visualization viz = entry.getValue(); + + // Return visualizations owned by user or public visualizations + if (viz.getOwner().equals(currentUser) || viz.isPublic()) { + visualizations.add(viz); + } + } + } catch (Exception e) { + logger.error("Error listing visualizations", e); + throw new DrillRuntimeException("Failed to list visualizations: " + e.getMessage(), e); + } + + return new VisualizationsResponse(visualizations); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Create visualization", description = "Creates a new visualization") + public Response createVisualization(CreateVisualizationRequest request) { + logger.debug("Creating visualization: {}", request.name); + + if (request.name == null || request.name.trim().isEmpty()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("Visualization name is required")) + .build(); + } + + if (request.chartType == null || request.chartType.trim().isEmpty()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new MessageResponse("Chart type is required")) + .build(); + } + + String id = UUID.randomUUID().toString(); + long now = Instant.now().toEpochMilli(); + + Visualization viz = new Visualization( + id, + request.name.trim(), + request.description, + request.savedQueryId, + request.chartType, + request.config != null ? request.config : new VisualizationConfig(), + getCurrentUser(), + now, + now, + request.isPublic, + request.sql, + request.defaultSchema + ); + + try { + PersistentStore store = getStore(); + store.put(id, viz); + } catch (Exception e) { + logger.error("Error creating visualization", e); + throw new DrillRuntimeException("Failed to create visualization: " + e.getMessage(), e); + } + + return Response.status(Response.Status.CREATED).entity(viz).build(); + } + + @GET + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Get visualization", description = "Returns a visualization by ID") + public Response getVisualization( + @Parameter(description = "Visualization ID") @PathParam("id") String id) { + logger.debug("Getting visualization: {}", id); + + try { + PersistentStore store = getStore(); + Visualization viz = store.get(id); + + if (viz == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Visualization not found")) + .build(); + } + + // Check access permissions + if (!viz.getOwner().equals(getCurrentUser()) && !viz.isPublic()) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Access denied")) + .build(); + } + + return Response.ok(viz).build(); + } catch (Exception e) { + logger.error("Error getting visualization", e); + throw new DrillRuntimeException("Failed to get visualization: " + e.getMessage(), e); + } + } + + @PUT + @Path("/{id}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Update visualization", description = "Updates an existing visualization") + public Response updateVisualization( + @Parameter(description = "Visualization ID") @PathParam("id") String id, + UpdateVisualizationRequest request) { + logger.debug("Updating visualization: {}", id); + + try { + PersistentStore store = getStore(); + Visualization viz = store.get(id); + + if (viz == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Visualization not found")) + .build(); + } + + // Only owner can update + if (!viz.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can update this visualization")) + .build(); + } + + // Update fields if provided + if (request.name != null) { + viz.setName(request.name.trim()); + } + if (request.description != null) { + viz.setDescription(request.description); + } + if (request.savedQueryId != null) { + viz.setSavedQueryId(request.savedQueryId); + } + if (request.chartType != null) { + viz.setChartType(request.chartType); + } + if (request.config != null) { + viz.setConfig(request.config); + } + if (request.isPublic != null) { + viz.setPublic(request.isPublic); + } + if (request.sql != null) { + viz.setSql(request.sql); + } + if (request.defaultSchema != null) { + viz.setDefaultSchema(request.defaultSchema); + } + + viz.setUpdatedAt(Instant.now().toEpochMilli()); + + store.put(id, viz); + + return Response.ok(viz).build(); + } catch (Exception e) { + logger.error("Error updating visualization", e); + throw new DrillRuntimeException("Failed to update visualization: " + e.getMessage(), e); + } + } + + @DELETE + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @Operation(summary = "Delete visualization", description = "Deletes a visualization") + public Response deleteVisualization( + @Parameter(description = "Visualization ID") @PathParam("id") String id) { + logger.debug("Deleting visualization: {}", id); + + try { + PersistentStore store = getStore(); + Visualization viz = store.get(id); + + if (viz == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity(new MessageResponse("Visualization not found")) + .build(); + } + + // Only owner can delete + if (!viz.getOwner().equals(getCurrentUser())) { + return Response.status(Response.Status.FORBIDDEN) + .entity(new MessageResponse("Only the owner can delete this visualization")) + .build(); + } + + store.delete(id); + + return Response.ok(new MessageResponse("Visualization deleted successfully")).build(); + } catch (Exception e) { + logger.error("Error deleting visualization", e); + throw new DrillRuntimeException("Failed to delete visualization: " + e.getMessage(), e); + } + } + + // ==================== Helper Methods ==================== + + private PersistentStore getStore() { + if (cachedStore == null) { + synchronized (VisualizationResources.class) { + if (cachedStore == null) { + try { + cachedStore = storeProvider.getOrCreateStore( + PersistentStoreConfig.newJacksonBuilder( + workManager.getContext().getLpPersistence().getMapper(), + Visualization.class + ) + .name(STORE_NAME) + .build() + ); + } catch (StoreException e) { + throw new DrillRuntimeException("Failed to access visualizations store", e); + } + } + } + } + return cachedStore; + } + + private String getCurrentUser() { + return principal.getName(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java index e4fc2ba2a4b..98190a948de 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java @@ -101,6 +101,7 @@ public class WebServer implements AutoCloseable { private static final int PORT_HUNT_TRIES = 100; private static final String BASE_STATIC_PATH = "/rest/static/"; private static final String DRILL_ICON_RESOURCE_RELATIVE_PATH = "img/drill.ico"; + private static final String SQLLAB_WEBAPP_PATH = "/webapp/"; private final DrillConfig config; private final MetricRegistry metrics; @@ -214,6 +215,12 @@ private ServletContextHandler createServletContextHandler(final boolean authEnab staticHolder.setInitParameter("pathInfoOnly", "true"); servletContextHandler.addServlet(staticHolder, "/static/*"); + // Add SQL Lab React SPA servlet + // Uses custom servlet for SPA client-side routing support + final ServletHolder sqlLabHolder = new ServletHolder("sqllab", SqlLabSpaServlet.class); + servletContextHandler.addServlet(sqlLabHolder, "/sqllab/*"); + logger.info("SQL Lab React app configured at /sqllab/*"); + // Store the dependencies in the holder BEFORE creating the servlet // When Jersey instantiates DrillRestServerApplication (which extends DrillRestServer), // it will retrieve these dependencies and pass them to the parent constructor @@ -260,7 +267,7 @@ private ServletContextHandler createServletContextHandler(final boolean authEnab holder.setInitParameter(CrossOriginFilter.ALLOW_CREDENTIALS_PARAM, String.valueOf(config.getBoolean(ExecConstants.HTTP_CORS_CREDENTIALS))); - for (String path : new String[]{"*.json", "/storage/*/enable/*", "/status*"}) { + for (String path : new String[]{"*.json", "/storage/*/enable/*", "/status*", "/api/*"}) { servletContextHandler.addFilter(holder, path, EnumSet.of(DispatcherType.REQUEST)); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/AnthropicProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/AnthropicProvider.java new file mode 100644 index 00000000000..3a0dd6c2e04 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/AnthropicProvider.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest.ai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * LLM provider for the Anthropic Claude API. + * Calls /v1/messages with stream:true, translates Anthropic SSE events + * (message_start, content_block_delta, etc.) to the normalized format. + */ +public class AnthropicProvider implements LlmProvider { + + private static final Logger logger = LoggerFactory.getLogger(AnthropicProvider.class); + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final MediaType JSON_TYPE = MediaType.parse("application/json"); + private static final String DEFAULT_ENDPOINT = "https://api.anthropic.com"; + private static final String ANTHROPIC_VERSION = "2023-06-01"; + + private final OkHttpClient httpClient; + + public AnthropicProvider() { + this.httpClient = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build(); + } + + @Override + public String getId() { + return "anthropic"; + } + + @Override + public String getDisplayName() { + return "Anthropic Claude"; + } + + @Override + public void streamChatCompletion(LlmConfig config, List messages, + List tools, OutputStream out) throws Exception { + + String endpoint = config.getApiEndpoint(); + if (endpoint == null || endpoint.isEmpty()) { + endpoint = DEFAULT_ENDPOINT; + } + if (endpoint.endsWith("/")) { + endpoint = endpoint.substring(0, endpoint.length() - 1); + } + String url = endpoint + "/v1/messages"; + + ObjectNode requestBody = buildRequestBody(config, messages, tools); + + Request request = new Request.Builder() + .url(url) + .post(RequestBody.create(requestBody.toString(), JSON_TYPE)) + .addHeader("Content-Type", "application/json") + .addHeader("x-api-key", config.getApiKey() != null ? config.getApiKey() : "") + .addHeader("anthropic-version", ANTHROPIC_VERSION) + .build(); + + try (Response response = httpClient.newCall(request).execute()) { + if (!response.isSuccessful()) { + String errorBody = ""; + ResponseBody body = response.body(); + if (body != null) { + errorBody = body.string(); + } + String errorMsg = "Anthropic API error " + response.code() + ": " + errorBody; + logger.error(errorMsg); + writeSseEvent(out, "error", "{\"message\":" + MAPPER.writeValueAsString(errorMsg) + "}"); + return; + } + + ResponseBody body = response.body(); + if (body == null) { + writeSseEvent(out, "error", "{\"message\":\"Empty response from Anthropic API\"}"); + return; + } + + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(body.byteStream(), StandardCharsets.UTF_8))) { + processAnthropicStream(reader, out); + } + } + } + + @Override + public ValidationResult validateConfig(LlmConfig config) { + if (config.getApiKey() == null || config.getApiKey().isEmpty()) { + return ValidationResult.error("API key is required for Anthropic"); + } + if (config.getModel() == null || config.getModel().isEmpty()) { + return ValidationResult.error("Model name is required"); + } + return ValidationResult.ok("Configuration is valid"); + } + + private ObjectNode buildRequestBody(LlmConfig config, List messages, + List tools) { + ObjectNode body = MAPPER.createObjectNode(); + body.put("model", config.getModel()); + body.put("max_tokens", config.getMaxTokens()); + body.put("temperature", config.getTemperature()); + body.put("stream", true); + + // Extract system message + String systemContent = null; + ArrayNode messagesArray = body.putArray("messages"); + + for (ChatMessage msg : messages) { + if ("system".equals(msg.getRole())) { + systemContent = msg.getContent(); + continue; + } + + ObjectNode msgNode = messagesArray.addObject(); + msgNode.put("role", msg.getRole()); + + if ("tool".equals(msg.getRole())) { + // Anthropic uses tool_result content blocks + ArrayNode contentArray = msgNode.putArray("content"); + ObjectNode block = contentArray.addObject(); + block.put("type", "tool_result"); + block.put("tool_use_id", msg.getToolCallId()); + block.put("content", msg.getContent() != null ? msg.getContent() : ""); + } else if ("assistant".equals(msg.getRole()) && msg.getToolCalls() != null + && !msg.getToolCalls().isEmpty()) { + // Assistant message with tool calls + ArrayNode contentArray = msgNode.putArray("content"); + + if (msg.getContent() != null && !msg.getContent().isEmpty()) { + ObjectNode textBlock = contentArray.addObject(); + textBlock.put("type", "text"); + textBlock.put("text", msg.getContent()); + } + + for (ToolCall tc : msg.getToolCalls()) { + ObjectNode toolUseBlock = contentArray.addObject(); + toolUseBlock.put("type", "tool_use"); + toolUseBlock.put("id", tc.getId()); + toolUseBlock.put("name", tc.getName()); + try { + toolUseBlock.set("input", MAPPER.readTree(tc.getArguments())); + } catch (Exception e) { + toolUseBlock.putObject("input"); + } + } + } else { + msgNode.put("content", msg.getContent() != null ? msg.getContent() : ""); + } + } + + if (systemContent != null) { + body.put("system", systemContent); + } + + // Tools + if (tools != null && !tools.isEmpty()) { + ArrayNode toolsArray = body.putArray("tools"); + for (ToolDefinition tool : tools) { + ObjectNode toolNode = toolsArray.addObject(); + toolNode.put("name", tool.getName()); + toolNode.put("description", tool.getDescription()); + toolNode.set("input_schema", MAPPER.valueToTree(tool.getParameters())); + } + } + + return body; + } + + private void processAnthropicStream(BufferedReader reader, OutputStream out) throws Exception { + // Track content blocks by index + Map blockTypes = new HashMap<>(); + Map toolUseIds = new HashMap<>(); + Map toolUseNames = new HashMap<>(); + boolean hasToolUse = false; + + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + + if (line.isEmpty()) { + continue; + } + + if (!line.startsWith("data: ")) { + continue; + } + + String data = line.substring(6).trim(); + + try { + JsonNode event = MAPPER.readTree(data); + String type = event.has("type") ? event.get("type").asText() : ""; + + switch (type) { + case "content_block_start": + handleContentBlockStart(event, blockTypes, toolUseIds, toolUseNames, out); + if (event.has("content_block")) { + JsonNode block = event.get("content_block"); + if ("tool_use".equals(block.path("type").asText())) { + hasToolUse = true; + } + } + break; + + case "content_block_delta": + handleContentBlockDelta(event, blockTypes, toolUseIds, out); + break; + + case "content_block_stop": + handleContentBlockStop(event, blockTypes, toolUseIds, out); + break; + + case "message_delta": + // Check for stop_reason + JsonNode messageDelta = event.get("delta"); + if (messageDelta != null && messageDelta.has("stop_reason")) { + String stopReason = messageDelta.get("stop_reason").asText(); + if ("tool_use".equals(stopReason) || hasToolUse) { + writeSseEvent(out, "done", "{\"finish_reason\":\"tool_calls\"}"); + } else { + writeSseEvent(out, "done", "{\"finish_reason\":\"stop\"}"); + } + return; + } + break; + + case "message_stop": + if (hasToolUse) { + writeSseEvent(out, "done", "{\"finish_reason\":\"tool_calls\"}"); + } else { + writeSseEvent(out, "done", "{\"finish_reason\":\"stop\"}"); + } + return; + + case "error": + JsonNode errorNode = event.get("error"); + String errorMsg = errorNode != null ? errorNode.toString() : "Unknown error"; + writeSseEvent(out, "error", "{\"message\":" + MAPPER.writeValueAsString(errorMsg) + "}"); + return; + + default: + // ping, message_start, etc. — skip + break; + } + } catch (Exception e) { + logger.warn("Error parsing Anthropic SSE event: {}", data, e); + } + } + } + + private void handleContentBlockStart(JsonNode event, + Map blockTypes, + Map toolUseIds, + Map toolUseNames, + OutputStream out) throws Exception { + + int index = event.path("index").asInt(0); + JsonNode block = event.get("content_block"); + if (block == null) { + return; + } + + String blockType = block.path("type").asText(); + blockTypes.put(index, blockType); + + if ("tool_use".equals(blockType)) { + String id = block.path("id").asText(); + String name = block.path("name").asText(); + toolUseIds.put(index, id); + toolUseNames.put(index, name); + + writeSseEvent(out, "delta", + "{\"type\":\"tool_call_start\",\"id\":" + MAPPER.writeValueAsString(id) + + ",\"name\":" + MAPPER.writeValueAsString(name) + "}"); + } + } + + private void handleContentBlockDelta(JsonNode event, + Map blockTypes, + Map toolUseIds, + OutputStream out) throws Exception { + + int index = event.path("index").asInt(0); + JsonNode delta = event.get("delta"); + if (delta == null) { + return; + } + + String deltaType = delta.path("type").asText(); + String blockType = blockTypes.getOrDefault(index, "text"); + + if ("text_delta".equals(deltaType)) { + String text = delta.path("text").asText(); + if (!text.isEmpty()) { + writeSseEvent(out, "delta", + "{\"type\":\"content\",\"content\":" + MAPPER.writeValueAsString(text) + "}"); + } + } else if ("input_json_delta".equals(deltaType)) { + String partial = delta.path("partial_json").asText(); + String id = toolUseIds.getOrDefault(index, ""); + if (!partial.isEmpty()) { + writeSseEvent(out, "delta", + "{\"type\":\"tool_call_delta\",\"id\":" + MAPPER.writeValueAsString(id) + + ",\"arguments\":" + MAPPER.writeValueAsString(partial) + "}"); + } + } + } + + private void handleContentBlockStop(JsonNode event, + Map blockTypes, + Map toolUseIds, + OutputStream out) throws Exception { + + int index = event.path("index").asInt(0); + String blockType = blockTypes.getOrDefault(index, "text"); + + if ("tool_use".equals(blockType)) { + String id = toolUseIds.getOrDefault(index, ""); + writeSseEvent(out, "delta", + "{\"type\":\"tool_call_end\",\"id\":" + MAPPER.writeValueAsString(id) + "}"); + } + } + + private static void writeSseEvent(OutputStream out, String event, String data) throws Exception { + String sse = "event: " + event + "\ndata: " + data + "\n\n"; + out.write(sse.getBytes(StandardCharsets.UTF_8)); + out.flush(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ChatMessage.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ChatMessage.java new file mode 100644 index 00000000000..a1a1792057c --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ChatMessage.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest.ai; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * Normalized chat message used across all LLM providers. + * Roles: "system", "user", "assistant", "tool". + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ChatMessage { + + @JsonProperty + private String role; + + @JsonProperty + private String content; + + @JsonProperty + private List toolCalls; + + @JsonProperty + private String toolCallId; + + @JsonProperty + private String name; + + public ChatMessage() { + } + + @JsonCreator + public ChatMessage( + @JsonProperty("role") String role, + @JsonProperty("content") String content, + @JsonProperty("toolCalls") List toolCalls, + @JsonProperty("toolCallId") String toolCallId, + @JsonProperty("name") String name) { + this.role = role; + this.content = content; + this.toolCalls = toolCalls; + this.toolCallId = toolCallId; + this.name = name; + } + + public static ChatMessage system(String content) { + return new ChatMessage("system", content, null, null, null); + } + + public static ChatMessage user(String content) { + return new ChatMessage("user", content, null, null, null); + } + + public static ChatMessage assistant(String content) { + return new ChatMessage("assistant", content, null, null, null); + } + + public static ChatMessage tool(String toolCallId, String name, String content) { + return new ChatMessage("tool", content, null, toolCallId, name); + } + + public String getRole() { + return role; + } + + public void setRole(String role) { + this.role = role; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public List getToolCalls() { + return toolCalls; + } + + public void setToolCalls(List toolCalls) { + this.toolCalls = toolCalls; + } + + public String getToolCallId() { + return toolCallId; + } + + public void setToolCallId(String toolCallId) { + this.toolCallId = toolCallId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/LlmConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/LlmConfig.java new file mode 100644 index 00000000000..02ee51ede3b --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/LlmConfig.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest.ai; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Configuration POJO for the AI copilot LLM connection. + * Stored via PersistentStore. + */ +public class LlmConfig { + + @JsonProperty + private String provider; + + @JsonProperty + private String apiEndpoint; + + @JsonProperty + private String apiKey; + + @JsonProperty + private String model; + + @JsonProperty + private int maxTokens; + + @JsonProperty + private double temperature; + + @JsonProperty + private boolean enabled; + + @JsonProperty + private String systemPrompt; + + @JsonProperty + private boolean sendDataToAi; + + public LlmConfig() { + this.provider = "openai"; + this.maxTokens = 4096; + this.temperature = 0.7; + this.enabled = false; + this.sendDataToAi = true; + } + + @JsonCreator + public LlmConfig( + @JsonProperty("provider") String provider, + @JsonProperty("apiEndpoint") String apiEndpoint, + @JsonProperty("apiKey") String apiKey, + @JsonProperty("model") String model, + @JsonProperty("maxTokens") int maxTokens, + @JsonProperty("temperature") double temperature, + @JsonProperty("enabled") boolean enabled, + @JsonProperty("systemPrompt") String systemPrompt, + @JsonProperty("sendDataToAi") Boolean sendDataToAi) { + this.provider = provider; + this.apiEndpoint = apiEndpoint; + this.apiKey = apiKey; + this.model = model; + this.maxTokens = maxTokens; + this.temperature = temperature; + this.enabled = enabled; + this.systemPrompt = systemPrompt; + this.sendDataToAi = sendDataToAi != null ? sendDataToAi : true; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } + + public String getApiEndpoint() { + return apiEndpoint; + } + + public void setApiEndpoint(String apiEndpoint) { + this.apiEndpoint = apiEndpoint; + } + + public String getApiKey() { + return apiKey; + } + + public void setApiKey(String apiKey) { + this.apiKey = apiKey; + } + + public String getModel() { + return model; + } + + public void setModel(String model) { + this.model = model; + } + + public int getMaxTokens() { + return maxTokens; + } + + public void setMaxTokens(int maxTokens) { + this.maxTokens = maxTokens; + } + + public double getTemperature() { + return temperature; + } + + public void setTemperature(double temperature) { + this.temperature = temperature; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getSystemPrompt() { + return systemPrompt; + } + + public void setSystemPrompt(String systemPrompt) { + this.systemPrompt = systemPrompt; + } + + public boolean isSendDataToAi() { + return sendDataToAi; + } + + public void setSendDataToAi(boolean sendDataToAi) { + this.sendDataToAi = sendDataToAi; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/LlmProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/LlmProvider.java new file mode 100644 index 00000000000..b6bd6005368 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/LlmProvider.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest.ai; + +import java.io.OutputStream; +import java.util.List; + +/** + * Interface for LLM providers that can stream chat completions. + * Each provider normalizes its vendor-specific SSE format to + * a common wire format consumed by the frontend. + */ +public interface LlmProvider { + + /** + * @return unique identifier for this provider (e.g. "openai", "anthropic") + */ + String getId(); + + /** + * @return human-readable display name + */ + String getDisplayName(); + + /** + * Stream a chat completion to the given output stream using normalized SSE events. + * + * @param config the LLM configuration (endpoint, key, model, etc.) + * @param messages the conversation messages + * @param tools tool definitions (may be empty) + * @param out the output stream to write SSE events to + * @throws Exception if streaming fails + */ + void streamChatCompletion(LlmConfig config, List messages, + List tools, OutputStream out) throws Exception; + + /** + * Validate the given configuration for this provider. + * + * @param config the configuration to validate + * @return validation result + */ + ValidationResult validateConfig(LlmConfig config); +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/LlmProviderRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/LlmProviderRegistry.java new file mode 100644 index 00000000000..b6d5c5533a3 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/LlmProviderRegistry.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest.ai; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Static registry of available LLM providers. + */ +public final class LlmProviderRegistry { + + private static final Map PROVIDERS = new LinkedHashMap<>(); + + static { + register(new OpenAiCompatibleProvider()); + register(new AnthropicProvider()); + } + + private LlmProviderRegistry() { + } + + public static void register(LlmProvider provider) { + PROVIDERS.put(provider.getId(), provider); + } + + public static LlmProvider get(String id) { + return PROVIDERS.get(id); + } + + public static Collection getAll() { + return PROVIDERS.values(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/OpenAiCompatibleProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/OpenAiCompatibleProvider.java new file mode 100644 index 00000000000..585d39cd951 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/OpenAiCompatibleProvider.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest.ai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * LLM provider for OpenAI-compatible APIs (OpenAI, Azure OpenAI, Ollama, etc.). + * Uses OkHttp to call /chat/completions with stream:true, + * reads SSE line-by-line, and normalizes deltas to the common wire format. + */ +public class OpenAiCompatibleProvider implements LlmProvider { + + private static final Logger logger = LoggerFactory.getLogger(OpenAiCompatibleProvider.class); + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final MediaType JSON_TYPE = MediaType.parse("application/json"); + private static final String DEFAULT_ENDPOINT = "https://api.openai.com/v1"; + + private final OkHttpClient httpClient; + + public OpenAiCompatibleProvider() { + this.httpClient = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build(); + } + + @Override + public String getId() { + return "openai"; + } + + @Override + public String getDisplayName() { + return "OpenAI Compatible"; + } + + @Override + public void streamChatCompletion(LlmConfig config, List messages, + List tools, OutputStream out) throws Exception { + + String endpoint = config.getApiEndpoint(); + if (endpoint == null || endpoint.isEmpty()) { + endpoint = DEFAULT_ENDPOINT; + } + // Remove trailing slash + if (endpoint.endsWith("/")) { + endpoint = endpoint.substring(0, endpoint.length() - 1); + } + String url = endpoint + "/chat/completions"; + + ObjectNode requestBody = buildRequestBody(config, messages, tools); + + Request.Builder reqBuilder = new Request.Builder() + .url(url) + .post(RequestBody.create(requestBody.toString(), JSON_TYPE)) + .addHeader("Content-Type", "application/json"); + + if (config.getApiKey() != null && !config.getApiKey().isEmpty()) { + reqBuilder.addHeader("Authorization", "Bearer " + config.getApiKey()); + } + + Request request = reqBuilder.build(); + + try (Response response = httpClient.newCall(request).execute()) { + if (!response.isSuccessful()) { + String errorBody = ""; + ResponseBody body = response.body(); + if (body != null) { + errorBody = body.string(); + } + String errorMsg = "LLM API error " + response.code() + ": " + errorBody; + logger.error(errorMsg); + writeSseEvent(out, "error", "{\"message\":" + MAPPER.writeValueAsString(errorMsg) + "}"); + return; + } + + ResponseBody body = response.body(); + if (body == null) { + writeSseEvent(out, "error", "{\"message\":\"Empty response from LLM API\"}"); + return; + } + + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(body.byteStream(), StandardCharsets.UTF_8))) { + processOpenAiStream(reader, out); + } + } + } + + @Override + public ValidationResult validateConfig(LlmConfig config) { + if (config.getApiKey() == null || config.getApiKey().isEmpty()) { + // Ollama doesn't require an API key, so only warn + String endpoint = config.getApiEndpoint(); + if (endpoint != null && !endpoint.contains("localhost") && !endpoint.contains("127.0.0.1")) { + return ValidationResult.error("API key is required for non-local endpoints"); + } + } + if (config.getModel() == null || config.getModel().isEmpty()) { + return ValidationResult.error("Model name is required"); + } + return ValidationResult.ok("Configuration is valid"); + } + + private ObjectNode buildRequestBody(LlmConfig config, List messages, + List tools) { + ObjectNode body = MAPPER.createObjectNode(); + body.put("model", config.getModel()); + body.put("stream", true); + body.put("max_tokens", config.getMaxTokens()); + body.put("temperature", config.getTemperature()); + + // Messages + ArrayNode messagesArray = body.putArray("messages"); + for (ChatMessage msg : messages) { + ObjectNode msgNode = messagesArray.addObject(); + msgNode.put("role", msg.getRole()); + + if (msg.getContent() != null) { + msgNode.put("content", msg.getContent()); + } + + if (msg.getToolCallId() != null) { + msgNode.put("tool_call_id", msg.getToolCallId()); + } + + if (msg.getName() != null) { + msgNode.put("name", msg.getName()); + } + + if (msg.getToolCalls() != null && !msg.getToolCalls().isEmpty()) { + ArrayNode toolCallsNode = msgNode.putArray("tool_calls"); + for (ToolCall tc : msg.getToolCalls()) { + ObjectNode tcNode = toolCallsNode.addObject(); + tcNode.put("id", tc.getId()); + tcNode.put("type", "function"); + ObjectNode fnNode = tcNode.putObject("function"); + fnNode.put("name", tc.getName()); + fnNode.put("arguments", tc.getArguments()); + } + } + } + + // Tools + if (tools != null && !tools.isEmpty()) { + ArrayNode toolsArray = body.putArray("tools"); + for (ToolDefinition tool : tools) { + ObjectNode toolNode = toolsArray.addObject(); + toolNode.put("type", "function"); + ObjectNode fnNode = toolNode.putObject("function"); + fnNode.put("name", tool.getName()); + fnNode.put("description", tool.getDescription()); + fnNode.set("parameters", MAPPER.valueToTree(tool.getParameters())); + } + } + + return body; + } + + private void processOpenAiStream(BufferedReader reader, OutputStream out) throws Exception { + // Track tool calls being assembled + Map toolCallIds = new HashMap<>(); + Map toolCallNames = new HashMap<>(); + Map toolCallArgs = new HashMap<>(); + + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + + if (line.isEmpty()) { + continue; + } + + if (!line.startsWith("data: ")) { + continue; + } + + String data = line.substring(6).trim(); + + if ("[DONE]".equals(data)) { + // Check if we have pending tool calls + if (!toolCallIds.isEmpty()) { + // Emit tool_call_end events + for (Map.Entry entry : toolCallIds.entrySet()) { + int idx = entry.getKey(); + writeSseEvent(out, "delta", + "{\"type\":\"tool_call_end\",\"id\":" + MAPPER.writeValueAsString(entry.getValue()) + "}"); + } + writeSseEvent(out, "done", "{\"finish_reason\":\"tool_calls\"}"); + } else { + writeSseEvent(out, "done", "{\"finish_reason\":\"stop\"}"); + } + return; + } + + try { + JsonNode chunk = MAPPER.readTree(data); + JsonNode choices = chunk.get("choices"); + if (choices == null || !choices.isArray() || choices.isEmpty()) { + continue; + } + + JsonNode choice = choices.get(0); + JsonNode delta = choice.get("delta"); + if (delta == null) { + continue; + } + + // Check finish_reason + JsonNode finishReason = choice.get("finish_reason"); + if (finishReason != null && !finishReason.isNull()) { + String reason = finishReason.asText(); + if ("tool_calls".equals(reason)) { + // Emit end events for all tracked tool calls + for (Map.Entry entry : toolCallIds.entrySet()) { + writeSseEvent(out, "delta", + "{\"type\":\"tool_call_end\",\"id\":" + MAPPER.writeValueAsString(entry.getValue()) + "}"); + } + writeSseEvent(out, "done", "{\"finish_reason\":\"tool_calls\"}"); + return; + } + if ("stop".equals(reason)) { + writeSseEvent(out, "done", "{\"finish_reason\":\"stop\"}"); + return; + } + } + + // Content delta + JsonNode content = delta.get("content"); + if (content != null && !content.isNull()) { + String text = content.asText(); + if (!text.isEmpty()) { + writeSseEvent(out, "delta", + "{\"type\":\"content\",\"content\":" + MAPPER.writeValueAsString(text) + "}"); + } + } + + // Tool call deltas + JsonNode toolCallsNode = delta.get("tool_calls"); + if (toolCallsNode != null && toolCallsNode.isArray()) { + for (JsonNode tc : toolCallsNode) { + int idx = tc.has("index") ? tc.get("index").asInt() : 0; + + // New tool call start + if (tc.has("id")) { + String id = tc.get("id").asText(); + toolCallIds.put(idx, id); + String name = ""; + if (tc.has("function") && tc.get("function").has("name")) { + name = tc.get("function").get("name").asText(); + } + toolCallNames.put(idx, name); + toolCallArgs.put(idx, new StringBuilder()); + + writeSseEvent(out, "delta", + "{\"type\":\"tool_call_start\",\"id\":" + MAPPER.writeValueAsString(id) + + ",\"name\":" + MAPPER.writeValueAsString(name) + "}"); + } + + // Argument delta + if (tc.has("function") && tc.get("function").has("arguments")) { + String argDelta = tc.get("function").get("arguments").asText(); + if (argDelta != null && !argDelta.isEmpty()) { + StringBuilder args = toolCallArgs.get(idx); + if (args != null) { + args.append(argDelta); + } + String tcId = toolCallIds.getOrDefault(idx, ""); + writeSseEvent(out, "delta", + "{\"type\":\"tool_call_delta\",\"id\":" + MAPPER.writeValueAsString(tcId) + + ",\"arguments\":" + MAPPER.writeValueAsString(argDelta) + "}"); + } + } + } + } + } catch (Exception e) { + logger.warn("Error parsing SSE chunk: {}", data, e); + } + } + } + + private static void writeSseEvent(OutputStream out, String event, String data) throws Exception { + String sse = "event: " + event + "\ndata: " + data + "\n\n"; + out.write(sse.getBytes(StandardCharsets.UTF_8)); + out.flush(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ToolCall.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ToolCall.java new file mode 100644 index 00000000000..223d26e9cdf --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ToolCall.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest.ai; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Represents a tool call from the LLM. + */ +public class ToolCall { + + @JsonProperty + private String id; + + @JsonProperty + private String name; + + @JsonProperty + private String arguments; + + public ToolCall() { + } + + @JsonCreator + public ToolCall( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("arguments") String arguments) { + this.id = id; + this.name = name; + this.arguments = arguments; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getArguments() { + return arguments; + } + + public void setArguments(String arguments) { + this.arguments = arguments; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ToolDefinition.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ToolDefinition.java new file mode 100644 index 00000000000..0e6e3dc0a99 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ToolDefinition.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest.ai; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +/** + * Tool definition sent to the LLM so it knows which tools it can call. + * The parameters field holds a JSON Schema object describing the tool's arguments. + */ +public class ToolDefinition { + + @JsonProperty + private String name; + + @JsonProperty + private String description; + + @JsonProperty + private Map parameters; + + public ToolDefinition() { + } + + @JsonCreator + public ToolDefinition( + @JsonProperty("name") String name, + @JsonProperty("description") String description, + @JsonProperty("parameters") Map parameters) { + this.name = name; + this.description = description; + this.parameters = parameters; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Map getParameters() { + return parameters; + } + + public void setParameters(Map parameters) { + this.parameters = parameters; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ValidationResult.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ValidationResult.java new file mode 100644 index 00000000000..29a84570602 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ai/ValidationResult.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest.ai; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Result of validating an LLM configuration. + */ +public class ValidationResult { + + @JsonProperty + private boolean success; + + @JsonProperty + private String message; + + public ValidationResult() { + } + + public ValidationResult(boolean success, String message) { + this.success = success; + this.message = message; + } + + public static ValidationResult ok(String message) { + return new ValidationResult(true, message); + } + + public static ValidationResult error(String message) { + return new ValidationResult(false, message); + } + + public boolean isSuccess() { + return success; + } + + public String getMessage() { + return message; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceConfig.java index e358c9d7e3a..caeb220f463 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceConfig.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceConfig.java @@ -25,6 +25,7 @@ * - location which is a path. * - writable flag to indicate whether the location supports creating new tables. * - default storage format for new tables created in this workspace. + * - optional description of the workspace */ @JsonIgnoreProperties(value = {"storageformat"}, ignoreUnknown = true) @@ -36,17 +37,27 @@ public class WorkspaceConfig { private final String location; private final boolean writable; private final String defaultInputFormat; + private final String description; private final boolean allowAccessOutsideWorkspace; // do not allow access outside the workspace by default. // For backward compatibility, the user can turn this // on. + public WorkspaceConfig(String location, + boolean writable, + String defaultInputFormat, + boolean allowAccessOutsideWorkspace) { + this(location, writable, defaultInputFormat, null, allowAccessOutsideWorkspace); + } + public WorkspaceConfig(@JsonProperty("location") String location, @JsonProperty("writable") boolean writable, @JsonProperty("defaultInputFormat") String defaultInputFormat, + @JsonProperty("description") String description, @JsonProperty("allowAccessOutsideWorkspace") boolean allowAccessOutsideWorkspace ) { this.location = location; this.writable = writable; this.defaultInputFormat = defaultInputFormat; + this.description = description; this.allowAccessOutsideWorkspace = allowAccessOutsideWorkspace; } @@ -54,6 +65,10 @@ public String getLocation() { return location; } + public String getDescription() { + return description; + } + public boolean isWritable() { return writable; } @@ -76,6 +91,7 @@ public int hashCode() { result = prime * result + ((location == null) ? 0 : location.hashCode()); result = prime * result + (writable ? 1231 : 1237); result = prime * result + (allowAccessOutsideWorkspace ? 1231 : 1237); + result = prime * result + ((description == null) ? 0 : description.hashCode()); return result; } @@ -105,6 +121,13 @@ public boolean equals(Object obj) { } else if (!location.equals(other.location)) { return false; } + if (description == null) { + if (other.description != null) { + return false; + } + } else if (!description.equals(other.description)) { + return false; + } if (writable != other.writable) { return false; } @@ -113,5 +136,4 @@ public boolean equals(Object obj) { } return true; } - } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java index ae6fdbd5d95..8324d0de0bf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java @@ -42,8 +42,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; import static org.apache.drill.exec.ExecConstants.DRILL_SYS_FILE_SUFFIX; @@ -120,18 +118,23 @@ public Iterator> getRange(int skip, int take) { return Collections.emptyIterator(); } - return fileStatuses.stream() + LinkedHashMap result = new LinkedHashMap<>(); + fileStatuses.stream() .map(this::extractKeyName) .sorted() .skip(skip) .limit(take) - .collect(Collectors.toMap( - Function.identity(), - this::get, - (o, n) -> n, - LinkedHashMap::new)) - .entrySet() - .iterator(); + .forEach(key -> { + try { + V value = get(key); + if (value != null) { + result.put(key, value); + } + } catch (Exception e) { + logger.warn("Skipping corrupted store entry [{}]: {}", key, e.getMessage()); + } + }); + return result.entrySet().iterator(); } @Override diff --git a/exec/java-exec/src/main/resources/rest/generic.ftl b/exec/java-exec/src/main/resources/rest/generic.ftl index f4fd6aae015..957d544b393 100644 --- a/exec/java-exec/src/main/resources/rest/generic.ftl +++ b/exec/java-exec/src/main/resources/rest/generic.ftl @@ -65,6 +65,7 @@ + } + > + { e.stopPropagation(); onRemove(); }} + color="blue" + style={{ cursor: 'pointer' }} + > + {filter.label} + + + + ); +} + +/** Editable tag for a numeric filter with operator selection. */ +function NumericFilterTag({ + filter, + onRemove, + onUpdate, +}: { + filter: DashboardFilter; + onRemove: () => void; + onUpdate: (update: Partial) => void; +}) { + const [open, setOpen] = useState(false); + const [op, setOp] = useState(filter.numericOp || '='); + const [val, setVal] = useState(filter.value); + const [valEnd, setValEnd] = useState(filter.numericEnd || filter.value); + + const handleOpen = useCallback((visible: boolean) => { + if (visible) { + setOp(filter.numericOp || '='); + setVal(filter.value); + setValEnd(filter.numericEnd || filter.value); + } + setOpen(visible); + }, [filter]); + + const handleApply = useCallback(() => { + if (val === '') { + return; + } + const update: Partial = { numericOp: op, value: val }; + if (op === 'between') { + update.numericEnd = valEnd; + } else { + update.numericEnd = undefined; + } + onUpdate(update); + setOpen(false); + }, [op, val, valEnd, onUpdate]); + + return ( + +
+ {filter.column} +
+ + updateField('fontFamily', value)} + options={FONT_OPTIONS} + style={{ width: '100%', marginBottom: 16 }} + /> + + {/* Border Radius */} + Border Radius + onConfigChange({ ...config, imageAlt: e.target.value })} + placeholder="Alt text (optional)" + addonBefore="Alt" + /> + {content && !imgError && sanitizeImageUrl(content) && ( +
+ {altText} setImgError(true)} + /> +
+ )} + + + ); + } + + if (!content || imgError) { + return ( +
+ +
+ ); + } + + const safeUrl = sanitizeImageUrl(content); + if (!safeUrl) { + return ( +
+ +
+ ); + } + + return ( +
+ {altText} setImgError(true)} + /> +
+ ); +} diff --git a/exec/java-exec/src/main/resources/webapp/src/components/dashboard/MarkdownPanel.tsx b/exec/java-exec/src/main/resources/webapp/src/components/dashboard/MarkdownPanel.tsx new file mode 100644 index 00000000000..10b82e5248c --- /dev/null +++ b/exec/java-exec/src/main/resources/webapp/src/components/dashboard/MarkdownPanel.tsx @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Input } from 'antd'; +import Markdown from 'react-markdown'; + +const { TextArea } = Input; + +interface MarkdownPanelProps { + content: string; + editMode: boolean; + onContentChange: (content: string) => void; +} + +export default function MarkdownPanel({ content, editMode, onContentChange }: MarkdownPanelProps) { + if (editMode) { + return ( +
+