Skip to content

DRILL-8548: Integrate Apache Ranger authorization for Drill #3056

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shfshihuafeng wants to merge 1 commit into
base: master
Choose a base branch
from
Open

DRILL-8548: Integrate Apache Ranger authorization for Drill #3056

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 185 additions & 0 deletions distribution/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,53 @@
<artifactId>drill-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.drill</groupId>
<artifactId>drill-ranger-plugin</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.hk2</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.hk2.external</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.ws.rs</groupId>
<artifactId>jakarta.ws.rs-api</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.inject</groupId>
<artifactId>jakarta.inject-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<!--
Ranger Admin service plugin (REST-based, JDK 8 bytecode, no Drill deps).
Deployed into Ranger Admin's WEB-INF/lib. No shade/fat jar needed:
the plugin uses only HttpURLConnection (JDK-standard) and Jackson
(provided by Ranger Admin).
-->
<dependency>
<groupId>org.apache.drill</groupId>
<artifactId>ranger-drill-service</artifactId>
<version>${project.version}</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.apache.drill</groupId>
<artifactId>drill-yarn</artifactId>
Expand Down Expand Up @@ -279,6 +326,144 @@

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-ranger-plugin-isolated-deps</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<!--
Force-copy the 2.x versions of Jersey, HK2, JAX-RS API,
and annotation APIs that Maven dependency mediation would
otherwise resolve to 3.x (because Drill's own REST server
depends on the 3.x versions at a shallower depth).

These jars go into the isolated ranger-drill-plugin-impl/
directory and are loaded by RangerPluginClassLoader, NOT
the Drillbit classloader. Using maven-dependency-plugin:copy
(not the assembly dependencySet) ensures we get the exact
versions required by Jersey 2.35 / HK2 2.6.1, regardless of
what Maven mediation picks for the main classpath.
-->
<artifactItems>
<!-- JAX-RS 2.x API (javax.ws.rs.* package) -->
<artifactItem>
<groupId>jakarta.ws.rs</groupId>
<artifactId>jakarta.ws.rs-api</artifactId>
<version>${jaxrs.api.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<!-- Jersey 2.x client runtime -->
<artifactItem>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>${jersey.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-common</artifactId>
<version>${jersey.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>jersey-hk2</artifactId>
<version>${jersey.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-server</artifactId>
<version>${jersey.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>${jersey.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-multipart</artifactId>
<version>${jersey.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.jersey.ext</groupId>
<artifactId>jersey-entity-filtering</artifactId>
<version>${jersey.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<!-- HK2 2.6.1 (DI container used by Jersey 2.x) -->
<artifactItem>
<groupId>org.glassfish.hk2</groupId>
<artifactId>hk2-locator</artifactId>
<version>${hk2.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.hk2</groupId>
<artifactId>hk2-api</artifactId>
<version>${hk2.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.hk2</groupId>
<artifactId>hk2-utils</artifactId>
<version>${hk2.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.hk2.external</groupId>
<artifactId>aopalliance-repackaged</artifactId>
<version>${hk2.ranger.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.glassfish.hk2</groupId>
<artifactId>osgi-resource-locator</artifactId>
<version>${osgi.resource.locator.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
<version>${jakarta.annotation.api.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>jakarta.inject</groupId>
<artifactId>jakarta.inject-api</artifactId>
<version>${jakarta.inject.api.version}</version>
<type>jar</type>
<outputDirectory>${project.build.directory}/ranger-drill-plugin-impl</outputDirectory>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
Expand Down
103 changes: 101 additions & 2 deletions distribution/src/assemble/component.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,67 @@
<useProjectArtifact>false</useProjectArtifact>
</dependencySet>

<!--
Ranger Admin service plugin. Kept in its own directory so it never lands on the
Drillbit classpath. Users copy this jar into Ranger Admin's
WEB-INF/classes/lib to enable Test Connection and resource autocomplete
in the Ranger policy editor.
-->
<dependencySet>
<includes>
<include>org.apache.drill:ranger-drill-service:jar</include>
</includes>
<outputDirectory>jars/ranger-service</outputDirectory>
<useProjectArtifact>false</useProjectArtifact>
</dependencySet>

<!--
Ranger framework jars (NOT Jersey/HK2 — those are force-copied by
maven-dependency-plugin:copy in distribution/pom.xml to ensure
correct 2.x versions are used).

These jars are NOT on the Drillbit's main classpath; they are loaded
by RangerPluginClassLoader from this directory. This isolation is
required because the plugin ships Jersey 2.35 (org.glassfish.jersey.*
+ javax.ws.rs.*) which conflicts with Drill's own Jersey 3.1.9
(org.glassfish.jersey.* + jakarta.ws.rs.*) on the main classpath.

Shared deps (SLF4J, Netty, Hadoop, Jackson) are 'provided' scope in
drill-ranger-plugin/pom.xml and intentionally NOT listed here — the
plugin classloader falls back to the Drillbit classloader for those.
-->
<dependencySet>
<includes>
<!-- Plugin itself -->
<include>org.apache.drill:drill-ranger-plugin:jar</include>
<!-- Ranger plugin framework -->
<include>org.apache.ranger:ranger-plugins-common:jar</include>
<include>org.apache.ranger:ranger-knox-plugin:jar</include>
<include>org.apache.ranger:ranger-audit-core:jar</include>
<include>org.apache.ranger:ranger-authz-api:jar</include>
<include>org.apache.ranger:ugsync-util:jar</include>
</includes>
<outputDirectory>jars/ranger-drill-plugin-impl</outputDirectory>
<unpack>false</unpack>
<useProjectArtifact>false</useProjectArtifact>
</dependencySet>

<!--
RangerPluginClassLoader must be on the Drillbit's main classpath because
RangerAccessAuthorizer (in drill-java-exec.jar) directly references it to
create the isolated classloader for the Ranger plugin. It cannot itself
be loaded by the isolated classloader.
-->
<dependencySet>
<includes>
<include>org.apache.ranger:ranger-plugin-classloader:jar</include>
</includes>
<outputDirectory>jars/3rdparty</outputDirectory>
<unpack>false</unpack>
<useProjectArtifact>false</useProjectArtifact>
<scope>compile</scope>
</dependencySet>

<dependencySet>
<outputDirectory>jars/classb</outputDirectory>
<unpack>false</unpack>
Expand Down Expand Up @@ -125,6 +186,22 @@
<include>org.jvnet.mimepull</include>
<include>org.reflections</include>
</includes>
<!--
Exclude Jersey 2.x jars (used by drill-ranger-plugin via
RangerPluginClassLoader). These go to jars/ranger-drill-plugin-impl/
instead, to avoid org.glassfish.jersey.* package conflicts with
Drill's own Jersey 3.1.9 on the main classpath.
-->
<excludes>
<exclude>org.glassfish.jersey.core:jersey-client:jar:${jersey.ranger.version}</exclude>
<exclude>org.glassfish.jersey.core:jersey-common:jar:${jersey.ranger.version}</exclude>
<exclude>org.glassfish.jersey.core:jersey-server:jar:${jersey.ranger.version}</exclude>
<exclude>org.glassfish.jersey.inject:jersey-hk2:jar:${jersey.ranger.version}</exclude>
<exclude>org.glassfish.jersey.media:jersey-media-json-jackson:jar:${jersey.ranger.version}</exclude>
<exclude>org.glassfish.jersey.media:jersey-media-multipart:jar:${jersey.ranger.version}</exclude>
<exclude>org.glassfish.jersey.ext:jersey-entity-filtering:jar:${jersey.ranger.version}</exclude>
<exclude>jakarta.ws.rs:jakarta.ws.rs-api:jar:${jaxrs.api.version}</exclude>
</excludes>
</dependencySet>
<dependencySet>
<outputDirectory>jars/3rdparty/</outputDirectory>
Expand Down Expand Up @@ -152,10 +229,16 @@
<exclude>org.apache.drill.exec</exclude>
<exclude>org.apache.drill.memory</exclude>
<exclude>org.apache.drill.metastore</exclude>
<!--
Ranger plugin jars and their transitive deps go to
jars/ranger-drill-plugin-impl/ (loaded by RangerPluginClassLoader),
NOT jars/3rdparty/ (main classpath).
-->
<exclude>org.apache.ranger</exclude>
<exclude>org.glassfish.hk2.external</exclude>
<exclude>org.glassfish.hk2</exclude>
<exclude>org.apache.zookeeper</exclude>
<exclude>org.eclipse.jetty</exclude>
<exclude>org.glassfish.hk2</exclude>
<exclude>org.glassfish.hk2.external</exclude>
<exclude>org.glassfish.jersey.containers</exclude>
<exclude>org.glassfish.jersey.core</exclude>
<exclude>org.glassfish.jersey.ext</exclude>
Expand Down Expand Up @@ -216,10 +299,26 @@
<directory>../sample-data</directory>
<outputDirectory>sample-data</outputDirectory>
</fileSet>
<!--
JAX-RS 2.x API (jakarta.ws.rs-api:2.1.6) and Jersey HK2 2.35 that were
force-copied by maven-dependency-plugin:copy in distribution/pom.xml.
These must use the 2.x versions, not the 3.x versions that Maven
dependency mediation would otherwise pick (because Drill's own REST
server depends on the 3.x versions at a shallower depth).
-->
<fileSet>
<directory>${project.build.directory}/ranger-drill-plugin-impl</directory>
<outputDirectory>jars/ranger-drill-plugin-impl</outputDirectory>
</fileSet>
<fileSet>
<directory>${project.build.directory}/winutils</directory>
<outputDirectory>winutils/bin</outputDirectory>
</fileSet>
<fileSet>
<directory>src/main/resources/ranger</directory>
<outputDirectory>conf/ranger</outputDirectory>
<fileMode>0640</fileMode>
</fileSet>
</fileSets>

<files>
Expand Down
5 changes: 5 additions & 0 deletions distribution/src/main/resources/drill-config.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,11 @@ export DRILLBIT_LOG_PATH="${DRILL_LOG_PREFIX}.log"
# Add Drill conf folder at the beginning of the classpath
CP="$DRILL_CONF_DIR"

# Add Ranger config directory if it exists (for ranger-drill-security.xml etc.)
if [ -d "$DRILL_CONF_DIR/ranger" ]; then
CP="$CP:$DRILL_CONF_DIR/ranger"
fi

# If both user and YARN-provided Java lib paths exist,
# combine them.

Expand Down
49 changes: 49 additions & 0 deletions distribution/src/main/resources/ranger/ranger-drill-audit.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<!-- Audit log destination. Set to Solr or HDFS for persistent audit storage.
If not configured, RangerDefaultAuditHandler logs to the Drillbit log. -->
<property>
<name>xasecure.audit.is.audit.to.solr</name>
<value>false</value>
</property>
<property>
<name>xasecure.audit.solr.url</name>
<value>http://ranger-admin-host:6083/solr/ranger_audits</value>
</property>

<property>
<name>xasecure.audit.is.audit.to.hdfs</name>
<value>false</value>
</property>
<property>
<name>xasecure.audit.hdfs.config.directory</name>
<value>hdfs://namenode:8020/ranger/audit</value>
</property>
<property>
<name>xasecure.audit.hdfs.config.file</name>
<value>/etc/hadoop/conf/core-site.xml</value>
</property>

<!-- Audit to log4j as fallback (always enabled) -->
<property>
<name>xasecure.audit.is.audit.to.log4j</name>
<value>true</value>
</property>
</configuration>
Loading
Loading