HADOOP-6864. Provide a JNI-based implementation of ShellBasedUnixGroupsNetgroupMapping (implementation of GroupMappingServiceProvider)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1052420 13f79535-47bb-0310-9956-ffa450edef68

Author: sborya

CHANGES.txt

@@ -242,6 +242,9 @@ Release 0.22.0 - Unreleased
     classpath. (Patrick Angeles via eli)
 
     HADOOP-6298. Add copyBytes to Text and BytesWritable. (omalley)
+  
+    HADOOP-6864. Provide a JNI-based implementation of ShellBasedUnixGroupsNetgroupMapping 
+    (implementation of GroupMappingServiceProvider) (Erik Seffl via boryas)
 
   OPTIMIZATIONS
 

build.xml

@@ -396,6 +396,15 @@
   	  <class name="org.apache.hadoop.io.nativeio.NativeIO" />
   	</javah>
 
+  	<javah
+  	  classpath="${build.classes}"
+  	  destdir="${build.native}/src/org/apache/hadoop/security"
+      force="yes"
+  	  verbose="yes"
+  	  >
+  	  <class name="org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping" />
+  	</javah>
+
 	<exec dir="${build.native}" executable="sh" failonerror="true">
 	  <env key="OS_NAME" value="${os.name}"/>
 	  <env key="OS_ARCH" value="${os.arch}"/>
@@ -717,6 +726,10 @@
         </syspropertyset>
         <sysproperty key="test.system.hdrc.deployed.hadoopconfdir"
                      value="@{hadoop.conf.dir.deployed}" />
+        <!-- user to group mapping class for TestAccessControlList -->
+        <syspropertyset dynamic="no">
+          <propertyref name="TestAccessControlListGroupMapping"/>
+        </syspropertyset>
         <formatter type="${test.junit.output.format}" />
         <batchtest todir="@{test.dir}" if="tests.notestcase">
           <fileset dir="@{fileset.dir}/core"

src/java/org/apache/hadoop/security/GroupMappingServiceProvider.java

@@ -39,4 +39,15 @@ public interface GroupMappingServiceProvider {
    * @throws IOException
    */
   public List<String> getGroups(String user) throws IOException;
+  /**
+   * Refresh the cache of groups and user mapping
+   * @throws IOException
+   */
+  public void cacheGroupsRefresh() throws IOException;
+  /**
+   * Caches the group user information
+   * @param groups list of groups to add to cache
+   * @throws IOException
+   */
+  public void cacheGroupsAdd(List<String> groups) throws IOException;
 }

src/java/org/apache/hadoop/security/Groups.java

@@ -98,22 +98,56 @@ public List<String> getGroups(String user) throws IOException {
    */
   public void refresh() {
     LOG.info("clearing userToGroupsMap cache");
+    try {
+      impl.cacheGroupsRefresh();
+    } catch (IOException e) {
+      LOG.warn("Error refreshing groups cache", e);
+    }
     userToGroupsMap.clear();
   }
-  
+
+  /**
+   * Add groups to cache
+   *
+   * @param groups list of groups to add to cache
+   */
+  public void cacheGroupsAdd(List<String> groups) {
+    try {
+      impl.cacheGroupsAdd(groups);
+    } catch (IOException e) {
+      LOG.warn("Error caching groups", e);
+    }
+  }
+
+  /**
+   * Class to hold the cached groups
+   */
   private static class CachedGroups {
     final long timestamp;
     final List<String> groups;
 
+    /**
+     * Create and initialize group cache
+     */
     CachedGroups(List<String> groups) {
       this.groups = groups;
       this.timestamp = System.currentTimeMillis();
     }
 
+    /**
+     * Returns time of last cache update
+     *
+     * @return time of last cache update
+     */
     public long getTimestamp() {
       return timestamp;
     }
 
+    /**
+     * Get list of cached groups
+     *
+     * @return cached groups
+     */
     public List<String> getGroups() {
       return groups;
     }
@@ -128,13 +162,15 @@ public List<String> getGroups() {
   public static Groups getUserToGroupsMappingService() {
     return getUserToGroupsMappingService(new Configuration()); 
   }
-  
+
   /**
    * Get the groups being used to map user-to-groups.
    * @param conf
    * @return the groups being used to map user-to-groups.
    */
-  public static Groups getUserToGroupsMappingService(Configuration conf) {
+  public static synchronized Groups getUserToGroupsMappingService(
+    Configuration conf) {
+
     if(GROUPS == null) {
       if(LOG.isDebugEnabled()) {
         LOG.debug(" Creating new Groups object");

src/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java

@@ -22,6 +22,9 @@
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.util.NativeCodeLoader;
@@ -31,17 +34,19 @@
  * that invokes libC calls to get the group
  * memberships of a given user.
  */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
 public class JniBasedUnixGroupsMapping implements GroupMappingServiceProvider {
 
   private static final Log LOG = 
-    LogFactory.getLog(ShellBasedUnixGroupsMapping.class);
+    LogFactory.getLog(JniBasedUnixGroupsMapping.class);
 
   native String[] getGroupForUser(String user);
 
   static {
     if (!NativeCodeLoader.isNativeCodeLoaded()) {
       throw new RuntimeException("Bailing out since native library couldn't " +
-      		                       "be loaded");
+        "be loaded");
     }
     LOG.info("Using JniBasedUnixGroupsMapping for Group resolution");
   }
@@ -52,9 +57,18 @@ public List<String> getGroups(String user) throws IOException {
     try {
       groups = getGroupForUser(user);
     } catch (Exception e) {
-      LOG.warn("Got exception while trying to obtain the groups for user " 
-               + user);
+      LOG.warn("Error getting groups for " + user, e);
     }
     return Arrays.asList(groups);
   }
+
+  @Override
+  public void cacheGroupsRefresh() throws IOException {
+    // does nothing in this provider of user to groups mapping
+  }
+
+  @Override
+  public void cacheGroupsAdd(List<String> groups) throws IOException {
+    // does nothing in this provider of user to groups mapping
+  }
 }

src/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java

@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.LinkedList;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.NativeCodeLoader;
+
+import org.apache.hadoop.security.NetgroupCache;
+
+/**
+ * A JNI-based implementation of {@link GroupMappingServiceProvider} 
+ * that invokes libC calls to get the group
+ * memberships of a given user.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class JniBasedUnixGroupsNetgroupMapping
+  extends JniBasedUnixGroupsMapping {
+  
+  private static final Log LOG = LogFactory.getLog(
+    JniBasedUnixGroupsNetgroupMapping.class);
+
+  native String[] getUsersForNetgroupJNI(String group);
+
+  static {
+    if (!NativeCodeLoader.isNativeCodeLoaded()) {
+      throw new RuntimeException("Bailing out since native library couldn't " +
+        "be loaded");
+    }
+    LOG.info("Using JniBasedUnixGroupsNetgroupMapping for Netgroup resolution");
+  }
+
+  /**
+   * Gets unix groups and netgroups for the user.
+   *
+   * It gets all unix groups as returned by id -Gn but it
+   * only returns netgroups that are used in ACLs (there is
+   * no way to get all netgroups for a given user, see
+   * documentation for getent netgroup)
+   */
+  @Override
+  public List<String> getGroups(String user) throws IOException {
+    // parent gets unix groups
+    List<String> groups = new LinkedList<String>(super.getGroups(user));
+    NetgroupCache.getNetgroups(user, groups);
+    return groups;
+  }
+
+  /**
+   * Refresh the netgroup cache
+   */
+  @Override
+  public void cacheGroupsRefresh() throws IOException {
+    List<String> groups = NetgroupCache.getNetgroupNames();
+    NetgroupCache.clear();
+    cacheGroupsAdd(groups);
+  }
+
+  /**
+   * Add a group to cache, only netgroups are cached
+   *
+   * @param groups list of group names to add to cache
+   */
+  @Override
+  public void cacheGroupsAdd(List<String> groups) throws IOException {
+    for(String group: groups) {
+      if(group.length() == 0) {
+        // better safe than sorry (should never happen)
+      } else if(group.charAt(0) == '@') {
+        if(!NetgroupCache.isCached(group)) {
+          NetgroupCache.add(group, getUsersForNetgroup(group));
+        }
+      } else {
+        // unix group, not caching
+      }
+    }
+  }
+
+  /**
+   * Calls JNI function to get users for a netgroup, since C functions
+   * are not reentrant we need to make this synchronized (see
+   * documentation for setnetgrent, getnetgrent and endnetgrent)
+   *
+   * @param netgroup return users for this netgroup
+   * @return list of users for a given netgroup
+   */
+  protected synchronized List<String> getUsersForNetgroup(String netgroup) {
+    String[] users = null;
+    try {
+      // JNI code does not expect '@' at the begining of the group name
+      users = getUsersForNetgroupJNI(netgroup.substring(1));
+    } catch (Exception e) {
+      LOG.warn("error getting users for netgroup " + netgroup, e);
+    }
+    if (users != null && users.length != 0) {
+      return Arrays.asList(users);
+    }
+    return new LinkedList<String>();
+  }
+}