HDFS-5556. Add some more NameNode cache statistics, cache pool stats (cmccabe)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1546143 13f79535-47bb-0310-9956-ffa450edef68

Author: cmccabe

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -218,6 +218,9 @@ Trunk (Unreleased)
     HDFS-5286. Flatten INodeDirectory hierarchy: Replace INodeDirectoryWithQuota
     with DirectoryWithQuotaFeature.  (szetszwo)
 
+    HDFS-5556. Add some more NameNode cache statistics, cache pool stats
+    (cmccabe)
+
   OPTIMIZATIONS
     HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe)
 

hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml

@@ -352,6 +352,11 @@
       <Method name="getReplication" />
       <Bug pattern="ICAST_QUESTIONABLE_UNSIGNED_RIGHT_SHIFT" />
     </Match>
+    <Match>
+      <Class name="org.apache.hadoop.hdfs.protocol.CacheDirective" />
+      <Method name="insertInternal" />
+      <Bug pattern="BC_UNCONFIRMED_CAST" />
+    </Match>
     <!-- These two are used for shutting down and kicking the CRMon, do not need strong sync -->
     <Match>
       <Class name="org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor" />

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java

@@ -109,6 +109,7 @@
 import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
+import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
@@ -2358,7 +2359,7 @@ public void removeCachePool(String poolName) throws IOException {
     }
   }
 
-  public RemoteIterator<CachePoolInfo> listCachePools() throws IOException {
+  public RemoteIterator<CachePoolEntry> listCachePools() throws IOException {
     checkOpen();
     try {
       return namenode.listCachePools("");

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -58,6 +58,7 @@
 import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
+import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
@@ -1713,12 +1714,12 @@ public void removeCachePool(String poolName) throws IOException {
   /**
    * List all cache pools.
    *
-   * @return A remote iterator from which you can get CachePoolInfo objects.
+   * @return A remote iterator from which you can get CachePoolEntry objects.
    *          Requests will be made as needed.
    * @throws IOException
    *          If there was an error listing cache pools.
    */
-  public RemoteIterator<CachePoolInfo> listCachePools() throws IOException {
+  public RemoteIterator<CachePoolEntry> listCachePools() throws IOException {
     return dfs.listCachePools();
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java

@@ -29,6 +29,7 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
+import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.tools.DFSAdmin;
@@ -213,12 +214,12 @@ public void removeCachePool(String poolName) throws IOException {
   /**
    * List all cache pools.
    *
-   * @return A remote iterator from which you can get CachePoolInfo objects.
+   * @return A remote iterator from which you can get CachePoolEntry objects.
    *          Requests will be made as needed.
    * @throws IOException
    *          If there was an error listing cache pools.
    */
-  public RemoteIterator<CachePoolInfo> listCachePools() throws IOException {
+  public RemoteIterator<CachePoolEntry> listCachePools() throws IOException {
     return dfs.listCachePools();
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java

@@ -21,6 +21,8 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.server.namenode.CachePool;
+import org.apache.hadoop.util.IntrusiveCollection;
+import org.apache.hadoop.util.IntrusiveCollection.Element;
 
 import com.google.common.base.Preconditions;
 
@@ -30,32 +32,32 @@
  * This is an implementation class, not part of the public API.
  */
 @InterfaceAudience.Private
-public final class CacheDirective {
-  private final long entryId;
+public final class CacheDirective implements IntrusiveCollection.Element {
+  private final long id;
   private final String path;
   private final short replication;
-  private final CachePool pool;
+  private CachePool pool;
   private long bytesNeeded;
   private long bytesCached;
   private long filesAffected;
+  private Element prev;
+  private Element next;
 
-  public CacheDirective(long entryId, String path,
-      short replication, CachePool pool) {
-    Preconditions.checkArgument(entryId > 0);
-    this.entryId = entryId;
+  public CacheDirective(long id, String path,
+      short replication) {
+    Preconditions.checkArgument(id > 0);
+    this.id = id;
     Preconditions.checkArgument(replication > 0);
     this.path = path;
-    Preconditions.checkNotNull(pool);
     this.replication = replication;
     Preconditions.checkNotNull(path);
-    this.pool = pool;
     this.bytesNeeded = 0;
     this.bytesCached = 0;
     this.filesAffected = 0;
   }
 
-  public long getEntryId() {
-    return entryId;
+  public long getId() {
+    return id;
   }
 
   public String getPath() {
@@ -70,9 +72,9 @@ public short getReplication() {
     return replication;
   }
 
-  public CacheDirectiveInfo toDirective() {
+  public CacheDirectiveInfo toInfo() {
     return new CacheDirectiveInfo.Builder().
-        setId(entryId).
+        setId(id).
         setPath(new Path(path)).
         setReplication(replication).
         setPool(pool.getPoolName()).
@@ -88,13 +90,13 @@ public CacheDirectiveStats toStats() {
   }
 
   public CacheDirectiveEntry toEntry() {
-    return new CacheDirectiveEntry(toDirective(), toStats());
+    return new CacheDirectiveEntry(toInfo(), toStats());
   }
 
   @Override
   public String toString() {
     StringBuilder builder = new StringBuilder();
-    builder.append("{ entryId:").append(entryId).
+    builder.append("{ id:").append(id).
       append(", path:").append(path).
       append(", replication:").append(replication).
       append(", pool:").append(pool).
@@ -113,12 +115,12 @@ public boolean equals(Object o) {
       return false;
     }
     CacheDirective other = (CacheDirective)o;
-    return entryId == other.entryId;
+    return id == other.id;
   }
 
   @Override
   public int hashCode() {
-    return new HashCodeBuilder().append(entryId).toHashCode();
+    return new HashCodeBuilder().append(id).toHashCode();
   }
 
   public long getBytesNeeded() {
@@ -156,4 +158,55 @@ public void clearFilesAffected() {
   public void incrementFilesAffected() {
     this.filesAffected++;
   }
+
+  @SuppressWarnings("unchecked")
+  @Override // IntrusiveCollection.Element
+  public void insertInternal(IntrusiveCollection<? extends Element> list,
+      Element prev, Element next) {
+    assert this.pool == null;
+    this.pool = ((CachePool.DirectiveList)list).getCachePool();
+    this.prev = prev;
+    this.next = next;
+  }
+
+  @Override // IntrusiveCollection.Element
+  public void setPrev(IntrusiveCollection<? extends Element> list, Element prev) {
+    assert list == pool.getDirectiveList();
+    this.prev = prev;
+  }
+
+  @Override // IntrusiveCollection.Element
+  public void setNext(IntrusiveCollection<? extends Element> list, Element next) {
+    assert list == pool.getDirectiveList();
+    this.next = next;
+  }
+
+  @Override // IntrusiveCollection.Element
+  public void removeInternal(IntrusiveCollection<? extends Element> list) {
+    assert list == pool.getDirectiveList();
+    this.pool = null;
+    this.prev = null;
+    this.next = null;
+  }
+
+  @Override // IntrusiveCollection.Element
+  public Element getPrev(IntrusiveCollection<? extends Element> list) {
+    if (list != pool.getDirectiveList()) {
+      return null;
+    }
+    return this.prev;
+  }
+
+  @Override // IntrusiveCollection.Element
+  public Element getNext(IntrusiveCollection<? extends Element> list) {
+    if (list != pool.getDirectiveList()) {
+      return null;
+    }
+    return this.next;
+  }
+
+  @Override // IntrusiveCollection.Element
+  public boolean isInList(IntrusiveCollection<? extends Element> list) {
+    return pool == null ? false : list == pool.getDirectiveList();
+  }
 };

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java

@@ -94,21 +94,21 @@ private CacheDirectiveStats(long bytesNeeded, long bytesCached,
   /**
    * @return The bytes needed.
    */
-  public Long getBytesNeeded() {
+  public long getBytesNeeded() {
     return bytesNeeded;
   }
 
   /**
    * @return The bytes cached.
    */
-  public Long getBytesCached() {
+  public long getBytesCached() {
     return bytesCached;
   }
 
   /**
    * @return The files affected.
    */
-  public Long getFilesAffected() {
+  public long getFilesAffected() {
     return filesAffected;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolEntry.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Describes a Cache Pool entry.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class CachePoolEntry {
+  private final CachePoolInfo info;
+  private final CachePoolStats stats;
+
+  public CachePoolEntry(CachePoolInfo info, CachePoolStats stats) {
+    this.info = info;
+    this.stats = stats;
+  }
+
+  public CachePoolInfo getInfo() {
+    return info;
+  }
+
+  public CachePoolStats getStats() {
+    return stats;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java

@@ -30,6 +30,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.InvalidRequestException;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
@@ -150,7 +151,10 @@ public int hashCode() {
 
   public static void validate(CachePoolInfo info) throws IOException {
     if (info == null) {
-      throw new IOException("CachePoolInfo is null");
+      throw new InvalidRequestException("CachePoolInfo is null");
+    }
+    if ((info.getWeight() != null) && (info.getWeight() < 0)) {
+      throw new InvalidRequestException("CachePool weight is negative.");
     }
     validateName(info.poolName);
   }