HDFS-8178. QJM doesn't move aside stale inprogress edits files. Contributed by Istvan Fajth.

Signed-off-by: Wei-Chiu Chuang <[email protected]>

Author: fapifta

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java

@@ -52,12 +52,7 @@ class JNStorage extends Storage {
   private final StorageDirectory sd;
   private StorageState state;
 
-  private static final List<Pattern> CURRENT_DIR_PURGE_REGEXES =
-      ImmutableList.of(
-        Pattern.compile("edits_\\d+-(\\d+)"),
-        Pattern.compile("edits_inprogress_(\\d+)(?:\\..*)?"));
-  
-  private static final List<Pattern> PAXOS_DIR_PURGE_REGEXES = 
+  private static final List<Pattern> PAXOS_DIR_PURGE_REGEXES =
       ImmutableList.of(Pattern.compile("(\\d+)"));
 
   private static final String STORAGE_EDITS_SYNC = "edits.sync";
@@ -181,8 +176,8 @@ File getRoot() {
    * the given txid.
    */
   void purgeDataOlderThan(long minTxIdToKeep) throws IOException {
-    purgeMatching(sd.getCurrentDir(),
-        CURRENT_DIR_PURGE_REGEXES, minTxIdToKeep);
+    fjm.purgeLogsOlderThan(minTxIdToKeep);
+
     purgeMatching(getOrCreatePaxosDir(),
         PAXOS_DIR_PURGE_REGEXES, minTxIdToKeep);
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java

@@ -75,7 +75,8 @@ public class FileJournalManager implements JournalManager {
   private static final Pattern EDITS_INPROGRESS_STALE_REGEX = Pattern.compile(
       NameNodeFile.EDITS_INPROGRESS.getName() + "_(\\d+).*(\\S+)");
 
-  private File currentInProgress = null;
+  @VisibleForTesting
+  File currentInProgress = null;
 
   /**
    * A FileJournalManager should maintain the largest Tx ID that has been
@@ -178,20 +179,50 @@ public void setLastReadableTxId(long id) {
     this.lastReadableTxId = id;
   }
 
+  /**
+   * Purges the unnecessary edits and edits_inprogress files.
+   *
+   * Edits files that are ending before the minTxIdToKeep are purged.
+   * Edits in progress files that are starting before minTxIdToKeep are purged.
+   * Edits in progress files that are marked as empty, trash, corrupted or
+   * stale by file extension and starting before minTxIdToKeep are purged.
+   * Edits in progress files that are after minTxIdToKeep, but before the
+   * current edits in progress files are marked as stale for clarity.
+   *
+   * In case file removal or rename is failing a warning is logged, but that
+   * does not fail the operation.
+   *
+   * @param minTxIdToKeep the lowest transaction ID that should be retained
+   * @throws IOException if listing the storage directory fails.
+   */
   @Override
   public void purgeLogsOlderThan(long minTxIdToKeep)
       throws IOException {
     LOG.info("Purging logs older than " + minTxIdToKeep);
     File[] files = FileUtil.listFiles(sd.getCurrentDir());
     List<EditLogFile> editLogs = matchEditLogs(files, true);
-    for (EditLogFile log : editLogs) {
-      if (log.getFirstTxId() < minTxIdToKeep &&
-          log.getLastTxId() < minTxIdToKeep) {
-        purger.purgeLog(log);
+    synchronized (this) {
+      for (EditLogFile log : editLogs) {
+        if (log.getFirstTxId() < minTxIdToKeep &&
+            log.getLastTxId() < minTxIdToKeep) {
+          purger.purgeLog(log);
+        } else if (isStaleInProgressLog(minTxIdToKeep, log)) {
+          purger.markStale(log);
+        }
       }
     }
   }
 
+  private boolean isStaleInProgressLog(long minTxIdToKeep, EditLogFile log) {
+    return log.isInProgress() &&
+        !log.getFile().equals(currentInProgress) &&
+        log.getFirstTxId() >= minTxIdToKeep &&
+        // at last we check if this segment is not already marked as .trash,
+        // .empty or .corrupted, in which case it does not match the strict
+        // regex pattern.
+        EDITS_INPROGRESS_REGEX.matcher(log.getFile().getName()).matches();
+  }
+
   /**
    * Find all editlog segments starting at or above the given txid.
    * @param firstTxId the txnid which to start looking
@@ -596,7 +627,12 @@ public void moveAsideEmptyFile() throws IOException {
       assert lastTxId == HdfsServerConstants.INVALID_TXID;
       renameSelf(".empty");
     }
-      
+
+    public void moveAsideStaleInprogressFile() throws IOException {
+      assert isInProgress;
+      renameSelf(".stale");
+    }
+
     private void renameSelf(String newSuffix) throws IOException {
       File src = file;
       File dst = new File(src.getParent(), src.getName() + newSuffix);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java

@@ -207,21 +207,22 @@ private long getImageTxIdToRetain(
   /**
    * Interface responsible for disposing of old checkpoints and edit logs.
    */
-  static interface StoragePurger {
+  interface StoragePurger {
     void purgeLog(EditLogFile log);
     void purgeImage(FSImageFile image);
+    void markStale(EditLogFile log);
   }
 
   static class DeletionStoragePurger implements StoragePurger {
     @Override
     public void purgeLog(EditLogFile log) {
-      LOG.info("Purging old edit log " + log);
+      LOG.info("Purging old edit log {}", log);
       deleteOrWarn(log.getFile());
     }
 
     @Override
     public void purgeImage(FSImageFile image) {
-      LOG.info("Purging old image " + image);
+      LOG.info("Purging old image {}", image);
       deleteOrWarn(image.getFile());
       deleteOrWarn(MD5FileUtils.getDigestFileForFile(image.getFile()));
     }
@@ -230,9 +231,19 @@ private static void deleteOrWarn(File file) {
       if (!file.delete()) {
         // It's OK if we fail to delete something -- we'll catch it
         // next time we swing through this directory.
-        LOG.warn("Could not delete " + file);
+        LOG.warn("Could not delete {}", file);
       }      
     }
+
+    public void markStale(EditLogFile log){
+      try {
+        log.moveAsideStaleInprogressFile();
+      } catch (IOException e) {
+        // It is ok to just log the rename failure and go on, we will try next
+        // time just as with deletions.
+        LOG.warn("Could not mark {} as stale", log, e);
+      }
+    }
   }
 
   /**