YARN-10135. FS-CS converter tool: issue warning on dynamic auto-create mapping rules. Contributed by Peter Bacsko

Author: szilard-nemeth

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java

@@ -76,6 +76,9 @@ public class FSConfigToCSConfigRuleHandler {
   public static final String FAIR_AS_DRF =
       "fairAsDrf.action";
 
+  public static final String MAPPED_DYNAMIC_QUEUE =
+      "mappedDynamicQueue.action";
+
   @VisibleForTesting
   enum RuleAction {
     WARNING,
@@ -122,6 +125,7 @@ public void initPropertyActions() {
     setActionForProperty(RESERVATION_SYSTEM);
     setActionForProperty(QUEUE_AUTO_CREATE);
     setActionForProperty(FAIR_AS_DRF);
+    setActionForProperty(MAPPED_DYNAMIC_QUEUE);
   }
 
   public void handleMaxCapacityPercentage(String queueName) {
@@ -180,7 +184,8 @@ public void handleQueueAutoCreate(String placementRule) {
     handle(QUEUE_AUTO_CREATE,
         null,
         format(
-            "Placement rules: queue auto-create is not supported (type: %s)",
+            "Placement rules: queue auto-create is not supported (type: %s),"
+            + " please configure auto-create-child-queue property manually",
             placementRule));
   }
 
@@ -192,6 +197,21 @@ public void handleFairAsDrf(String queueName) {
             queueName));
   }
 
+  public void handleDynamicMappedQueue(String mapping, boolean create) {
+    String msg = "Mapping rule %s is dynamic - this might cause inconsistent"
+        + " behaviour compared to FS.";
+
+    if (create) {
+      msg += " Also, setting auto-create-child-queue=true is"
+          + " necessary, because the create flag was set to true on the"
+          + " original placement rule.";
+    }
+
+    handle(MAPPED_DYNAMIC_QUEUE,
+        null,
+        format(msg, mapping));
+  }
+
   private void handle(String actionName, String fsSetting, String message) {
     RuleAction action = actions.get(actionName);
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java

@@ -57,7 +57,7 @@ Map<String, String> convertPlacementPolicy(PlacementManager placementManager,
 
         // nested rule
         if (userRule.getParentRule() != null) {
-          handleNestedRule(mapping, userRule);
+          handleNestedRule(mapping, userRule, ruleHandler);
         } else {
           if (!userAsDefaultQueue) {
             if (mapping.length() > 0) {
@@ -102,20 +102,28 @@ Map<String, String> convertPlacementPolicy(PlacementManager placementManager,
   }
 
   private void handleNestedRule(StringBuilder mapping,
-      UserPlacementRule userRule) {
+      UserPlacementRule userRule, FSConfigToCSConfigRuleHandler ruleHandler) {
     PlacementRule pr = userRule.getParentRule();
     if (mapping.length() > 0) {
       mapping.append(RULE_SEPARATOR);
     }
     if (pr instanceof PrimaryGroupPlacementRule) {
-      mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER);
+      String mappingString = "u:" + USER + ":" + PRIMARY_GROUP + "." + USER;
+      ruleHandler.handleDynamicMappedQueue(mappingString,
+          ((PrimaryGroupPlacementRule) pr).getCreateFlag());
+      mapping.append(mappingString);
     } else if (pr instanceof SecondaryGroupExistingPlacementRule) {
+      String mappingString = "u:" + USER + ":" + SECONDARY_GROUP + "." + USER;
+      ruleHandler.handleDynamicMappedQueue(mappingString,
+          ((SecondaryGroupExistingPlacementRule) pr).getCreateFlag());
       mapping.append("u:" + USER + ":" + SECONDARY_GROUP + "." + USER);
     } else if (pr instanceof DefaultPlacementRule) {
       DefaultPlacementRule defaultRule = (DefaultPlacementRule) pr;
-      mapping.append("u:" + USER + ":")
-        .append(defaultRule.defaultQueueName)
-        .append("." + USER);
+      String mappingString =
+          "u:" + USER + ":" + defaultRule.defaultQueueName + "." + USER;
+      ruleHandler.handleDynamicMappedQueue(mappingString,
+          defaultRule.getCreateFlag());
+      mapping.append(mappingString);
     } else {
       throw new UnsupportedOperationException("Unsupported nested rule: "
           + pr.getClass().getCanonicalName());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java

@@ -19,6 +19,7 @@
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QUEUE_MAPPING;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
@@ -180,7 +181,8 @@ public void testConvertNestedPrimaryGroupRule() {
     Map<String, String> properties = convert(false);
 
     verifyMapping(properties, "u:%user:%primary_group.%user");
-    verifyZeroInteractions(ruleHandler);
+    verify(ruleHandler).handleDynamicMappedQueue(
+        eq("u:%user:%primary_group.%user"), eq(false));
   }
 
   @Test
@@ -194,7 +196,8 @@ public void testConvertNestedSecondaryGroupRule() {
     Map<String, String> properties = convert(false);
 
     verifyMapping(properties, "u:%user:%secondary_group.%user");
-    verifyZeroInteractions(ruleHandler);
+    verify(ruleHandler).handleDynamicMappedQueue(
+        eq("u:%user:%secondary_group.%user"), eq(false));
   }
 
   @Test
@@ -209,7 +212,8 @@ public void testConvertNestedDefaultRule() {
     Map<String, String> properties = convert(false);
 
     verifyMapping(properties, "u:%user:abc.%user");
-    verifyZeroInteractions(ruleHandler);
+    verify(ruleHandler).handleDynamicMappedQueue(
+        eq("u:%user:abc.%user"), eq(false));
   }
 
   @Test