[FLINK-10880] Add Documentation.ExcludeFromDocumentation to exclude ConfigOptions from documentation

The annotation Documentation.ExcludeFromDocumentation can be used to annotate ConfigOptions
with in order to not include them in the documentation.

Author: tillrohrmann

flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java

@@ -59,6 +59,19 @@ public final class Documentation {
 		int position() default Integer.MAX_VALUE;
 	}
 
+	/**
+	 * Annotation used on config option fields to exclude the config option from documentation.
+	 */
+	@Target(ElementType.FIELD)
+	@Retention(RetentionPolicy.RUNTIME)
+	@Internal
+	public @interface ExcludeFromDocumentation {
+		/**
+		 * The optional reason why the config option is excluded from documentation.
+		 */
+		String value() default "";
+	}
+
 	private Documentation(){
 	}
 }

flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java

@@ -202,7 +202,7 @@ static List<OptionWithMetaInfo> extractConfigOptions(Class<?> clazz) {
 			List<OptionWithMetaInfo> configOptions = new ArrayList<>(8);
 			Field[] fields = clazz.getFields();
 			for (Field field : fields) {
-				if (field.getType().equals(ConfigOption.class) && field.getAnnotation(Deprecated.class) == null) {
+				if (isConfigOption(field) && shouldBeDocumented(field)) {
 					configOptions.add(new OptionWithMetaInfo((ConfigOption<?>) field.get(null), field));
 				}
 			}
@@ -213,6 +213,15 @@ static List<OptionWithMetaInfo> extractConfigOptions(Class<?> clazz) {
 		}
 	}
 
+	private static boolean isConfigOption(Field field) {
+		return field.getType().equals(ConfigOption.class);
+	}
+
+	private static boolean shouldBeDocumented(Field field) {
+		return field.getAnnotation(Deprecated.class) == null &&
+			field.getAnnotation(Documentation.ExcludeFromDocumentation.class) == null;
+	}
+
 	/**
 	 * Transforms this configuration group into HTML formatted table.
 	 * Options are sorted alphabetically by key.

flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java

@@ -266,4 +266,45 @@ public void testCommonOptions() throws IOException, ClassNotFoundException {
 
 		assertEquals(expected, output);
 	}
+
+	static class TestConfigGroupWithExclusion {
+		public static ConfigOption<Integer> firstOption = ConfigOptions
+			.key("first.option.a")
+			.defaultValue(2)
+			.withDescription("This is example description for the first option.");
+
+		@Documentation.ExcludeFromDocumentation
+		public static ConfigOption<String> excludedOption = ConfigOptions
+			.key("excluded.option.a")
+			.noDefaultValue()
+			.withDescription("This should not be documented.");
+	}
+
+	/**
+	 * Tests that {@link ConfigOption} annotated with {@link Documentation.ExcludeFromDocumentation}
+	 * are not documented.
+	 */
+	@Test
+	public void testConfigOptionExclusion() {
+		final String expectedTable =
+			"<table class=\"table table-bordered\">\n" +
+				"    <thead>\n" +
+				"        <tr>\n" +
+				"            <th class=\"text-left\" style=\"width: 20%\">Key</th>\n" +
+				"            <th class=\"text-left\" style=\"width: 15%\">Default</th>\n" +
+				"            <th class=\"text-left\" style=\"width: 65%\">Description</th>\n" +
+				"        </tr>\n" +
+				"    </thead>\n" +
+				"    <tbody>\n" +
+				"        <tr>\n" +
+				"            <td><h5>first.option.a</h5></td>\n" +
+				"            <td style=\"word-wrap: break-word;\">2</td>\n" +
+				"            <td>This is example description for the first option.</td>\n" +
+				"        </tr>\n" +
+				"    </tbody>\n" +
+				"</table>\n";
+		final String htmlTable = ConfigOptionsDocGenerator.generateTablesForClass(TestConfigGroupWithExclusion.class).get(0).f1;
+
+		assertEquals(expectedTable, htmlTable);
+	}
 }