Add JUL bridge (elastic#96683)

This commit adds the Log4j JUL bridge so that messages using JUL are
more nicely converted to log4j messages. Currently these messages are
captured via the stdout logging stream. This commit also adds a log4j
filter to replace the logging stream filtering mechanism used to quiet
some Lucene log messages that may be confusing to users.

closes elastic#94613

Author: rjernst

gradle/verification-metadata.xml

@@ -2478,6 +2478,11 @@
             <sha256 value="47f768ffd66107a66f0c2a19445ab1e42ce6719a7f30f9aa9ef96157c83949fd" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.logging.log4j" name="log4j-jul" version="2.19.0">
+         <artifact name="log4j-jul-2.19.0.jar">
+            <sha256 value="c3f0cbd1e455b1f3443c1bf0860fa3a91f5ae721d1acfeff393629fdefc23b6b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.logging.log4j" name="log4j-slf4j-impl" version="2.19.0">
          <artifact name="log4j-slf4j-impl-2.19.0.jar">
             <sha256 value="015d5c229f3cd5c0ebf175c1da08d596d94043362ae9d92637d88848c90537c8" origin="Generated by Gradle"/>

qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java

@@ -144,7 +144,7 @@ public void testLoggingLevelsFromSettings() throws IOException, UserException {
         final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
         final Configuration config = ctx.getConfiguration();
         final Map<String, LoggerConfig> loggerConfigs = config.getLoggers();
-        assertThat(loggerConfigs.size(), equalTo(3));
+        assertThat(loggerConfigs.size(), equalTo(5));
         assertThat(loggerConfigs, hasKey(""));
         assertThat(loggerConfigs.get("").getLevel(), equalTo(rootLevel));
         assertThat(loggerConfigs, hasKey("foo"));

server/build.gradle

@@ -64,6 +64,7 @@ dependencies {
   // logging
   api "org.apache.logging.log4j:log4j-api:${versions.log4j}"
   api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
+  implementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
 
   api "net.java.dev.jna:jna:${versions.jna}"
 
@@ -248,6 +249,7 @@ tasks.named("thirdPartyAudit").configure {
 
 tasks.named("dependencyLicenses").configure {
     mapping from: /lucene-.*/, to: 'lucene'
+    mapping from: /log4j-.*/, to: 'log4j'
     dependencies = project.configurations.runtimeClasspath.fileCollection {
         it.group.startsWith('org.elasticsearch') == false ||
                 // keep the following org.elasticsearch jars in

server/licenses/log4j-api-LICENSE.txt server/licenses/log4j-LICENSE.txtserver/licenses/log4j-api-LICENSE.txt renamed to server/licenses/log4j-LICENSE.txt

@@ -39,6 +39,7 @@
 
     requires org.apache.logging.log4j;
     requires org.apache.logging.log4j.core;
+    requires org.apache.logging.log4j.jul;
 
     requires org.apache.lucene.analysis.common;
     requires org.apache.lucene.backward_codecs;

server/licenses/log4j-api-NOTICE.txt server/licenses/log4j-NOTICE.txtserver/licenses/log4j-api-NOTICE.txt renamed to server/licenses/log4j-NOTICE.txt

@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Filter;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.appender.ConsoleAppender;
 import org.apache.logging.log4j.core.config.AbstractConfiguration;
@@ -21,11 +22,13 @@
 import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
 import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory;
 import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
+import org.apache.logging.log4j.core.config.builder.impl.DefaultConfigurationBuilder;
 import org.apache.logging.log4j.core.config.composite.CompositeConfiguration;
 import org.apache.logging.log4j.core.config.plugins.util.PluginManager;
 import org.apache.logging.log4j.core.config.properties.PropertiesConfiguration;
 import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationBuilder;
 import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationFactory;
+import org.apache.logging.log4j.jul.Log4jBridgeHandler;
 import org.apache.logging.log4j.status.StatusConsoleListener;
 import org.apache.logging.log4j.status.StatusData;
 import org.apache.logging.log4j.status.StatusListener;
@@ -222,6 +225,7 @@ public PropertiesConfiguration getConfiguration(final LoggerContext loggerContex
                         properties.setProperty(name, value.replace("%marker", "[%node_name]%marker "));
                     }
                 }
+
                 // end hack
                 return new PropertiesConfigurationBuilder().setConfigurationSource(source)
                     .setRootProperties(properties)
@@ -241,6 +245,8 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
         });
         assert configurations.isEmpty() == false;
 
+        configurations.add(createStaticConfiguration(context));
+
         context.start(new CompositeConfiguration(configurations));
 
         configureLoggerLevels(settings);
@@ -257,26 +263,13 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
                 );
         }
 
+        Log4jBridgeHandler.install(true, "", true);
+
         // Redirect stdout/stderr to log4j. While we ensure Elasticsearch code does not write to those streams,
         // third party libraries may do that. Note that we do NOT close the streams because other code may have
         // grabbed a handle to the streams and intend to write to it, eg log4j for writing to the console
-        System.setOut(
-            new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO, List.of()), false, StandardCharsets.UTF_8)
-        );
-        System.setErr(
-            new PrintStream(
-                new LoggingOutputStream(
-                    LogManager.getLogger("stderr"),
-                    Level.WARN,
-                    // MMapDirectory messages come from Lucene, suggesting to users as a warning that they should enable preview features in
-                    // the JDK. Vector logs come from Lucene too, but only if the used explicitly disables the Vector API - no point warning
-                    // in this case.
-                    List.of("MMapDirectory", "VectorUtilProvider", "WARNING: Java vector incubator module is not readable")
-                ),
-                false,
-                StandardCharsets.UTF_8
-            )
-        );
+        System.setOut(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO), false, StandardCharsets.UTF_8));
+        System.setErr(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stderr"), Level.WARN), false, StandardCharsets.UTF_8));
 
         final Logger rootLogger = LogManager.getRootLogger();
         Appender appender = Loggers.findAppender(rootLogger, ConsoleAppender.class);
@@ -289,6 +282,29 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
         }
     }
 
+    /**
+     * Creates a log4j configuration that is not changeable by users.
+     */
+    private static AbstractConfiguration createStaticConfiguration(LoggerContext context) {
+        var builder = new DefaultConfigurationBuilder<>();
+        builder.setConfigurationSource(ConfigurationSource.NULL_SOURCE);
+        builder.setLoggerContext(context);
+
+        // adding filters for confusing Lucene messages
+        addRegexFilter(builder, "org.apache.lucene.store.MemorySegmentIndexInputProvider", "Using MemorySegmentIndexInput.*");
+        addRegexFilter(builder, "org.apache.lucene.util.VectorUtilProvider", ".* incubator module is not readable.*");
+
+        return builder.build();
+    }
+
+    private static void addRegexFilter(DefaultConfigurationBuilder<BuiltConfiguration> builder, String loggerName, String pattern) {
+        var filterBuilder = builder.newFilter("RegexFilter", Filter.Result.DENY, Filter.Result.NEUTRAL);
+        filterBuilder.addAttribute("regex", pattern);
+        var loggerBuilder = builder.newLogger(loggerName);
+        loggerBuilder.add(filterBuilder);
+        builder.add(loggerBuilder);
+    }
+
     /**
      * Removes the appender for the console, if one exists.
      */