Scan stable plugins for named components upon install (elastic#92528)

stable plugins not build with ES's gradle plugin will not have named_components.json file.
To allow these plugins to expose their named components, a scan can be performed upon install.

relates elastic#88980

Author: pgomulka

distribution/tools/plugin-cli/build.gradle

@@ -10,9 +10,18 @@ apply plugin: 'elasticsearch.build'
 
 archivesBaseName = 'elasticsearch-plugin-cli'
 
+tasks.named("dependencyLicenses").configure {
+  mapping from: /asm-.*/, to: 'asm'
+}
+
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:elasticsearch-cli")
+  implementation project(":libs:elasticsearch-plugin-api")
+  implementation project(":libs:elasticsearch-plugin-scanner")
+  implementation 'org.ow2.asm:asm:9.3'
+  implementation 'org.ow2.asm:asm-tree:9.3'
+
   api "org.bouncycastle:bcpg-fips:1.0.4"
   api "org.bouncycastle:bc-fips:1.0.2"
   testImplementation project(":test:framework")

distribution/tools/plugin-cli/licenses/asm-LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright (c) 2012 France Télécom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.

distribution/tools/plugin-cli/licenses/asm-NOTICE.txt

@@ -0,0 +1 @@
+ 

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -38,9 +38,12 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.jdk.JarHell;
+import org.elasticsearch.plugin.scanner.ClassReaders;
+import org.elasticsearch.plugin.scanner.NamedComponentScanner;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginDescriptor;
 import org.elasticsearch.plugins.PluginsUtils;
+import org.objectweb.asm.ClassReader;
 
 import java.io.BufferedReader;
 import java.io.Closeable;
@@ -82,6 +85,7 @@
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
@@ -197,6 +201,7 @@ public class InstallPluginAction implements Closeable {
     private Environment env;
     private boolean batch;
     private Proxy proxy = null;
+    private NamedComponentScanner scanner = new NamedComponentScanner();
 
     public InstallPluginAction(Terminal terminal, Environment env, boolean batch) {
         this.terminal = terminal;
@@ -208,7 +213,6 @@ public void setProxy(Proxy proxy) {
         this.proxy = proxy;
     }
 
-    // pkg private for testing
     public void execute(List<InstallablePlugin> plugins) throws Exception {
         if (plugins.isEmpty()) {
             throw new UserException(ExitCodes.USAGE, "at least one plugin id is required");
@@ -867,9 +871,24 @@ private PluginDescriptor loadPluginInfo(Path pluginRoot) throws Exception {
         // check for jar hell before any copying
         jarHellCheck(info, pluginRoot, env.pluginsFile(), env.modulesFile());
 
+        if (info.isStable() && hasNamedComponentFile(pluginRoot) == false) {
+            generateNameComponentFile(pluginRoot);
+        }
         return info;
     }
 
+    private void generateNameComponentFile(Path pluginRoot) throws IOException {
+        Stream<ClassReader> classPath = ClassReaders.ofClassPath().stream(); // contains plugin-api
+        List<ClassReader> classReaders = Stream.concat(ClassReaders.ofDirWithJars(pluginRoot).stream(), classPath).toList();
+        Map<String, Map<String, String>> namedComponentsMap = scanner.scanForNamedClasses(classReaders);
+        Path outputFile = pluginRoot.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME);
+        scanner.writeToFile(namedComponentsMap, outputFile);
+    }
+
+    private boolean hasNamedComponentFile(Path pluginRoot) {
+        return Files.exists(pluginRoot.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME));
+    }
+
     private static final String LIB_TOOLS_PLUGIN_CLI_CLASSPATH_JAR;
 
     static {

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java

@@ -48,11 +48,14 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.plugin.scanner.NamedComponentScanner;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginDescriptor;
 import org.elasticsearch.plugins.PluginTestUtil;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.PosixPermissionsResetter;
+import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
+import org.elasticsearch.test.jar.JarUtils;
 import org.junit.After;
 import org.junit.Before;
 
@@ -86,6 +89,7 @@
 import java.util.Arrays;
 import java.util.Date;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -124,13 +128,16 @@ public class InstallPluginActionTests extends ESTestCase {
     private MockTerminal terminal;
     private Tuple<Path, Environment> env;
     private Path pluginDir;
+    private NamedComponentScanner namedComponentScanner;
 
     private final boolean isPosix;
     private final boolean isReal;
     private final String javaIoTmpdir;
 
     @SuppressForbidden(reason = "sets java.io.tmpdir")
     public InstallPluginActionTests(FileSystem fs, Function<String, Path> temp) {
+        assert "false".equals(System.getProperty("tests.security.manager")) : "-Dtests.security.manager=false has to be set";
+
         this.temp = temp;
         this.isPosix = fs.supportedFileAttributeViews().contains("posix");
         this.isReal = fs == PathUtils.getDefaultFileSystem();
@@ -152,6 +159,7 @@ void jarHellCheck(PluginDescriptor candidateInfo, Path candidate, Path pluginsDi
                 // no jarhell check
             }
         };
+
         defaultAction = new InstallPluginAction(terminal, env.v2(), false);
     }
 
@@ -199,7 +207,9 @@ private static Configuration toPosix(Configuration configuration) {
         return configuration.toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
     }
 
-    /** Creates a test environment with bin, config and plugins directories. */
+    /**
+     * Creates a test environment with bin, config and plugins directories.
+     */
     static Tuple<Path, Environment> createEnv(Function<String, Path> temp) throws IOException {
         Path home = temp.apply("install-plugin-command-tests");
         Files.createDirectories(home.resolve("bin"));
@@ -216,7 +226,9 @@ static Path createPluginDir(Function<String, Path> temp) {
         return temp.apply("pluginDir");
     }
 
-    /** creates a fake jar file with empty class files */
+    /**
+     * creates a fake jar file with empty class files
+     */
     static void writeJar(Path jar, String... classes) throws IOException {
         try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(jar))) {
             for (String clazz : classes) {
@@ -237,13 +249,47 @@ static Path writeZip(Path structure, String prefix) throws IOException {
         return zip;
     }
 
-    /** creates a plugin .zip and returns the url for testing */
+    /**
+     * creates a plugin .zip and returns the url for testing
+     */
     static InstallablePlugin createPluginZip(String name, Path structure, String... additionalProps) throws IOException {
         return createPlugin(name, structure, additionalProps);
     }
 
+    static void writeStablePlugin(String name, Path structure, boolean hasNamedComponentFile, String... additionalProps)
+        throws IOException {
+        String[] properties = pluginProperties(name, additionalProps, true);
+        PluginTestUtil.writeStablePluginProperties(structure, properties);
+
+        if (hasNamedComponentFile) {
+            PluginTestUtil.writeNamedComponentsFile(structure, namedComponentsJSON());
+        }
+        Path jar = structure.resolve("plugin.jar");
+
+        JarUtils.createJarWithEntries(jar, Map.of("p/A.class", InMemoryJavaCompiler.compile("p.A", """
+            package p;
+            import org.elasticsearch.plugin.*;
+            import org.elasticsearch.plugins.cli.test_model.*;
+            @NamedComponent("a_component")
+            public class A implements ExtensibleInterface{}
+            """), "p/B.class", InMemoryJavaCompiler.compile("p.B", """
+            package p;
+            import org.elasticsearch.plugin.*;
+            import org.elasticsearch.plugins.cli.test_model.*;
+            @NamedComponent("b_component")
+            public class B implements ExtensibleInterface{}
+            """)));
+    }
+
     static void writePlugin(String name, Path structure, String... additionalProps) throws IOException {
-        String[] properties = Stream.concat(
+        String[] properties = pluginProperties(name, additionalProps, false);
+        PluginTestUtil.writePluginProperties(structure, properties);
+        String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin";
+        writeJar(structure.resolve("plugin.jar"), className);
+    }
+
+    private static String[] pluginProperties(String name, String[] additionalProps, boolean isStable) {
+        return Stream.of(
             Stream.of(
                 "description",
                 "fake desc",
@@ -254,15 +300,12 @@ static void writePlugin(String name, Path structure, String... additionalProps)
                 "elasticsearch.version",
                 Version.CURRENT.toString(),
                 "java.version",
-                System.getProperty("java.specification.version"),
-                "classname",
-                "FakePlugin"
+                System.getProperty("java.specification.version")
+
             ),
+            isStable ? Stream.empty() : Stream.of("classname", "FakePlugin"),
             Arrays.stream(additionalProps)
-        ).toArray(String[]::new);
-        PluginTestUtil.writePluginProperties(structure, properties);
-        String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin";
-        writeJar(structure.resolve("plugin.jar"), className);
+        ).flatMap(Function.identity()).toArray(String[]::new);
     }
 
     static void writePluginSecurityPolicy(Path pluginDir, String... permissions) throws IOException {
@@ -276,6 +319,12 @@ static void writePluginSecurityPolicy(Path pluginDir, String... permissions) thr
         Files.write(pluginDir.resolve("plugin-security.policy"), securityPolicyContent.toString().getBytes(StandardCharsets.UTF_8));
     }
 
+    static InstallablePlugin createStablePlugin(String name, Path structure, boolean hasNamedComponentFile, String... additionalProps)
+        throws IOException {
+        writeStablePlugin(name, structure, hasNamedComponentFile, additionalProps);
+        return new InstallablePlugin(name, writeZip(structure, null).toUri().toURL().toString());
+    }
+
     static InstallablePlugin createPlugin(String name, Path structure, String... additionalProps) throws IOException {
         writePlugin(name, structure, additionalProps);
         return new InstallablePlugin(name, writeZip(structure, null).toUri().toURL().toString());
@@ -310,6 +359,11 @@ void assertPlugin(String name, Path original, Environment environment) throws IO
         assertInstallCleaned(environment);
     }
 
+    void assertNamedComponentFile(String name, Path pluginDir, String expectedContent) throws IOException {
+        Path namedComponents = pluginDir.resolve(name).resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME);
+        assertThat(Files.readString(namedComponents), equalTo(expectedContent));
+    }
+
     void assertPluginInternal(String name, Path pluginsFile, Path originalPlugin) throws IOException {
         Path got = pluginsFile.resolve(name);
         assertTrue("dir " + name + " exists", Files.exists(got));
@@ -1507,4 +1561,42 @@ public void testInstallMigratedPlugins() throws Exception {
             assertThat(terminal.getErrorOutput(), containsString("[" + id + "] is no longer a plugin"));
         }
     }
+
+    public void testStablePluginWithNamedComponentsFile() throws Exception {
+        InstallablePlugin stablePluginZip = createStablePlugin("stable1", pluginDir, true);
+        installPlugins(List.of(stablePluginZip), env.v1());
+        assertPlugin("stable1", pluginDir, env.v2());
+        assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testStablePluginWithoutNamedComponentsFile() throws Exception {
+        // named component will have to be generated upon install
+        InstallablePlugin stablePluginZip = createStablePlugin("stable1", pluginDir, false);
+
+        installPlugins(List.of(stablePluginZip), env.v1());
+
+        assertPlugin("stable1", pluginDir, env.v2());
+        assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON());
+    }
+
+    private Map<String, Map<String, String>> namedComponentsMap() {
+        Map<String, Map<String, String>> result = new LinkedHashMap<>();
+        Map<String, String> extensibles = new LinkedHashMap<>();
+        extensibles.put("a_component", "p.A");
+        extensibles.put("b_component", "p.B");
+        result.put("org.elasticsearch.plugins.cli.test_model.ExtensibleInterface", extensibles);
+        return result;
+    }
+
+    private static String namedComponentsJSON() {
+        return """
+            {
+              "org.elasticsearch.plugins.cli.test_model.ExtensibleInterface": {
+                "a_component": "p.A",
+                "b_component": "p.B"
+              }
+            }
+            """.replaceAll("[\n\r\s]", "");
+    }
 }

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/test_model/ExtensibleInterface.java

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.cli.test_model;
+
+import org.elasticsearch.plugin.Extensible;
+
+@Extensible
+public interface ExtensibleInterface {}

docs/changelog/92528.yaml

@@ -0,0 +1,5 @@
+pr: 92528
+summary: Scan stable plugins for named components upon install
+area: Infra/CLI
+type: enhancement
+issues: []

libs/plugin-scanner/build.gradle

@@ -19,8 +19,8 @@ dependencies {
   api project(':libs:elasticsearch-plugin-api')
   api project(":libs:elasticsearch-x-content")
 
-  implementation 'org.ow2.asm:asm:9.3'
-  implementation 'org.ow2.asm:asm-tree:9.3'
+  api 'org.ow2.asm:asm:9.3'
+  api 'org.ow2.asm:asm-tree:9.3'
 
   testImplementation "junit:junit:${versions.junit}"
   testImplementation(project(":test:framework")) {

libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassReaders.java

@@ -40,11 +40,10 @@ public class ClassReaders {
      * This method must be used within a try-with-resources statement or similar
      * control structure.
      */
-    public static List<ClassReader> ofDirWithJars(String path) {
-        if (path == null) {
+    public static List<ClassReader> ofDirWithJars(Path dir) {
+        if (dir == null) {
             return Collections.emptyList();
         }
-        Path dir = Paths.get(path);
         try (var stream = Files.list(dir)) {
             return ofPaths(stream);
         } catch (IOException e) {