Merge branch 'dif-chat-comp' of github.com:szabosteve/elasticsearch i…

…nto dif-chat-comp

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -60,6 +60,7 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -77,7 +78,7 @@ public class ScriptScoreBenchmark {
     private final PluginsService pluginsService = new PluginsService(
         Settings.EMPTY,
         null,
-        PluginsLoader.createPluginsLoader(null, Path.of(System.getProperty("plugins.dir")))
+        PluginsLoader.createPluginsLoader(Set.of(), PluginsLoader.loadPluginsBundles(Path.of(System.getProperty("plugins.dir"))))
     );
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 

docs/changelog/120108.yaml

@@ -0,0 +1,13 @@
+pr: 120108
+summary: Remove the ability to read frozen indices
+area: Indices APIs
+type: breaking
+issues: []
+breaking:
+  title: Remove the ability to read frozen indices
+  area: Index setting
+  details: >-
+    The ability to read frozen indices has been removed. (Frozen indices are no longer useful due to improvements in
+    heap memory usage. The ability to freeze indices was removed in 8.0.)
+  impact: Users must unfreeze any frozen indices before upgrading.
+  notable: false

docs/changelog/120590.yaml

@@ -0,0 +1,5 @@
+pr: 120590
+summary: Map `scope.name` as a dimension
+area: Data streams
+type: bug
+issues: []

docs/changelog/120591.yaml

@@ -0,0 +1,5 @@
+pr: 120591
+summary: Increase field limit for OTel metrics to 10 000
+area: TSDB
+type: enhancement
+issues: []

docs/reference/rest-api/info.asciidoc

@@ -95,10 +95,6 @@ Example response:
           "available" : true,
           "enabled" : true
       },
-      "frozen_indices" : {
-         "available" : true,
-         "enabled" : true
-      },
       "graph" : {
          "available" : true,
          "enabled" : true

docs/reference/rest-api/usage.asciidoc

@@ -356,11 +356,6 @@ GET /_xpack/usage
     "full_copy_indices_count" : 0,
     "shared_cache_indices_count" : 0
   },
-  "frozen_indices" : {
-    "available" : true,
-    "enabled" : true,
-    "indices_count" : 0
-  },
   "spatial" : {
     "available" : true,
     "enabled" : true

docs/reference/search/search-your-data/search-across-clusters.asciidoc

@@ -109,7 +109,7 @@ PUT _cluster/settings
 // end::ccs-remote-cluster-setup[]
 
 <1> Since `skip_unavailable` was not set on `cluster_three`, it uses
-the default of `false`. See the <<skip-unavailable-clusters>>
+the default of `true`. See the <<skip-unavailable-clusters>>
 section for details.
 
 

libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java

@@ -398,4 +398,16 @@ public interface EntitlementChecker {
     void check$sun_nio_ch_DatagramChannelImpl$send(Class<?> callerClass, DatagramChannel that, ByteBuffer src, SocketAddress target);
 
     void check$sun_nio_ch_DatagramChannelImpl$receive(Class<?> callerClass, DatagramChannel that, ByteBuffer dst);
+
+    ////////////////////
+    //
+    // Load native libraries
+    //
+    void check$java_lang_Runtime$load(Class<?> callerClass, Runtime that, String filename);
+
+    void check$java_lang_Runtime$loadLibrary(Class<?> callerClass, Runtime that, String libname);
+
+    void check$java_lang_System$$load(Class<?> callerClass, String filename);
+
+    void check$java_lang_System$$loadLibrary(Class<?> callerClass, String libname);
 }

libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/LoadNativeLibrariesCheckActions.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.qa.common;
+
+class LoadNativeLibrariesCheckActions {
+    static void runtimeLoad() {
+        try {
+            Runtime.getRuntime().load("libSomeLibFile.so");
+        } catch (UnsatisfiedLinkError ignored) {
+            // The library does not exist, so we expect to fail loading it
+        }
+    }
+
+    static void systemLoad() {
+        try {
+            System.load("libSomeLibFile.so");
+        } catch (UnsatisfiedLinkError ignored) {
+            // The library does not exist, so we expect to fail loading it
+        }
+    }
+
+    static void runtimeLoadLibrary() {
+        try {
+            Runtime.getRuntime().loadLibrary("SomeLib");
+        } catch (UnsatisfiedLinkError ignored) {
+            // The library does not exist, so we expect to fail loading it
+        }
+    }
+
+    static void systemLoadLibrary() {
+        try {
+            System.loadLibrary("SomeLib");
+        } catch (UnsatisfiedLinkError ignored) {
+            // The library does not exist, so we expect to fail loading it
+        }
+    }
+}

libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/NetworkAccessCheckActions.java

@@ -88,6 +88,9 @@ static void createLDAPCertStore() throws NoSuchAlgorithmException {
         } catch (InvalidAlgorithmParameterException ex) {
             // Assert we actually hit the class we care about, LDAPCertStore (or its impl)
             assert Arrays.stream(ex.getStackTrace()).anyMatch(e -> e.getClassName().endsWith("LDAPCertStore"));
+        } catch (NoSuchAlgorithmException e) {
+            // In some environments (e.g. with FIPS enabled) the LDAPCertStore is not present, so this will fail.
+            // This is OK, as this means the class we care about (LDAPCertStore) is not even present
         }
     }
 

libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java

@@ -192,7 +192,12 @@ static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
         entry("datagram_channel_bind", forPlugins(NetworkAccessCheckActions::datagramChannelBind)),
         entry("datagram_channel_connect", forPlugins(NetworkAccessCheckActions::datagramChannelConnect)),
         entry("datagram_channel_send", forPlugins(NetworkAccessCheckActions::datagramChannelSend)),
-        entry("datagram_channel_receive", forPlugins(NetworkAccessCheckActions::datagramChannelReceive))
+        entry("datagram_channel_receive", forPlugins(NetworkAccessCheckActions::datagramChannelReceive)),
+
+        entry("runtime_load", forPlugins(LoadNativeLibrariesCheckActions::runtimeLoad)),
+        entry("runtime_load_library", forPlugins(LoadNativeLibrariesCheckActions::runtimeLoadLibrary)),
+        entry("system_load", forPlugins(LoadNativeLibrariesCheckActions::systemLoad)),
+        entry("system_load_library", forPlugins(LoadNativeLibrariesCheckActions::systemLoadLibrary))
     )
         .filter(entry -> entry.getValue().fromJavaVersion() == null || Runtime.version().feature() >= entry.getValue().fromJavaVersion())
         .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));

libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml

@@ -7,3 +7,4 @@ ALL-UNNAMED:
       properties:
         - es.entitlements.checkSetSystemProperty
         - es.entitlements.checkClearSystemProperty
+  - load_native_libraries

libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml

@@ -7,3 +7,4 @@ org.elasticsearch.entitlement.qa.common:
       properties:
         - es.entitlements.checkSetSystemProperty
         - es.entitlements.checkClearSystemProperty
+  - load_native_libraries

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -17,32 +17,27 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
 import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
+import org.elasticsearch.entitlement.runtime.policy.Policy;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Collection;
+import java.util.Map;
 import java.util.function.Function;
 
 import static java.util.Objects.requireNonNull;
 
 public class EntitlementBootstrap {
 
-    public record BootstrapArgs(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {
+    public record BootstrapArgs(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
         public BootstrapArgs {
-            requireNonNull(pluginData);
+            requireNonNull(pluginPolicies);
             requireNonNull(pluginResolver);
         }
     }
 
-    public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {
-        public PluginData {
-            requireNonNull(pluginPath);
-        }
-    }
-
     private static BootstrapArgs bootstrapArgs;
 
     public static BootstrapArgs bootstrapArgs() {
@@ -52,16 +47,16 @@ public static BootstrapArgs bootstrapArgs() {
     /**
      * Activates entitlement checking. Once this method returns, calls to methods protected by Entitlements from classes without a valid
      * policy will throw {@link org.elasticsearch.entitlement.runtime.api.NotEntitledException}.
-     * @param pluginData a collection of (plugin path, boolean, boolean), that holds the paths of all the installed Elasticsearch modules
-     *                   and plugins, whether they are Java modular or not, and whether they are Elasticsearch modules or external plugins.
+     *
+     * @param pluginPolicies a map holding policies for plugins (and modules), by plugin (or module) name.
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
      */
-    public static void bootstrap(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {
+    public static void bootstrap(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");
         }
-        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginData, pluginResolver);
+        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver);
         exportInitializationToAgent();
         loadAgent(findAgentJar());
         selfTest();

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -9,7 +9,6 @@
 
 package org.elasticsearch.entitlement.initialization;
 
-import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.internal.provider.ProviderLocator;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
 import org.elasticsearch.entitlement.bridge.EntitlementChecker;
@@ -23,31 +22,21 @@
 import org.elasticsearch.entitlement.runtime.policy.Entitlement;
 import org.elasticsearch.entitlement.runtime.policy.ExitVMEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.InboundNetworkEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.LoadNativeLibrariesEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.OutboundNetworkEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
 import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
-import org.elasticsearch.entitlement.runtime.policy.PolicyParser;
 import org.elasticsearch.entitlement.runtime.policy.Scope;
 
-import java.io.IOException;
 import java.lang.instrument.Instrumentation;
-import java.lang.module.ModuleFinder;
-import java.lang.module.ModuleReference;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED;
-
 /**
  * Called by the agent during {@code agentmain} to configure the entitlement system,
  * instantiate and configure an {@link EntitlementChecker},
@@ -57,7 +46,6 @@
  */
 public class EntitlementInitialization {
 
-    private static final String POLICY_FILE_NAME = "entitlement-policy.yaml";
     private static final Module ENTITLEMENTS_MODULE = PolicyManager.class.getModule();
 
     private static ElasticsearchEntitlementChecker manager;
@@ -90,8 +78,8 @@ private static Class<?>[] findClassesToRetransform(Class<?>[] loadedClasses, Set
         return retransform.toArray(new Class<?>[0]);
     }
 
-    private static PolicyManager createPolicyManager() throws IOException {
-        Map<String, Policy> pluginPolicies = createPluginPolicies(EntitlementBootstrap.bootstrapArgs().pluginData());
+    private static PolicyManager createPolicyManager() {
+        Map<String, Policy> pluginPolicies = EntitlementBootstrap.bootstrapArgs().pluginPolicies();
 
         // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it
         var serverPolicy = new Policy(
@@ -105,7 +93,8 @@ private static PolicyManager createPolicyManager() throws IOException {
                         new ExitVMEntitlement(),
                         new CreateClassLoaderEntitlement(),
                         new InboundNetworkEntitlement(),
-                        new OutboundNetworkEntitlement()
+                        new OutboundNetworkEntitlement(),
+                        new LoadNativeLibrariesEntitlement()
                     )
                 ),
                 new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())),
@@ -119,62 +108,7 @@ private static PolicyManager createPolicyManager() throws IOException {
         return new PolicyManager(serverPolicy, agentEntitlements, pluginPolicies, resolver, ENTITLEMENTS_MODULE);
     }
 
-    private static Map<String, Policy> createPluginPolicies(Collection<EntitlementBootstrap.PluginData> pluginData) throws IOException {
-        Map<String, Policy> pluginPolicies = new HashMap<>(pluginData.size());
-        for (var entry : pluginData) {
-            Path pluginRoot = entry.pluginPath();
-            String pluginName = pluginRoot.getFileName().toString();
-
-            final Policy policy = loadPluginPolicy(pluginRoot, entry.isModular(), pluginName, entry.isExternalPlugin());
-
-            pluginPolicies.put(pluginName, policy);
-        }
-        return pluginPolicies;
-    }
-
-    private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName, boolean isExternalPlugin)
-        throws IOException {
-        Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
-
-        final Set<String> moduleNames = getModuleNames(pluginRoot, isModular);
-        final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin);
-
-        // TODO: should this check actually be part of the parser?
-        for (Scope scope : policy.scopes()) {
-            if (moduleNames.contains(scope.moduleName()) == false) {
-                throw new IllegalStateException(
-                    Strings.format(
-                        "Invalid module name in policy: plugin [%s] does not have module [%s]; available modules [%s]; policy file [%s]",
-                        pluginName,
-                        scope.moduleName(),
-                        String.join(", ", moduleNames),
-                        policyFile
-                    )
-                );
-            }
-        }
-        return policy;
-    }
-
-    private static Policy parsePolicyIfExists(String pluginName, Path policyFile, boolean isExternalPlugin) throws IOException {
-        if (Files.exists(policyFile)) {
-            return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName, isExternalPlugin).parsePolicy();
-        }
-        return new Policy(pluginName, List.of());
-    }
-
-    private static Set<String> getModuleNames(Path pluginRoot, boolean isModular) {
-        if (isModular) {
-            ModuleFinder moduleFinder = ModuleFinder.of(pluginRoot);
-            Set<ModuleReference> moduleReferences = moduleFinder.findAll();
-
-            return moduleReferences.stream().map(mr -> mr.descriptor().name()).collect(Collectors.toUnmodifiableSet());
-        }
-        // When isModular == false we use the same "ALL-UNNAMED" constant as the JDK to indicate (any) unnamed module for this plugin
-        return Set.of(ALL_UNNAMED);
-    }
-
-    private static ElasticsearchEntitlementChecker initChecker() throws IOException {
+    private static ElasticsearchEntitlementChecker initChecker() {
         final PolicyManager policyManager = createPolicyManager();
 
         int javaVersion = Runtime.version().feature();

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java

@@ -732,4 +732,24 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) {
     public void check$sun_nio_ch_DatagramChannelImpl$receive(Class<?> callerClass, DatagramChannel that, ByteBuffer dst) {
         policyManager.checkInboundNetworkAccess(callerClass);
     }
+
+    @Override
+    public void check$java_lang_Runtime$load(Class<?> callerClass, Runtime that, String filename) {
+        policyManager.checkLoadingNativeLibraries(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_Runtime$loadLibrary(Class<?> callerClass, Runtime that, String libname) {
+        policyManager.checkLoadingNativeLibraries(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_System$$load(Class<?> callerClass, String filename) {
+        policyManager.checkLoadingNativeLibraries(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_System$$loadLibrary(Class<?> callerClass, String libname) {
+        policyManager.checkLoadingNativeLibraries(callerClass);
+    }
 }