diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java deleted file mode 100644 index 849e05742f2b8..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import com.google.common.annotations.VisibleForTesting; - -import org.elasticsearch.gradle.VersionProperties; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.TreeMap; - -import static java.util.Comparator.comparing; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toList; - -/** - * Generates the page that contains breaking changes deprecations for a minor release series. - */ -public class BreakingChangesGenerator { - - static void update(File migrationTemplateFile, File migrationOutputFile, List entries) throws IOException { - try (FileWriter output = new FileWriter(migrationOutputFile)) { - output.write( - generateMigrationFile( - QualifiedVersion.of(VersionProperties.getElasticsearch()), - Files.readString(migrationTemplateFile.toPath()), - entries - ) - ); - } - } - - @VisibleForTesting - static String generateMigrationFile(QualifiedVersion version, String template, List entries) throws IOException { - final Map>> deprecationsByNotabilityByArea = entries.stream() - .map(ChangelogEntry::getDeprecation) - .filter(Objects::nonNull) - .sorted(comparing(ChangelogEntry.Deprecation::getTitle)) - .collect( - groupingBy( - ChangelogEntry.Deprecation::isNotable, - TreeMap::new, - groupingBy(ChangelogEntry.Deprecation::getArea, TreeMap::new, toList()) - ) - ); - - final Map>> breakingByNotabilityByArea = entries.stream() - .map(ChangelogEntry::getBreaking) - .filter(Objects::nonNull) - .sorted(comparing(ChangelogEntry.Breaking::getTitle)) - .collect( - groupingBy( - ChangelogEntry.Breaking::isNotable, - TreeMap::new, - groupingBy(ChangelogEntry.Breaking::getArea, TreeMap::new, toList()) - ) - ); - - final Map bindings = new HashMap<>(); - bindings.put("breakingByNotabilityByArea", breakingByNotabilityByArea); - bindings.put("deprecationsByNotabilityByArea", deprecationsByNotabilityByArea); - bindings.put("isElasticsearchSnapshot", version.isSnapshot()); - bindings.put("majorDotMinor", version.major() + "." + version.minor()); - bindings.put("majorDotMinorDotRevision", version.major() + "." + version.minor() + "." + version.revision()); - bindings.put("majorMinor", String.valueOf(version.major()) + version.minor()); - bindings.put("nextMajor", (version.major() + 1) + ".0"); - bindings.put("version", version); - - return TemplateUtils.render(template, bindings); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java new file mode 100644 index 0000000000000..9df0faaeb7262 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.release; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.Directory; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.RegularFile; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.options.Option; +import org.gradle.process.ExecOperations; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.annotation.Nullable; +import javax.inject.Inject; + +import static java.util.stream.Collectors.toList; + +public class BundleChangelogsTask extends DefaultTask { + private static final Logger LOGGER = Logging.getLogger(BundleChangelogsTask.class); + + private final ConfigurableFileCollection changelogs; + + private final RegularFileProperty bundleFile; + private final DirectoryProperty changelogDirectory; + + private final GitWrapper gitWrapper; + + @Nullable + private String branch; + @Nullable + private String bcRef; + // private boolean updateExisting; + private boolean finalize; + + // @Option( + // option = "update-existing", + // description = "Only update entries that are already in the bundle. Useful for updating the bundle after a BC has been cut." + // ) + // public void setUpdateExisting(boolean updateExisting) { + // this.updateExisting = updateExisting; + // } + + @Option(option = "branch", description = "Branch (or other ref) to use for generating the changelog bundle.") + public void setBranch(String branch) { + this.branch = branch; + } + + @Option( + option = "bc-ref", + description = "A source ref, typically the sha of a BC, that should be used to source PRs for changelog entries. " + + "The actual content of the changelogs will come from the 'branch' ref. " + + "You should generally always use bc-ref." + ) + public void setBcRef(String ref) { + this.bcRef = ref; + } + + @Option(option = "finalize", description = "Specify that the bundle is finalized, i.e. that the version has been released.") + public void setFinalize(boolean finalize) { + this.finalize = finalize; + } + + private static final ObjectMapper yamlMapper = new ObjectMapper( + new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) + .disable(YAMLGenerator.Feature.SPLIT_LINES) + .enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR) + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE) + ).setSerializationInclusion(JsonInclude.Include.NON_NULL); + + @Inject + public BundleChangelogsTask(ObjectFactory objectFactory, ExecOperations execOperations) { + changelogs = objectFactory.fileCollection(); + + bundleFile = objectFactory.fileProperty(); + changelogDirectory = objectFactory.directoryProperty(); + + gitWrapper = new GitWrapper(execOperations); + } + + @TaskAction + public void executeTask() throws IOException { + if (branch == null) { + throw new IllegalArgumentException("'branch' not specified."); + } + + final String upstreamRemote = gitWrapper.getUpstream(); + Set entriesFromBc = Set.of(); + + try { + var usingBcRef = bcRef != null && !bcRef.isEmpty(); + if (usingBcRef) { + // Check out all the changelogs that existed at the time of the BC + checkoutChangelogs(gitWrapper, upstreamRemote, bcRef); + entriesFromBc = this.changelogDirectory.getAsFileTree().getFiles().stream().map(File::getName).collect(Collectors.toSet()); + + // Then add/update changelogs from the HEAD of the branch + // We do an "add" here, rather than checking out the entire directory, in case changelogs have been removed for some reason + addChangelogsFromRef(gitWrapper, upstreamRemote, branch); + } else { + checkoutChangelogs(gitWrapper, upstreamRemote, branch); + } + Properties props = new Properties(); + props.load(new StringReader(gitWrapper.runCommand("git", "show", branch + ":build-tools-internal/version.properties"))); + String version = props.getProperty("elasticsearch"); + + LOGGER.info("Finding changelog files..."); + + Set finalEntriesFromBc = entriesFromBc; + List entries = this.changelogDirectory.getAsFileTree().getFiles().stream().filter(f -> { + // When not using a bc ref, we just take everything from the branch/sha passed in + if (!usingBcRef) { + return true; + } + + // If the changelog was present in the BC sha, use it + if (finalEntriesFromBc.contains(f.getName())) { + return true; + } + + // Otherwise, let's check to see if a reference to the PR exists in the commit log for the sha + // This specifically covers the case of a PR being merged into the BC with a missing changelog file, and the file added + // later. + var prNumber = f.getName().replace(".yaml", ""); + return !gitWrapper.runCommand("git", "log", bcRef, "--grep", "(#" + prNumber + ")").trim().isEmpty(); + }).map(ChangelogEntry::parse).sorted(Comparator.comparing(ChangelogEntry::getPr)).collect(toList()); + + ChangelogBundle existingBundle = null; + // if (updateExisting) { + // var existingBundleFile = new File("docs/release-notes/changelog-bundles/" + version + ".yml"); + // if (existingBundleFile.exists()) { + // var bundle = ChangelogBundle.parse(existingBundleFile); + // existingBundle = bundle; + // entries = entries.stream() + // .filter(e -> bundle.changelogs().stream().anyMatch(c -> c.getPr().equals(e.getPr()))) + // .toList(); + // } + // } + + var isReleased = finalize == true; + + ChangelogBundle bundle = new ChangelogBundle(version, isReleased, Instant.now().toString(), entries); + + yamlMapper.writeValue(new File("docs/release-notes/changelog-bundles/" + version + ".yml"), bundle); + } finally { + gitWrapper.runCommand("git", "restore", "-s@", "-SW", "--", "docs/changelog"); + } + } + + private static void checkoutChangelogs(GitWrapper gitWrapper, String upstream, String ref) { + gitWrapper.updateRemote(upstream); + // TODO check for changes first + gitWrapper.runCommand("rm", "-rf", "docs/changelog"); + var refSpec = upstream + "/" + ref; + if (ref.contains("upstream/")) { + refSpec = ref.replace("upstream/", upstream + "/"); + } else if (ref.matches("^[0-9a-f]+$")) { + refSpec = ref; + } + gitWrapper.runCommand("git", "checkout", refSpec, "--", "docs/changelog"); + } + + private static void addChangelogsFromRef(GitWrapper gitWrapper, String upstream, String ref) { + var refSpec = upstream + "/" + ref; + if (ref.contains("upstream/")) { + refSpec = ref.replace("upstream/", upstream + "/"); + } else if (ref.matches("^[0-9a-f]+$")) { + refSpec = ref; + } + + gitWrapper.runCommand("git", "checkout", refSpec, "--", "docs/changelog/*.yaml"); + } + + @InputDirectory + public DirectoryProperty getChangelogDirectory() { + return changelogDirectory; + } + + public void setChangelogDirectory(Directory dir) { + this.changelogDirectory.set(dir); + } + + @InputFiles + public FileCollection getChangelogs() { + return changelogs; + } + + public void setChangelogs(FileCollection files) { + this.changelogs.setFrom(files); + } + + @OutputFile + public RegularFileProperty getBundleFile() { + return bundleFile; + } + + public void setBundleFile(RegularFile file) { + this.bundleFile.set(file); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java new file mode 100644 index 0000000000000..355ea64355eb8 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.release; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; + +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; + +public record ChangelogBundle(String version, boolean released, String generated, List changelogs) { + + private static final Logger LOGGER = Logging.getLogger(GenerateReleaseNotesTask.class); + private static final ObjectMapper yamlMapper = new ObjectMapper( + new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES).disable(YAMLGenerator.Feature.SPLIT_LINES) + ); + + public ChangelogBundle(String version, String generated, List changelogs) { + this(version, false, generated, changelogs); + } + + public static ChangelogBundle parse(File file) { + try { + return yamlMapper.readValue(file, ChangelogBundle.class); + } catch (IOException e) { + LOGGER.error("Failed to parse changelog bundle from " + file.getAbsolutePath(), e); + throw new UncheckedIOException(e); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java index ff3ad2f944c52..e7b8cad1c8e49 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.release; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -35,12 +36,12 @@ public class ChangelogEntry { private static final Logger LOGGER = Logging.getLogger(GenerateReleaseNotesTask.class); private Integer pr; - private List issues; + private String summary; private String area; private String type; - private String summary; - private Highlight highlight; + private List issues; private Breaking breaking; + private Highlight highlight; private Deprecation deprecation; private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); @@ -193,6 +194,7 @@ public void setBody(String body) { this.body = body; } + @JsonIgnore public String getAnchor() { return generatedAnchor(this.title); } @@ -278,6 +280,7 @@ public void setNotable(boolean notable) { this.notable = notable; } + @JsonIgnore public String getAnchor() { return generatedAnchor(this.title); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index 87aa03f0445c5..2d91fb18e9f2a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -15,34 +15,31 @@ import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.Directory; +import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.FileCollection; import org.gradle.api.file.RegularFile; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFile; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; import org.gradle.process.ExecOperations; -import java.io.File; import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.Comparator; import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Stream; import javax.inject.Inject; -import static java.util.Comparator.naturalOrder; import static java.util.stream.Collectors.toSet; /** @@ -53,17 +50,17 @@ public class GenerateReleaseNotesTask extends DefaultTask { private final ConfigurableFileCollection changelogs; - private final RegularFileProperty releaseNotesIndexTemplate; private final RegularFileProperty releaseNotesTemplate; private final RegularFileProperty releaseHighlightsTemplate; private final RegularFileProperty breakingChangesTemplate; - private final RegularFileProperty migrationIndexTemplate; + private final RegularFileProperty deprecationsTemplate; - private final RegularFileProperty releaseNotesIndexFile; private final RegularFileProperty releaseNotesFile; private final RegularFileProperty releaseHighlightsFile; - private final RegularFileProperty breakingChangesMigrationFile; - private final RegularFileProperty migrationIndexFile; + private final RegularFileProperty breakingChangesFile; + private final RegularFileProperty deprecationsFile; + + private final DirectoryProperty changelogBundleDirectory; private final GitWrapper gitWrapper; @@ -71,17 +68,17 @@ public class GenerateReleaseNotesTask extends DefaultTask { public GenerateReleaseNotesTask(ObjectFactory objectFactory, ExecOperations execOperations) { changelogs = objectFactory.fileCollection(); - releaseNotesIndexTemplate = objectFactory.fileProperty(); releaseNotesTemplate = objectFactory.fileProperty(); releaseHighlightsTemplate = objectFactory.fileProperty(); breakingChangesTemplate = objectFactory.fileProperty(); - migrationIndexTemplate = objectFactory.fileProperty(); + deprecationsTemplate = objectFactory.fileProperty(); - releaseNotesIndexFile = objectFactory.fileProperty(); releaseNotesFile = objectFactory.fileProperty(); releaseHighlightsFile = objectFactory.fileProperty(); - breakingChangesMigrationFile = objectFactory.fileProperty(); - migrationIndexFile = objectFactory.fileProperty(); + breakingChangesFile = objectFactory.fileProperty(); + deprecationsFile = objectFactory.fileProperty(); + + changelogBundleDirectory = objectFactory.directoryProperty(); gitWrapper = new GitWrapper(execOperations); } @@ -94,64 +91,29 @@ public void executeTask() throws IOException { findAndUpdateUpstreamRemote(gitWrapper); } - LOGGER.info("Finding changelog files..."); - - final Map> filesByVersion = partitionFilesByVersion( - gitWrapper, - currentVersion, - this.changelogs.getFiles() - ); - - final List entries = new ArrayList<>(); - final Map> changelogsByVersion = new HashMap<>(); - - filesByVersion.forEach((version, files) -> { - Set entriesForVersion = files.stream().map(ChangelogEntry::parse).collect(toSet()); - entries.addAll(entriesForVersion); - changelogsByVersion.put(version, entriesForVersion); - }); + LOGGER.info("Finding changelog bundles..."); - final Set versions = getVersions(gitWrapper, currentVersion); + List bundles = this.changelogBundleDirectory.getAsFileTree() + .getFiles() + .stream() + .map(ChangelogBundle::parse) + .sorted(Comparator.comparing(ChangelogBundle::released).reversed().thenComparing(ChangelogBundle::generated).reversed()) + .toList(); - LOGGER.info("Updating release notes index..."); - ReleaseNotesIndexGenerator.update( - versions, - this.releaseNotesIndexTemplate.get().getAsFile(), - this.releaseNotesIndexFile.get().getAsFile() - ); + // Ensure that each changelog/PR only shows up once, in its earliest release + var uniquePrs = new HashSet(); + for (int i = bundles.size() - 1; i >= 0; i--) { + var bundle = bundles.get(i); + if (!bundle.released()) { + bundle.changelogs().removeAll(bundle.changelogs().stream().filter(c -> uniquePrs.contains(c.getPr())).toList()); + } + uniquePrs.addAll(bundle.changelogs().stream().map(ChangelogEntry::getPr).toList()); + } LOGGER.info("Generating release notes..."); - final QualifiedVersion qualifiedVersion = QualifiedVersion.of(currentVersion); - ReleaseNotesGenerator.update( - this.releaseNotesTemplate.get().getAsFile(), - this.releaseNotesFile.get().getAsFile(), - qualifiedVersion, - changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) - ); - - // Only update breaking changes and migration guide for new minors - if (qualifiedVersion.revision() == 0) { - LOGGER.info("Generating release highlights..."); - ReleaseHighlightsGenerator.update( - this.releaseHighlightsTemplate.get().getAsFile(), - this.releaseHighlightsFile.get().getAsFile(), - entries - ); - - LOGGER.info("Generating breaking changes / deprecations notes..."); - BreakingChangesGenerator.update( - this.breakingChangesTemplate.get().getAsFile(), - this.breakingChangesMigrationFile.get().getAsFile(), - entries - ); - - LOGGER.info("Updating migration/index..."); - MigrationIndexGenerator.update( - getMinorVersions(versions), - this.migrationIndexTemplate.get().getAsFile(), - this.migrationIndexFile.get().getAsFile() - ); - } + ReleaseNotesGenerator.update(this.releaseNotesTemplate.get().getAsFile(), this.releaseNotesFile.get().getAsFile(), bundles); + ReleaseNotesGenerator.update(this.breakingChangesTemplate.get().getAsFile(), this.breakingChangesFile.get().getAsFile(), bundles); + ReleaseNotesGenerator.update(this.deprecationsTemplate.get().getAsFile(), this.deprecationsFile.get().getAsFile(), bundles); } /** @@ -187,79 +149,6 @@ static Set getMinorVersions(Set versions) { return versions.stream().map(MinorVersion::of).collect(toSet()); } - /** - * Group a set of files by the version in which they first appeared, up until the supplied version. Any files not - * present in an earlier version are assumed to have been introduced in the specified version. - * - *

This method works by finding all git tags prior to {@param versionString} in the same minor series, and - * examining the git tree for that tag. By doing this over each tag, it is possible to see how the contents - * of the changelog directory changed over time. - * - * @param gitWrapper used to call `git` - * @param versionString the "current" version. Does not require a tag in git. - * @param allFilesInCheckout the files to partition - * @return a mapping from version to the files added in that version. - */ - @VisibleForTesting - static Map> partitionFilesByVersion( - GitWrapper gitWrapper, - String versionString, - Set allFilesInCheckout - ) { - if (needsGitTags(versionString) == false) { - return Map.of(QualifiedVersion.of(versionString), allFilesInCheckout); - } - - QualifiedVersion currentVersion = QualifiedVersion.of(versionString); - - // Find all tags for this minor series, using a wildcard tag pattern. - String tagWildcard = String.format(Locale.ROOT, "v%d.%d*", currentVersion.major(), currentVersion.minor()); - - final List earlierVersions = gitWrapper.listVersions(tagWildcard) - // Only keep earlier versions, and if `currentVersion` is a prerelease, then only prereleases too. - .filter( - each -> each.isBefore(currentVersion) - && (currentVersion.isSnapshot() || (currentVersion.hasQualifier() == each.hasQualifier())) - ) - .sorted(naturalOrder()) - .toList(); - - if (earlierVersions.isEmpty()) { - throw new GradleException("Failed to find git tags prior to [v" + currentVersion + "]"); - } - - Map> partitionedFiles = new HashMap<>(); - - Set mutableAllFilesInCheckout = new HashSet<>(allFilesInCheckout); - - // 1. For each earlier version - earlierVersions.forEach(earlierVersion -> { - // 2. Find all the changelog files it contained - Set filesInTreeForVersion = gitWrapper.listFiles("v" + earlierVersion, "docs/changelog") - .map(line -> Path.of(line).getFileName().toString()) - .collect(toSet()); - - Set filesForVersion = new HashSet<>(); - partitionedFiles.put(earlierVersion, filesForVersion); - - // 3. Find the `File` object for each one - final Iterator filesIterator = mutableAllFilesInCheckout.iterator(); - while (filesIterator.hasNext()) { - File nextFile = filesIterator.next(); - if (filesInTreeForVersion.contains(nextFile.getName())) { - // 4. And remove it so that it is associated with the earlier version - filesForVersion.add(nextFile); - filesIterator.remove(); - } - } - }); - - // 5. Associate whatever is left with the current version. - partitionedFiles.put(currentVersion, mutableAllFilesInCheckout); - - return partitionedFiles; - } - /** * Ensure the upstream git remote is up-to-date. The upstream is whatever git remote references `elastic/elasticsearch`. * @param gitWrapper used to call `git` @@ -317,13 +206,13 @@ public void setChangelogs(FileCollection files) { this.changelogs.setFrom(files); } - @InputFile - public RegularFileProperty getReleaseNotesIndexTemplate() { - return releaseNotesIndexTemplate; + @InputDirectory + public DirectoryProperty getChangelogBundleDirectory() { + return changelogBundleDirectory; } - public void setReleaseNotesIndexTemplate(RegularFile file) { - this.releaseNotesIndexTemplate.set(file); + public void setChangelogBundleDirectory(Directory dir) { + this.changelogBundleDirectory.set(dir); } @InputFile @@ -354,21 +243,12 @@ public void setBreakingChangesTemplate(RegularFile file) { } @InputFile - public RegularFileProperty getMigrationIndexTemplate() { - return migrationIndexTemplate; - } - - public void setMigrationIndexTemplate(RegularFile file) { - this.migrationIndexTemplate.set(file); - } - - @OutputFile - public RegularFileProperty getReleaseNotesIndexFile() { - return releaseNotesIndexFile; + public RegularFileProperty getDeprecationsTemplate() { + return deprecationsTemplate; } - public void setReleaseNotesIndexFile(RegularFile file) { - this.releaseNotesIndexFile.set(file); + public void setDeprecationsTemplate(RegularFile file) { + this.deprecationsTemplate.set(file); } @OutputFile @@ -390,20 +270,20 @@ public void setReleaseHighlightsFile(RegularFile file) { } @OutputFile - public RegularFileProperty getBreakingChangesMigrationFile() { - return breakingChangesMigrationFile; + public RegularFileProperty getBreakingChangesFile() { + return breakingChangesFile; } - public void setBreakingChangesMigrationFile(RegularFile file) { - this.breakingChangesMigrationFile.set(file); + public void setBreakingChangesFile(RegularFile file) { + this.breakingChangesFile.set(file); } @OutputFile - public RegularFileProperty getMigrationIndexFile() { - return migrationIndexFile; + public RegularFileProperty getDeprecationsFile() { + return deprecationsFile; } - public void setMigrationIndexFile(RegularFile file) { - this.migrationIndexFile.set(file); + public void setDeprecationsFile(RegularFile file) { + this.deprecationsFile.set(file); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java index f50b3c0890220..6c726d4a16f80 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.release; +import org.gradle.api.GradleException; import org.gradle.process.ExecOperations; import java.io.ByteArrayOutputStream; @@ -87,4 +88,14 @@ public Stream listVersions(String pattern) { public Stream listFiles(String ref, String path) { return runCommand("git", "ls-tree", "--name-only", "-r", ref, path).lines(); } + + public String getUpstream() { + String upstream = listRemotes().entrySet() + .stream() + .filter(entry -> entry.getValue().contains("elastic/elasticsearch")) + .findFirst() + .map(Map.Entry::getKey) + .orElseThrow(() -> new GradleException("Couldn't find a git remote for [elastic/elasticsearch]")); + return upstream; + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java deleted file mode 100644 index 668da0da6d0d0..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import com.google.common.annotations.VisibleForTesting; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; - -import static java.util.Comparator.reverseOrder; - -/** - * This class ensures that the migrate/index page has the appropriate anchors and include directives - * for the current repository version. - */ -public class MigrationIndexGenerator { - - static void update(Set versions, File indexTemplate, File indexFile) throws IOException { - try (FileWriter indexFileWriter = new FileWriter(indexFile)) { - indexFileWriter.write(generateFile(versions, Files.readString(indexTemplate.toPath()))); - } - } - - @VisibleForTesting - static String generateFile(Set versionsSet, String template) throws IOException { - final Set versions = new TreeSet<>(reverseOrder()); - versions.addAll(versionsSet); - final List includeVersions = versions.stream().map(MinorVersion::underscore).collect(Collectors.toList()); - - final Map bindings = new HashMap<>(); - bindings.put("versions", versions); - bindings.put("includeVersions", includeVersions); - - return TemplateUtils.render(template, bindings); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java index 26236779a2390..489b3561150fc 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java @@ -149,8 +149,7 @@ static Stream findPreviousVersion(GitWrapper gitWrapper, Quali final String currentMajorPattern = "v" + version.major() + ".*"; final String previousMajorPattern = "v" + (version.major() - 1) + ".*"; - return Stream.concat(gitWrapper.listVersions(currentMajorPattern), gitWrapper.listVersions(previousMajorPattern)) - .filter(v -> v.isBefore(version)); + return gitWrapper.listVersions(previousMajorPattern).filter(v -> v.isBefore(version)); } /** diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java deleted file mode 100644 index 25e4bec79c353..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import com.google.common.annotations.VisibleForTesting; - -import org.elasticsearch.gradle.VersionProperties; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * Generates the release highlights notes, for changelog files that contain the highlight field. - */ -public class ReleaseHighlightsGenerator { - static void update(File templateFile, File outputFile, List entries) throws IOException { - try (FileWriter output = new FileWriter(outputFile)) { - output.write( - generateFile(QualifiedVersion.of(VersionProperties.getElasticsearch()), Files.readString(templateFile.toPath()), entries) - ); - } - } - - @VisibleForTesting - static String generateFile(QualifiedVersion version, String template, List entries) throws IOException { - final List priorVersions = new ArrayList<>(); - - if (version.minor() > 0) { - final int major = version.major(); - for (int minor = version.minor() - 1; minor >= 0; minor--) { - String majorMinor = major + "." + minor; - priorVersions.add("{ref-bare}/" + majorMinor + "/release-highlights.html[" + majorMinor + "]"); - } - } - - final Map> groupedHighlights = entries.stream() - .map(ChangelogEntry::getHighlight) - .filter(Objects::nonNull) - .sorted(Comparator.comparingInt(ChangelogEntry.Highlight::getPr)) - .collect(Collectors.groupingBy(ChangelogEntry.Highlight::isNotable, Collectors.toList())); - - final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); - final List nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); - - final Map bindings = new HashMap<>(); - bindings.put("priorVersions", priorVersions); - bindings.put("notableHighlights", notableHighlights); - bindings.put("nonNotableHighlights", nonNotableHighlights); - - return TemplateUtils.render(template, bindings); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java index df0e35c487f54..5979c654075c9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java @@ -15,13 +15,16 @@ import java.io.FileWriter; import java.io.IOException; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Objects; import java.util.TreeMap; import static java.util.Comparator.comparing; +import static java.util.Comparator.comparingInt; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; @@ -30,6 +33,17 @@ * type of change, then by team area. */ public class ReleaseNotesGenerator { + + private record ChangelogsBundleWrapper( + QualifiedVersion version, + ChangelogBundle bundle, + Map>> changelogsByTypeByArea, + QualifiedVersion unqualifiedVersion, + String versionWithoutSeparator, + List notableHighlights, + List nonNotableHighlights + ) {} + /** * These mappings translate change types into the headings as they should appear in the release notes. */ @@ -39,40 +53,112 @@ public class ReleaseNotesGenerator { TYPE_LABELS.put("breaking", "Breaking changes"); TYPE_LABELS.put("breaking-java", "Breaking Java changes"); TYPE_LABELS.put("bug", "Bug fixes"); + TYPE_LABELS.put("fixes", "Fixes"); TYPE_LABELS.put("deprecation", "Deprecations"); TYPE_LABELS.put("enhancement", "Enhancements"); TYPE_LABELS.put("feature", "New features"); + TYPE_LABELS.put("features-enhancements", "Features and enhancements"); TYPE_LABELS.put("new-aggregation", "New aggregation"); TYPE_LABELS.put("regression", "Regressions"); TYPE_LABELS.put("upgrade", "Upgrades"); } - static void update(File templateFile, File outputFile, QualifiedVersion version, Set changelogs) throws IOException { + /** + * These are the types of changes that are considered "Features and Enhancements" in the release notes. + */ + private static final List FEATURE_ENHANCEMENT_TYPES = List.of("feature", "new-aggregation", "enhancement", "upgrade"); + + static void update(File templateFile, File outputFile, List bundles) throws IOException { final String templateString = Files.readString(templateFile.toPath()); try (FileWriter output = new FileWriter(outputFile)) { - output.write(generateFile(templateString, version, changelogs)); + output.write(generateFile(templateString, bundles)); } } @VisibleForTesting - static String generateFile(String template, QualifiedVersion version, Set changelogs) throws IOException { - final var changelogsByTypeByArea = buildChangelogBreakdown(changelogs); + static String generateFile(String template, List bundles) throws IOException { + var bundlesWrapped = new ArrayList(); + + for (var bundle : bundles) { + var changelogs = bundle.changelogs(); + final var changelogsByTypeByArea = buildChangelogBreakdown(changelogs); + + final Map> groupedHighlights = changelogs.stream() + .map(ChangelogEntry::getHighlight) + .filter(Objects::nonNull) + .sorted(comparingInt(ChangelogEntry.Highlight::getPr)) + .collect(groupingBy(ChangelogEntry.Highlight::isNotable, toList())); + + final var notableHighlights = groupedHighlights.getOrDefault(true, List.of()); + final var nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); + + final var version = QualifiedVersion.of(bundle.version()); + final var versionWithoutSeparator = version.withoutQualifier().toString().replaceAll("\\.", ""); + + final var wrapped = new ChangelogsBundleWrapper( + version, + bundle, + changelogsByTypeByArea, + version.withoutQualifier(), + versionWithoutSeparator, + notableHighlights, + nonNotableHighlights + ); + + bundlesWrapped.add(wrapped); + } + + // final var changelogsByTypeByArea = buildChangelogBreakdown(changelogs); + // + // final Map> groupedHighlights = changelogs.stream() + // .map(ChangelogEntry::getHighlight) + // .filter(Objects::nonNull) + // .sorted(comparingInt(ChangelogEntry.Highlight::getPr)) + // .collect(groupingBy(ChangelogEntry.Highlight::isNotable, toList())); + // + // final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); + // final List nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); final Map bindings = new HashMap<>(); - bindings.put("version", version); - bindings.put("changelogsByTypeByArea", changelogsByTypeByArea); + // bindings.put("version", version); + // bindings.put("changelogsByTypeByArea", changelogsByTypeByArea); bindings.put("TYPE_LABELS", TYPE_LABELS); + // bindings.put("unqualifiedVersion", version.withoutQualifier()); + // bindings.put("versionWithoutSeparator", version.withoutQualifier().toString().replaceAll("\\.", "")); + // bindings.put("notableHighlights", notableHighlights); + // bindings.put("nonNotableHighlights", nonNotableHighlights); + bindings.put("changelogBundles", bundlesWrapped); return TemplateUtils.render(template, bindings); } - private static Map>> buildChangelogBreakdown(Set changelogs) { + /** + * The new markdown release notes are grouping several of the old change types together. + * This method maps the change type that developers use in the changelogs to the new type that the release notes cares about. + */ + private static String getTypeFromEntry(ChangelogEntry entry) { + if (entry.getBreaking() != null) { + return "breaking"; + } + + if (FEATURE_ENHANCEMENT_TYPES.contains(entry.getType())) { + return "features-enhancements"; + } + + if (entry.getType().equals("bug")) { + return "fixes"; + } + + return entry.getType(); + } + + private static Map>> buildChangelogBreakdown(Collection changelogs) { Map>> changelogsByTypeByArea = changelogs.stream() .collect( groupingBy( // Entries with breaking info are always put in the breaking section - entry -> entry.getBreaking() == null ? entry.getType() : "breaking", + entry -> getTypeFromEntry(entry), TreeMap::new, // Group changelogs for each type by their team area groupingBy( diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java deleted file mode 100644 index b8c195f47d824..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import com.google.common.annotations.VisibleForTesting; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; - -import static java.util.Comparator.reverseOrder; - -/** - * This class ensures that the release notes index page has the appropriate anchors and include directives - * for the current repository version. - */ -public class ReleaseNotesIndexGenerator { - - static void update(Set versions, File indexTemplate, File indexFile) throws IOException { - try (FileWriter indexFileWriter = new FileWriter(indexFile)) { - indexFileWriter.write(generateFile(versions, Files.readString(indexTemplate.toPath()))); - } - } - - @VisibleForTesting - static String generateFile(Set versionsSet, String template) throws IOException { - final Set versions = new TreeSet<>(reverseOrder()); - - // For the purpose of generating the index, snapshot versions are the same as released versions. Prerelease versions are not. - versionsSet.stream().map(v -> v.isSnapshot() ? v.withoutQualifier() : v).forEach(versions::add); - - final List includeVersions = versions.stream().map(QualifiedVersion::toString).collect(Collectors.toList()); - - final Map bindings = new HashMap<>(); - bindings.put("versions", versions); - bindings.put("includeVersions", includeVersions); - - return TemplateUtils.render(template, bindings); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index 22af46a4e5ccd..a696d034be0bf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -55,9 +55,8 @@ public void apply(Project project) { project.getTasks().register("tagVersions", TagVersionsTask.class); project.getTasks().register("setCompatibleVersions", SetCompatibleVersionsTask.class, t -> t.setThisVersion(version)); - final FileTree yamlFiles = projectDirectory.dir("docs/changelog") - .getAsFileTree() - .matching(new PatternSet().include("**/*.yml", "**/*.yaml")); + final Directory changeLogDirectory = projectDirectory.dir("docs/changelog"); + final FileTree yamlFiles = changeLogDirectory.getAsFileTree().matching(new PatternSet().include("**/*.yml", "**/*.yaml")); final Provider validateChangelogsTask = project.getTasks() .register("validateChangelogs", ValidateYamlAgainstSchemaTask.class, task -> { @@ -68,6 +67,15 @@ public void apply(Project project) { task.setReport(new File(project.getBuildDir(), "reports/validateYaml.txt")); }); + final Function> configureBundleTask = shouldConfigureYamlFiles -> task -> { + task.setGroup("Documentation"); + task.setDescription("Generates release notes from changelog files held in this checkout"); + task.setChangelogs(yamlFiles); + task.setChangelogDirectory(changeLogDirectory); + task.setBundleFile(projectDirectory.file("docs/release-notes/changelogs-" + version.toString() + ".yml")); + task.getOutputs().upToDateWhen(o -> false); + }; + final Function> configureGenerateTask = shouldConfigureYamlFiles -> task -> { task.setGroup("Documentation"); if (shouldConfigureYamlFiles) { @@ -77,36 +85,26 @@ public void apply(Project project) { task.setDescription("Generates stub release notes e.g. after feature freeze"); } - task.setReleaseNotesIndexTemplate(projectDirectory.file(RESOURCES + "templates/release-notes-index.asciidoc")); - task.setReleaseNotesIndexFile(projectDirectory.file("docs/reference/release-notes.asciidoc")); - - task.setReleaseNotesTemplate(projectDirectory.file(RESOURCES + "templates/release-notes.asciidoc")); - task.setReleaseNotesFile( - projectDirectory.file( - String.format( - "docs/reference/release-notes/%d.%d.%d.asciidoc", - version.getMajor(), - version.getMinor(), - version.getRevision() - ) - ) - ); + task.setReleaseNotesTemplate(projectDirectory.file(RESOURCES + "templates/index.md")); + task.setReleaseNotesFile(projectDirectory.file("docs/release-notes/index.md")); task.setReleaseHighlightsTemplate(projectDirectory.file(RESOURCES + "templates/release-highlights.asciidoc")); task.setReleaseHighlightsFile(projectDirectory.file("docs/reference/release-notes/highlights.asciidoc")); - task.setBreakingChangesTemplate(projectDirectory.file(RESOURCES + "templates/breaking-changes.asciidoc")); - task.setBreakingChangesMigrationFile( - projectDirectory.file( - String.format("docs/reference/migration/migrate_%d_%d.asciidoc", version.getMajor(), version.getMinor()) - ) - ); - task.setMigrationIndexTemplate(projectDirectory.file(RESOURCES + "templates/migration-index.asciidoc")); - task.setMigrationIndexFile(projectDirectory.file("docs/reference/migration/index.asciidoc")); + task.setBreakingChangesTemplate(projectDirectory.file(RESOURCES + "templates/breaking-changes.md")); + task.setBreakingChangesFile(projectDirectory.file("docs/release-notes/breaking-changes.md")); + + task.setDeprecationsTemplate(projectDirectory.file(RESOURCES + "templates/deprecations.md")); + task.setDeprecationsFile(projectDirectory.file("docs/release-notes/deprecations.md")); + + task.setChangelogBundleDirectory(projectDirectory.dir("docs/release-notes/changelog-bundles")); + + task.getOutputs().upToDateWhen(o -> false); task.dependsOn(validateChangelogsTask); }; + project.getTasks().register("bundleChangelogs", BundleChangelogsTask.class).configure(configureBundleTask.apply(true)); project.getTasks().register("generateReleaseNotes", GenerateReleaseNotesTask.class).configure(configureGenerateTask.apply(true)); project.getTasks() .register("generateStubReleaseNotes", GenerateReleaseNotesTask.class) diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc b/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc deleted file mode 100644 index 855a443a8081f..0000000000000 --- a/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc +++ /dev/null @@ -1,98 +0,0 @@ -[[migrating-${majorDotMinor}]] -== Migrating to ${majorDotMinor} -++++ -${majorDotMinor} -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} ${majorDotMinor}. - -See also <> and <>. -<% if (isElasticsearchSnapshot) { %> -coming::[${majorDotMinorDotRevision}] -<% } %> - -[discrete] -[[breaking-changes-${majorDotMinor}]] -=== Breaking changes -<% if (breakingByNotabilityByArea.isEmpty()) { %> -There are no breaking changes in {es} ${majorDotMinor}. -<% } else { %> -The following changes in {es} ${majorDotMinor} might affect your applications -and prevent them from operating normally. -Before upgrading to ${majorDotMinor}, review these changes and take the described steps -to mitigate the impact. -<% - if (breakingByNotabilityByArea.getOrDefault(true, []).isEmpty()) { %> - -There are no notable breaking changes in {es} ${majorDotMinor}. -But there are some less critical breaking changes. -<% } - [true, false].each { isNotable -> - def breakingByArea = breakingByNotabilityByArea.getOrDefault(isNotable, []) - if (breakingByArea.isEmpty() == false) { - breakingByArea.eachWithIndex { area, breakingChanges, i -> - print "\n[discrete]\n" - print "[[breaking_${majorMinor}_${ area.toLowerCase().replaceAll("[^a-z0-9]+", "_") }_changes]]\n" - print "==== ${area} changes\n" - - for (breaking in breakingChanges) { %> -[[${ breaking.anchor }]] -.${breaking.title} -[%collapsible] -==== -*Details* + -${breaking.details.trim()} - -*Impact* + -${breaking.impact.trim()} -==== -<% - } - } - - } - } -} - -if (deprecationsByNotabilityByArea.isEmpty() == false) { %> - -[discrete] -[[deprecated-${majorDotMinor}]] -=== Deprecations - -The following functionality has been deprecated in {es} ${majorDotMinor} -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to ${majorDotMinor}. - -To find out if you are using any deprecated functionality, -enable <>. -<% - [true, false].each { isNotable -> - def deprecationsByArea = deprecationsByNotabilityByArea.getOrDefault(isNotable, []) - if (deprecationsByArea.isEmpty() == false) { - deprecationsByArea.eachWithIndex { area, deprecations, i -> - print "\n[discrete]\n" - print "[[deprecations_${majorMinor}_${ area.toLowerCase().replaceAll("[^a-z0-9]+", "_") }]]\n" - print "==== ${area} deprecations\n" - - for (deprecation in deprecations) { %> -[[${ deprecation.anchor }]] -.${deprecation.title} -[%collapsible] -==== -*Details* + -${deprecation.details.trim()} - -*Impact* + -${deprecation.impact.trim()} -==== -<% - } - } - - } - } -} %> diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.md b/build-tools-internal/src/main/resources/templates/breaking-changes.md new file mode 100644 index 0000000000000..a4cbd8ce7c164 --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.md @@ -0,0 +1,50 @@ +--- +navigation_title: "Breaking changes" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes.html +--- + +# Elasticsearch breaking changes [elasticsearch-breaking-changes] + +Breaking changes can impact your Elastic applications, potentially disrupting normal operations. Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. + +If you are migrating from a version prior to version 9.0, you must first upgrade to the last 8.x version available. To learn how to upgrade, check out [Upgrade](docs-content://deploy-manage/upgrade.md). + +% ## Next version [elasticsearch-nextversion-breaking-changes] +<% +for(bundle in changelogBundles) { + def version = bundle.version + def versionForIds = bundle.version.toString().equals('9.0.0') ? bundle.versionWithoutSeparator : bundle.version + def changelogsByTypeByArea = bundle.changelogsByTypeByArea + def unqualifiedVersion = bundle.unqualifiedVersion + def coming = !bundle.bundle.released + + if (coming) { + print "\n" + print "```{applies_to}\n" + print "stack: coming ${version}\n" + print "```" + } +%> +## ${unqualifiedVersion} [elasticsearch-${versionForIds}-breaking-changes] +<% + if (!changelogsByTypeByArea['breaking']) { + print "\nNo breaking changes in this version.\n" + } else { + for (team in (changelogsByTypeByArea['breaking'] ?: [:]).keySet()) { + print "\n${team}:\n"; + + for (change in changelogsByTypeByArea['breaking'][team]) { + print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" + if (change.issues != null && change.issues.empty == false) { + print change.issues.size() == 1 ? " (issue: " : " (issues: " + print change.issues.collect { "[#${it}](https://github.com/elastic/elasticsearch/issues/${it})" }.join(", ") + print ")" + } + print "\n" + } + } + + print "\n\n" + } +} diff --git a/build-tools-internal/src/main/resources/templates/deprecations.md b/build-tools-internal/src/main/resources/templates/deprecations.md new file mode 100644 index 0000000000000..38c58682a999b --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/deprecations.md @@ -0,0 +1,53 @@ +--- +navigation_title: "Deprecations" +--- + +# {{es}} deprecations [elasticsearch-deprecations] + +Over time, certain Elastic functionality becomes outdated and is replaced or removed. To help with the transition, Elastic deprecates functionality for a period before removal, giving you time to update your applications. + +Review the deprecated functionality for Elasticsearch. While deprecations have no immediate impact, we strongly encourage you update your implementation after you upgrade. To learn how to upgrade, check out [Upgrade](docs-content://deploy-manage/upgrade.md). + +To give you insight into what deprecated features you’re using, {{es}}: + +* Returns a `Warn` HTTP header whenever you submit a request that uses deprecated functionality. +* [Logs deprecation warnings](docs-content://deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md#deprecation-logging) when deprecated functionality is used. +* [Provides a deprecation info API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-migration-deprecations) that scans a cluster’s configuration and mappings for deprecated functionality. + +% ## Next version [elasticsearch-nextversion-deprecations] +<% +for(bundle in changelogBundles) { + def version = bundle.version + def versionForIds = bundle.version.toString().equals('9.0.0') ? bundle.versionWithoutSeparator : bundle.version + def changelogsByTypeByArea = bundle.changelogsByTypeByArea + def unqualifiedVersion = bundle.unqualifiedVersion + def coming = !bundle.bundle.released + + if (coming) { + print "\n" + print "```{applies_to}\n" + print "stack: coming ${version}\n" + print "```" + } +%> +## ${unqualifiedVersion} [elasticsearch-${versionForIds}-deprecations] +<% + if (!changelogsByTypeByArea['deprecation']) { + print "\nNo deprecations in this version.\n" + } else { + for (team in (changelogsByTypeByArea['deprecation'] ?: [:]).keySet()) { + print "\n${team}:\n"; + + for (change in changelogsByTypeByArea['deprecation'][team]) { + print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" + if (change.issues != null && change.issues.empty == false) { + print change.issues.size() == 1 ? " (issue: " : " (issues: " + print change.issues.collect { "[#${it}](https://github.com/elastic/elasticsearch/issues/${it})" }.join(", ") + print ")" + } + print "\n" + } + } + print "\n\n" + } +} diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md new file mode 100644 index 0000000000000..cb417eb49dc64 --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -0,0 +1,76 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html +--- + +# Elasticsearch release notes [elasticsearch-release-notes] + +Review the changes, fixes, and more in each version of Elasticsearch. + +To check for security updates, go to [Security announcements for the Elastic stack](https://discuss.elastic.co/c/announcements/security-announcements/31). + +% Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. + +% ## version.next [elasticsearch-next-release-notes] + +% ### Features and enhancements [elasticsearch-next-features-enhancements] +% * + +% ### Fixes [elasticsearch-next-fixes] +% * +<% +for(bundle in changelogBundles) { + def version = bundle.version + def versionForIds = bundle.version.toString().equals('9.0.0') ? bundle.versionWithoutSeparator : bundle.version + def changelogsByTypeByArea = bundle.changelogsByTypeByArea + def notableHighlights = bundle.notableHighlights + def nonNotableHighlights = bundle.nonNotableHighlights + def unqualifiedVersion = bundle.unqualifiedVersion + def coming = !bundle.bundle.released + + if (coming) { + print "\n" + print "```{applies_to}\n" + print "stack: coming ${version}\n" + print "```" + } +%> +## ${unqualifiedVersion} [elasticsearch-${versionForIds}-release-notes] +<% +if (!notableHighlights.isEmpty() || !nonNotableHighlights.isEmpty()) { + print "\n### Highlights [elasticsearch-${versionForIds}-highlights]\n" +} + +for (highlights in [notableHighlights, nonNotableHighlights]) { + if (!highlights.isEmpty()) { + for (highlight in highlights) { %> +::::{dropdown} ${highlight.title} +${highlight.body.trim()} +:::: +<% } + } +} + +for (changeType in ['features-enhancements', 'fixes', 'regression']) { + if (changelogsByTypeByArea[changeType] == null || changelogsByTypeByArea[changeType].empty) { + continue; + } +%> +### ${ TYPE_LABELS.getOrDefault(changeType, 'No mapping for TYPE_LABELS[' + changeType + ']') } [elasticsearch-${versionForIds}-${changeType}] +<% for (team in changelogsByTypeByArea[changeType].keySet()) { + print "\n${team}:\n"; + + for (change in changelogsByTypeByArea[changeType][team]) { + print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" + if (change.issues != null && change.issues.empty == false) { + print change.issues.size() == 1 ? " (issue: " : " (issues: " + print change.issues.collect { "[#${it}](https://github.com/elastic/elasticsearch/issues/${it})" }.join(", ") + print ")" + } + print "\n" + } +} +} +print "\n" +} diff --git a/build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc b/build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc deleted file mode 100644 index ba30aea3bf14e..0000000000000 --- a/build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc +++ /dev/null @@ -1,12 +0,0 @@ -[[es-release-notes]] -= Release notes - -[partintro] --- - -This section summarizes the changes in each release. - -<% versions.each { print "* <>\n" } %> --- - -<% includeVersions.each { print "include::release-notes/${ it }.asciidoc[]\n" } %> diff --git a/build-tools-internal/src/main/resources/templates/release-notes.asciidoc b/build-tools-internal/src/main/resources/templates/release-notes.asciidoc deleted file mode 100644 index 096608435333a..0000000000000 --- a/build-tools-internal/src/main/resources/templates/release-notes.asciidoc +++ /dev/null @@ -1,45 +0,0 @@ -<% -def unqualifiedVersion = version.withoutQualifier() -%>[[release-notes-$unqualifiedVersion]] -== {es} version ${unqualifiedVersion} -<% if (version.isSnapshot()) { %> -coming[$unqualifiedVersion] -<% } %> -Also see <>. -<% if (changelogsByTypeByArea["security"] != null) { %> -[discrete] -[[security-updates-${unqualifiedVersion}]] -=== Security updates - -<% for (change in changelogsByTypeByArea.remove("security").remove("_all_")) { - print "* ${change.summary}\n" -} -} -if (changelogsByTypeByArea["known-issue"] != null) { %> -[discrete] -[[known-issues-${unqualifiedVersion}]] -=== Known issues - -<% for (change in changelogsByTypeByArea.remove("known-issue").remove("_all_")) { - print "* ${change.summary}\n" -} -} -for (changeType in changelogsByTypeByArea.keySet()) { %> -[[${ changeType }-${ unqualifiedVersion }]] -[float] -=== ${ TYPE_LABELS.getOrDefault(changeType, 'No mapping for TYPE_LABELS[' + changeType + ']') } -<% for (team in changelogsByTypeByArea[changeType].keySet()) { - print "\n${team}::\n"; - - for (change in changelogsByTypeByArea[changeType][team]) { - print "* ${change.summary} {es-pull}${change.pr}[#${change.pr}]" - if (change.issues != null && change.issues.empty == false) { - print change.issues.size() == 1 ? " (issue: " : " (issues: " - print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") - print ")" - } - print "\n" - } -} -} -print "\n\n" diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java deleted file mode 100644 index 5d51f3514cbc3..0000000000000 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import java.util.Objects; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -public class BreakingChangesGeneratorTest { - - /** - * Check that the breaking changes can be correctly generated. - */ - @Test - public void generateIndexFile_rendersCorrectMarkup() throws Exception { - // given: - final String template = getResource("/templates/breaking-changes.asciidoc"); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc" - ); - - final List entries = getEntries(); - - // when: - final String actualOutput = BreakingChangesGenerator.generateMigrationFile( - QualifiedVersion.of("8.4.0-SNAPSHOT"), - template, - entries - ); - - // then: - assertThat(actualOutput, equalTo(expectedOutput)); - } - - private List getEntries() { - ChangelogEntry entry1 = new ChangelogEntry(); - ChangelogEntry.Breaking breaking1 = new ChangelogEntry.Breaking(); - entry1.setBreaking(breaking1); - - breaking1.setNotable(true); - breaking1.setTitle("Breaking change number 1"); - breaking1.setArea("API"); - breaking1.setDetails("Breaking change details 1"); - breaking1.setImpact("Breaking change impact description 1"); - - ChangelogEntry entry2 = new ChangelogEntry(); - ChangelogEntry.Breaking breaking2 = new ChangelogEntry.Breaking(); - entry2.setBreaking(breaking2); - - breaking2.setNotable(true); - breaking2.setTitle("Breaking change number 2"); - breaking2.setArea("Cluster and node setting"); - breaking2.setDetails("Breaking change details 2"); - breaking2.setImpact("Breaking change impact description 2"); - - ChangelogEntry entry3 = new ChangelogEntry(); - ChangelogEntry.Breaking breaking3 = new ChangelogEntry.Breaking(); - entry3.setBreaking(breaking3); - - breaking3.setNotable(false); - breaking3.setTitle("Breaking change number 3"); - breaking3.setArea("Transform"); - breaking3.setDetails("Breaking change details 3"); - breaking3.setImpact("Breaking change impact description 3"); - - ChangelogEntry entry4 = new ChangelogEntry(); - ChangelogEntry.Breaking breaking4 = new ChangelogEntry.Breaking(); - entry4.setBreaking(breaking4); - - breaking4.setNotable(true); - breaking4.setTitle("Breaking change number 4"); - breaking4.setArea("Cluster and node setting"); - breaking4.setDetails("Breaking change details 4"); - breaking4.setImpact("Breaking change impact description 4"); - breaking4.setEssSettingChange(true); - - ChangelogEntry entry5 = new ChangelogEntry(); - ChangelogEntry.Deprecation deprecation5 = new ChangelogEntry.Deprecation(); - entry5.setDeprecation(deprecation5); - - deprecation5.setNotable(true); - deprecation5.setTitle("Deprecation change number 5"); - deprecation5.setArea("Cluster and node setting"); - deprecation5.setDetails("Deprecation change details 5"); - deprecation5.setImpact("Deprecation change impact description 5"); - deprecation5.setEssSettingChange(false); - - ChangelogEntry entry6 = new ChangelogEntry(); - ChangelogEntry.Deprecation deprecation6 = new ChangelogEntry.Deprecation(); - entry6.setDeprecation(deprecation6); - - deprecation6.setNotable(true); - deprecation6.setTitle("Deprecation change number 6"); - deprecation6.setArea("Cluster and node setting"); - deprecation6.setDetails("Deprecation change details 6"); - deprecation6.setImpact("Deprecation change impact description 6"); - deprecation6.setEssSettingChange(false); - - ChangelogEntry entry7 = new ChangelogEntry(); - ChangelogEntry.Deprecation deprecation7 = new ChangelogEntry.Deprecation(); - entry7.setDeprecation(deprecation7); - - deprecation7.setNotable(false); - deprecation7.setTitle("Deprecation change number 7"); - deprecation7.setArea("Cluster and node setting"); - deprecation7.setDetails("Deprecation change details 7"); - deprecation7.setImpact("Deprecation change impact description 7"); - deprecation7.setEssSettingChange(false); - - return List.of(entry1, entry2, entry3, entry4, entry5, entry6, entry7); - } - - private String getResource(String name) throws Exception { - return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8); - } -} diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java deleted file mode 100644 index 4e8320d8c0b65..0000000000000 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import java.util.Objects; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; - -public class ReleaseHighlightsGeneratorTest { - - /** - * Check that the release highlights can be correctly generated when there are no highlights. - */ - @Test - public void generateFile_withNoHighlights_rendersCorrectMarkup() throws Exception { - // given: - final String template = getResource("/templates/release-highlights.asciidoc"); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.noHighlights.generateFile.asciidoc" - ); - - // when: - final String actualOutput = ReleaseHighlightsGenerator.generateFile(QualifiedVersion.of("8.4.0-SNAPSHOT"), template, List.of()); - - // then: - assertThat(actualOutput, equalTo(expectedOutput)); - } - - /** - * Check that the release highlights can be correctly generated. - */ - @Test - public void generateFile_rendersCorrectMarkup() throws Exception { - // given: - final String template = getResource("/templates/release-highlights.asciidoc"); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc" - ); - - final List entries = getEntries(); - - // when: - final String actualOutput = ReleaseHighlightsGenerator.generateFile(QualifiedVersion.of("8.4.0-SNAPSHOT"), template, entries); - - // then: - assertThat(actualOutput, equalTo(expectedOutput)); - } - - private List getEntries() { - ChangelogEntry entry123 = makeChangelogEntry(123, true); - ChangelogEntry entry456 = makeChangelogEntry(456, true); - ChangelogEntry entry789 = makeChangelogEntry(789, false); - // Return unordered list, to test correct re-ordering - return List.of(entry456, entry123, entry789); - } - - private ChangelogEntry makeChangelogEntry(int pr, boolean notable) { - ChangelogEntry entry = new ChangelogEntry(); - entry.setPr(pr); - ChangelogEntry.Highlight highlight = new ChangelogEntry.Highlight(); - entry.setHighlight(highlight); - - highlight.setNotable(notable); - highlight.setTitle("Notable release highlight number " + pr); - highlight.setBody("Notable release body number " + pr); - - return entry; - } - - private String getResource(String name) throws Exception { - return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8); - } -} diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java index cbb53698c586b..96b8a32a0745f 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java @@ -19,24 +19,73 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThat; public class ReleaseNotesGeneratorTest { - /** - * Check that the release notes can be correctly generated. - */ + private static final List CHANGE_TYPES = List.of( + "breaking", + "breaking-java", + "bug", + "fixes", + "deprecation", + "enhancement", + "feature", + "features-enhancements", + "new-aggregation", + "regression", + "upgrade" + ); + + @Test + public void generateFile_index_rendersCorrectMarkup() throws Exception { + testTemplate("index.md"); + } + + @Test + public void generateFile_index_noHighlights_rendersCorrectMarkup() throws Exception { + Set entries = getEntries(); + entries = entries.stream().filter(e -> e.getHighlight() == null).collect(Collectors.toSet()); + + testTemplate("index.md", "index.no-highlights.md", entries); + } + + @Test + public void generateFile_index_noChanges_rendersCorrectMarkup() throws Exception { + Set entries = new HashSet<>(); + + testTemplate("index.md", "index.no-changes.md", entries); + } + + @Test + public void generateFile_breakingChanges_rendersCorrectMarkup() throws Exception { + testTemplate("breaking-changes.md"); + } + @Test - public void generateFile_rendersCorrectMarkup() throws Exception { + public void generateFile_deprecations_rendersCorrectMarkup() throws Exception { + testTemplate("deprecations.md"); + } + + public void testTemplate(String templateFilename) throws Exception { + testTemplate(templateFilename, templateFilename, null); + } + + public void testTemplate(String templateFilename, String outputFilename) throws Exception { + testTemplate(templateFilename, outputFilename, null); + } + + public void testTemplate(String templateFilename, String outputFilename, Set entries) throws Exception { // given: - final String template = getResource("/templates/release-notes.asciidoc"); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.generateFile.asciidoc" - ); + final String template = getResource("/templates/" + templateFilename); + final String expectedOutput = getResource("/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest." + outputFilename); - final Set entries = getEntries(); + if (entries == null) { + entries = getEntries(); + } // when: final String actualOutput = ReleaseNotesGenerator.generateFile(template, QualifiedVersion.of("8.2.0-SNAPSHOT"), entries); @@ -47,23 +96,13 @@ public void generateFile_rendersCorrectMarkup() throws Exception { private Set getEntries() { final Set entries = new HashSet<>(); - entries.addAll(buildEntries(1, 2)); - entries.addAll(buildEntries(2, 2)); - entries.addAll(buildEntries(3, 2)); - - // Security issues are presented first in the notes - final ChangelogEntry securityEntry = new ChangelogEntry(); - securityEntry.setArea("Security"); - securityEntry.setType("security"); - securityEntry.setSummary("Test security issue"); - entries.add(securityEntry); - - // known issues are presented after security issues - final ChangelogEntry knownIssue = new ChangelogEntry(); - knownIssue.setArea("Search"); - knownIssue.setType("known-issue"); - knownIssue.setSummary("Test known issue"); - entries.add(knownIssue); + for (int i = 0; i < CHANGE_TYPES.size(); i++) { + entries.addAll(buildEntries(i, 2)); + } + + entries.add(makeHighlightsEntry(5001, false)); + entries.add(makeHighlightsEntry(5000, true)); + entries.add(makeHighlightsEntry(5002, true)); return entries; } @@ -71,11 +110,9 @@ private Set getEntries() { private List buildEntries(int seed, int count) { // Sample of possible areas from `changelog-schema.json` final List areas = List.of("Aggregation", "Cluster", "Indices", "Mappings", "Search", "Security"); - // Possible change types, with `breaking`, `breaking-java`, `known-issue` and `security` removed. - final List types = List.of("bug", "deprecation", "enhancement", "feature", "new-aggregation", "regression", "upgrade"); final String area = areas.get(seed % areas.size()); - final String type = types.get(seed % types.size()); + final String type = CHANGE_TYPES.get(seed % CHANGE_TYPES.size()); final List entries = new ArrayList<>(count); @@ -101,6 +138,30 @@ private List buildEntries(int seed, int count) { return entries; } + private List getHighlightsEntries() { + ChangelogEntry entry123 = makeHighlightsEntry(123, true); + ChangelogEntry entry456 = makeHighlightsEntry(456, true); + ChangelogEntry entry789 = makeHighlightsEntry(789, false); + // Return unordered list, to test correct re-ordering + return List.of(entry456, entry123, entry789); + } + + private ChangelogEntry makeHighlightsEntry(int pr, boolean notable) { + ChangelogEntry entry = new ChangelogEntry(); + entry.setPr(pr); + ChangelogEntry.Highlight highlight = new ChangelogEntry.Highlight(); + entry.setHighlight(highlight); + + highlight.setNotable(notable); + highlight.setTitle((notable ? "[Notable] " : "") + "Release highlight number " + pr); + highlight.setBody("Release highlight body number " + pr); + entry.setType("feature"); + entry.setArea("Search"); + entry.setSummary(""); + + return entry; + } + private String getResource(String name) throws Exception { return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8); } diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java deleted file mode 100644 index 2a32040035ec7..0000000000000 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; - -public class ReleaseNotesIndexGeneratorTest { - - /** - * Check that a release notes index can be generated. - */ - @Test - public void generateFile_rendersCorrectMarkup() throws Exception { - // given: - final Set versions = Stream.of( - "8.0.0-alpha1", - "8.0.0-beta2", - "8.0.0-rc3", - "8.0.0", - "8.0.1", - "8.0.2", - "8.1.0", - "8.1.1", - "8.2.0-SNAPSHOT" - ).map(QualifiedVersion::of).collect(Collectors.toSet()); - - final String template = getResource("/templates/release-notes-index.asciidoc"); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.generateFile.asciidoc" - ); - - // when: - final String actualOutput = ReleaseNotesIndexGenerator.generateFile(versions, template); - - // then: - assertThat(actualOutput, equalTo(expectedOutput)); - } - - private String getResource(String name) throws Exception { - return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8); - } -} diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc deleted file mode 100644 index ee78d10fd4b07..0000000000000 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc +++ /dev/null @@ -1,134 +0,0 @@ -[[migrating-8.4]] -== Migrating to 8.4 -++++ -8.4 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.4. - -See also <> and <>. - -coming::[8.4.0] - - -[discrete] -[[breaking-changes-8.4]] -=== Breaking changes - -The following changes in {es} 8.4 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.4, review these changes and take the described steps -to mitigate the impact. - -[discrete] -[[breaking_84_api_changes]] -==== API changes - -[[breaking_change_number_1]] -.Breaking change number 1 -[%collapsible] -==== -*Details* + -Breaking change details 1 - -*Impact* + -Breaking change impact description 1 -==== - -[discrete] -[[breaking_84_cluster_and_node_setting_changes]] -==== Cluster and node setting changes - -[[breaking_change_number_2]] -.Breaking change number 2 -[%collapsible] -==== -*Details* + -Breaking change details 2 - -*Impact* + -Breaking change impact description 2 -==== - -[[breaking_change_number_4]] -.Breaking change number 4 -[%collapsible] -==== -*Details* + -Breaking change details 4 - -*Impact* + -Breaking change impact description 4 -==== - -[discrete] -[[breaking_84_transform_changes]] -==== Transform changes - -[[breaking_change_number_3]] -.Breaking change number 3 -[%collapsible] -==== -*Details* + -Breaking change details 3 - -*Impact* + -Breaking change impact description 3 -==== - - -[discrete] -[[deprecated-8.4]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.4 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.4. - -To find out if you are using any deprecated functionality, -enable <>. - -[discrete] -[[deprecations_84_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[deprecation_change_number_5]] -.Deprecation change number 5 -[%collapsible] -==== -*Details* + -Deprecation change details 5 - -*Impact* + -Deprecation change impact description 5 -==== - -[[deprecation_change_number_6]] -.Deprecation change number 6 -[%collapsible] -==== -*Details* + -Deprecation change details 6 - -*Impact* + -Deprecation change impact description 6 -==== - -[discrete] -[[deprecations_84_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[deprecation_change_number_7]] -.Deprecation change number 7 -[%collapsible] -==== -*Details* + -Deprecation change details 7 - -*Impact* + -Deprecation change impact description 7 -==== - diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md new file mode 100644 index 0000000000000..038521f311242 --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md @@ -0,0 +1,22 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes.html +--- + +# Elasticsearch breaking changes [elasticsearch-breaking-changes] +Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. + +To learn how to upgrade, check out . + +% ## Next version [elasticsearch-nextversion-breaking-changes] +% **Release date:** Month day, year + +## 8.2.0 [elasticsearch-820-breaking-changes] +**Release date:** April 01, 2025 + +Aggregation: +* Test changelog entry 0_0 [#0](https://github.com/elastic/elasticsearch/pull/0) (issue: {es-issue}1[#1]) +* Test changelog entry 0_1 [#2](https://github.com/elastic/elasticsearch/pull/2) (issues: {es-issue}3[#3], {es-issue}4[#4]) + + diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md new file mode 100644 index 0000000000000..6d36fc53181e7 --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md @@ -0,0 +1,26 @@ +--- +navigation_title: "Elasticsearch" +--- + +# {{es}} deprecations [elasticsearch-deprecations] +Review the deprecated functionality for your {{es}} version. While deprecations have no immediate impact, we strongly encourage you update your implementation after you upgrade. + +To learn how to upgrade, check out . + +To give you insight into what deprecated features you’re using, {{es}}: + +* Returns a `Warn` HTTP header whenever you submit a request that uses deprecated functionality. +* [Logs deprecation warnings](docs-content://deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md#deprecation-logging) when deprecated functionality is used. +* [Provides a deprecation info API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-migration-deprecations) that scans a cluster’s configuration and mappings for deprecated functionality. + +% ## Next version [elasticsearch-nextversion-deprecations] +% **Release date:** Month day, year + +## 8.2.0 [elasticsearch-820-deprecations] +**Release date:** April 01, 2025 + +Search: +* Test changelog entry 4_0 [#4000](https://github.com/elastic/elasticsearch/pull/4000) (issue: {es-issue}4001[#4001]) +* Test changelog entry 4_1 [#4002](https://github.com/elastic/elasticsearch/pull/4002) (issues: {es-issue}4003[#4003], {es-issue}4004[#4004]) + + diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.generateFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.generateFile.asciidoc deleted file mode 100644 index 77b00f9ba9852..0000000000000 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.generateFile.asciidoc +++ /dev/null @@ -1,44 +0,0 @@ -[[release-notes-8.2.0]] -== {es} version 8.2.0 - -coming[8.2.0] - -Also see <>. - -[discrete] -[[security-updates-8.2.0]] -=== Security updates - -* Test security issue - -[discrete] -[[known-issues-8.2.0]] -=== Known issues - -* Test known issue - -[[deprecation-8.2.0]] -[float] -=== Deprecations - -Cluster:: -* Test changelog entry 1_0 {es-pull}1000[#1000] (issue: {es-issue}1001[#1001]) -* Test changelog entry 1_1 {es-pull}1002[#1002] (issues: {es-issue}1003[#1003], {es-issue}1004[#1004]) - -[[enhancement-8.2.0]] -[float] -=== Enhancements - -Indices:: -* Test changelog entry 2_0 {es-pull}2000[#2000] (issue: {es-issue}2001[#2001]) -* Test changelog entry 2_1 {es-pull}2002[#2002] (issues: {es-issue}2003[#2003], {es-issue}2004[#2004]) - -[[feature-8.2.0]] -[float] -=== New features - -Mappings:: -* Test changelog entry 3_0 {es-pull}3000[#3000] (issue: {es-issue}3001[#3001]) -* Test changelog entry 3_1 {es-pull}3002[#3002] (issues: {es-issue}3003[#3003], {es-issue}3004[#3004]) - - diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md new file mode 100644 index 0000000000000..dc4320f684535 --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md @@ -0,0 +1,91 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-8.2.0.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.2.html +--- + +# Elasticsearch release notes [elasticsearch-release-notes] + +Review the changes, fixes, and more in each version of Elasticsearch. + +To check for security updates, go to [Security announcements for the Elastic stack](https://discuss.elastic.co/c/announcements/security-announcements/31). + +% Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. + +% ## version.next [elasticsearch-next-release-notes] +% **Release date:** Month day, year + +% ### Features and enhancements [elasticsearch-next-features-enhancements] +% * + +% ### Fixes [elasticsearch-next-fixes] +% * + +## 8.2.0 [elasticsearch-820-release-notes] +**Release date:** April 01, 2025 + +### Highlights [elasticsearch-820-highlights] + +::::{dropdown} [Notable] Release highlight number 5000 +Release highlight body number 5000 + +For more information, check [PR #5000](https://github.com/elastic/elasticsearch/pull/5000). +:::: + +::::{dropdown} [Notable] Release highlight number 5002 +Release highlight body number 5002 + +For more information, check [PR #5002](https://github.com/elastic/elasticsearch/pull/5002). +:::: + +::::{dropdown} Release highlight number 5001 +Release highlight body number 5001 + +For more information, check [PR #5001](https://github.com/elastic/elasticsearch/pull/5001). +:::: + +### Features and enhancements [elasticsearch-820-features-enhancements] + +Aggregation: +* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: [#6000](https://github.com/elastic/elasticsearch/pull/6000)) +* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: [#6002](https://github.com/elastic/elasticsearch/pull/6002), [#6002](https://github.com/elastic/elasticsearch/pull/6002)) + +Cluster: +* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: [#7000](https://github.com/elastic/elasticsearch/pull/7000)) +* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: [#7002](https://github.com/elastic/elasticsearch/pull/7002), [#7002](https://github.com/elastic/elasticsearch/pull/7002)) + +Indices: +* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: [#8000](https://github.com/elastic/elasticsearch/pull/8000)) +* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: [#8002](https://github.com/elastic/elasticsearch/pull/8002), [#8002](https://github.com/elastic/elasticsearch/pull/8002)) + +Search: +* [#5002](https://github.com/elastic/elasticsearch/pull/5002) +* [#5000](https://github.com/elastic/elasticsearch/pull/5000) +* [#5001](https://github.com/elastic/elasticsearch/pull/5001) +* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: [#10000](https://github.com/elastic/elasticsearch/pull/10000)) +* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: [#10002](https://github.com/elastic/elasticsearch/pull/10002), [#10002](https://github.com/elastic/elasticsearch/pull/10002)) + +Security: +* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: [#5000](https://github.com/elastic/elasticsearch/pull/5000)) +* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: [#5002](https://github.com/elastic/elasticsearch/pull/5002), [#5002](https://github.com/elastic/elasticsearch/pull/5002)) + +### Fixes [elasticsearch-820-fixes] + +Indices: +* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: [#2000](https://github.com/elastic/elasticsearch/pull/2000)) +* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: [#2002](https://github.com/elastic/elasticsearch/pull/2002), [#2002](https://github.com/elastic/elasticsearch/pull/2002)) + +Mappings: +* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: [#3000](https://github.com/elastic/elasticsearch/pull/3000)) +* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: [#3002](https://github.com/elastic/elasticsearch/pull/3002), [#3002](https://github.com/elastic/elasticsearch/pull/3002)) + +### Regressions [elasticsearch-820-regression] + +Mappings: +* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: [#9000](https://github.com/elastic/elasticsearch/pull/9000)) +* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: [#9002](https://github.com/elastic/elasticsearch/pull/9002), [#9002](https://github.com/elastic/elasticsearch/pull/9002)) + + diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md new file mode 100644 index 0000000000000..ea2b62653ae8b --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md @@ -0,0 +1,30 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-8.2.0.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.2.html +--- + +# Elasticsearch release notes [elasticsearch-release-notes] + +Review the changes, fixes, and more in each version of Elasticsearch. + +To check for security updates, go to [Security announcements for the Elastic stack](https://discuss.elastic.co/c/announcements/security-announcements/31). + +% Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. + +% ## version.next [elasticsearch-next-release-notes] +% **Release date:** Month day, year + +% ### Features and enhancements [elasticsearch-next-features-enhancements] +% * + +% ### Fixes [elasticsearch-next-fixes] +% * + +## 8.2.0 [elasticsearch-820-release-notes] +**Release date:** April 01, 2025 + + diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md new file mode 100644 index 0000000000000..831c21251bbd1 --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md @@ -0,0 +1,68 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-8.2.0.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.2.html +--- + +# Elasticsearch release notes [elasticsearch-release-notes] + +Review the changes, fixes, and more in each version of Elasticsearch. + +To check for security updates, go to [Security announcements for the Elastic stack](https://discuss.elastic.co/c/announcements/security-announcements/31). + +% Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. + +% ## version.next [elasticsearch-next-release-notes] +% **Release date:** Month day, year + +% ### Features and enhancements [elasticsearch-next-features-enhancements] +% * + +% ### Fixes [elasticsearch-next-fixes] +% * + +## 8.2.0 [elasticsearch-820-release-notes] +**Release date:** April 01, 2025 + +### Features and enhancements [elasticsearch-820-features-enhancements] + +Aggregation: +* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: [#6000](https://github.com/elastic/elasticsearch/pull/6000)) +* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: [#6002](https://github.com/elastic/elasticsearch/pull/6002), [#6002](https://github.com/elastic/elasticsearch/pull/6002)) + +Cluster: +* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: [#7000](https://github.com/elastic/elasticsearch/pull/7000)) +* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: [#7002](https://github.com/elastic/elasticsearch/pull/7002), [#7002](https://github.com/elastic/elasticsearch/pull/7002)) + +Indices: +* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: [#8000](https://github.com/elastic/elasticsearch/pull/8000)) +* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: [#8002](https://github.com/elastic/elasticsearch/pull/8002), [#8002](https://github.com/elastic/elasticsearch/pull/8002)) + +Search: +* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: [#10000](https://github.com/elastic/elasticsearch/pull/10000)) +* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: [#10002](https://github.com/elastic/elasticsearch/pull/10002), [#10002](https://github.com/elastic/elasticsearch/pull/10002)) + +Security: +* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: [#5000](https://github.com/elastic/elasticsearch/pull/5000)) +* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: [#5002](https://github.com/elastic/elasticsearch/pull/5002), [#5002](https://github.com/elastic/elasticsearch/pull/5002)) + +### Fixes [elasticsearch-820-fixes] + +Indices: +* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: [#2000](https://github.com/elastic/elasticsearch/pull/2000)) +* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: [#2002](https://github.com/elastic/elasticsearch/pull/2002), [#2002](https://github.com/elastic/elasticsearch/pull/2002)) + +Mappings: +* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: [#3000](https://github.com/elastic/elasticsearch/pull/3000)) +* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: [#3002](https://github.com/elastic/elasticsearch/pull/3002), [#3002](https://github.com/elastic/elasticsearch/pull/3002)) + +### Regressions [elasticsearch-820-regression] + +Mappings: +* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: [#9000](https://github.com/elastic/elasticsearch/pull/9000)) +* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: [#9002](https://github.com/elastic/elasticsearch/pull/9002), [#9002](https://github.com/elastic/elasticsearch/pull/9002)) + + diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.generateFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.generateFile.asciidoc deleted file mode 100644 index f07a5f21db195..0000000000000 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.generateFile.asciidoc +++ /dev/null @@ -1,30 +0,0 @@ -[[es-release-notes]] -= Release notes - -[partintro] --- - -This section summarizes the changes in each release. - -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> - --- - -include::release-notes/8.2.0.asciidoc[] -include::release-notes/8.1.1.asciidoc[] -include::release-notes/8.1.0.asciidoc[] -include::release-notes/8.0.2.asciidoc[] -include::release-notes/8.0.1.asciidoc[] -include::release-notes/8.0.0.asciidoc[] -include::release-notes/8.0.0-rc3.asciidoc[] -include::release-notes/8.0.0-beta2.asciidoc[] -include::release-notes/8.0.0-alpha1.asciidoc[] - diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index cc8fac24dc8ce..dda79d1e308b6 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -12,10 +12,12 @@ If you are migrating from a version prior to version 9.0, you must first upgrade % ## Next version [elasticsearch-nextversion-breaking-changes] -## 9.1.0 [elasticsearch-910-breaking-changes] +```{applies_to} +stack: coming 9.0.1 +``` +## 9.0.1 [elasticsearch-9.0.1-breaking-changes] -ES|QL -: * Allow partial results by default in ES|QL [#125060](https://github.com/elastic/elasticsearch/pull/125060) +No breaking changes in this version. ## 9.0.0 [elasticsearch-900-breaking-changes] diff --git a/docs/release-notes/changelog-bundles/9.0.0.yml b/docs/release-notes/changelog-bundles/9.0.0.yml new file mode 100644 index 0000000000000..4b073d4fd8637 --- /dev/null +++ b/docs/release-notes/changelog-bundles/9.0.0.yml @@ -0,0 +1,2521 @@ +version: 9.0.0 +released: true +generated: 2025-04-14T18:16:08.465456Z +changelogs: + - pr: 90529 + summary: Output a consistent format when generating error json + area: Infra/REST API + type: breaking + issues: + - 89387 + breaking: + area: REST API + title: Error JSON structure has changed when detailed errors are disabled + details: |- + This change modifies the JSON format of error messages returned to REST clients + when detailed messages are turned off. + Previously, JSON returned when an exception occurred, and `http.detailed_errors.enabled: false` was set, + just consisted of a single `"error"` text field with some basic information. + Setting `http.detailed_errors.enabled: true` (the default) changed this field + to an object with more detailed information. + With this change, non-detailed errors now have the same structure as detailed errors. `"error"` will now always + be an object with, at a minimum, a `"type"` and `"reason"` field. Additional fields are included when detailed + errors are enabled. + To use the previous structure for non-detailed errors, use the v8 REST API. + impact: |- + If you have set `http.detailed_errors.enabled: false` (the default is `true`) + the structure of JSON when any exceptions occur now matches the structure when + detailed errors are enabled. + To use the previous structure for non-detailed errors, use the v8 REST API. + notable: false + essSettingChange: false + - pr: 104125 + summary: Disable machine learning on macOS x86_64 + area: Machine Learning + type: breaking + issues: [] + breaking: + area: Packaging + title: Disable machine learning on macOS x86_64 + details: "The machine learning plugin is permanently disabled on macOS x86_64. For the last three years Apple has been selling hardware based on the arm64 architecture, and support will increasingly focus on this architecture in the future. Changes to upstream dependencies of Elastic's machine learning functionality have made it unviable for Elastic to continue to build machine learning on macOS x86_64." + impact: "To continue to use machine learning functionality on macOS please switch to an arm64 machine (Apple silicon). Alternatively, it will still be possible to run Elasticsearch with machine learning enabled in a Docker container on macOS x86_64." + notable: false + essSettingChange: false + - pr: 111104 + summary: "ESQL: Enable async get to support formatting" + area: ES|QL + type: feature + issues: + - 110926 + - pr: 111494 + summary: Extensible Completion Postings Formats + area: Suggesters + type: enhancement + issues: [] + - pr: 111852 + summary: Add DeBERTa-V2/V3 tokenizer + area: Machine Learning + type: enhancement + issues: [] + - pr: 112258 + summary: Updated Date Range to Follow Documentation When Assuming Missing Values + area: Search + type: bug + issues: + - 111484 + - pr: 112567 + summary: Track shard snapshot progress during node shutdown + area: Snapshot/Restore + type: enhancement + issues: [] + - pr: 112834 + summary: Increase `replica_unassigned_buffer_time` default from 3s to 5s + area: Health + type: enhancement + issues: [] + - pr: 112903 + summary: Remove unsupported legacy value for `discovery.type` + area: Cluster Coordination + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Remove unsupported legacy value for `discovery.type` + details: "Earlier versions of {es} had a `discovery.type` setting which permitted values that referred to legacy discovery types. From v9.0.0 onwards, the only supported values for this setting are `multi-node` (the default) and `single-node`." + impact: Remove any value for `discovery.type` from your `elasticsearch.yml` configuration file except for `multi-node` and `single-node`. + notable: false + essSettingChange: false + - pr: 113102 + summary: Trigger merges after recovery + area: Recovery + type: enhancement + issues: [] + - pr: 113120 + summary: ESQL - enabling scoring with METADATA `_score` + area: ES|QL + type: enhancement + issues: [] + - pr: 113131 + summary: Emit deprecation warning when executing one of the rollup APIs + area: Rollup + type: deprecation + issues: [] + deprecation: + area: Rollup + title: Emit deprecation warning when executing one of the rollup APIs + details: Rollup is already deprecated since 8.11.0 via documentation and since 8.15.0 it is no longer possible to create new rollup jobs in clusters without rollup usage. This change updates the rollup APIs to emit a deprecation warning. + impact: Returning a deprecation warning when using one of the rollup APIs. + notable: false + essSettingChange: false + - pr: 113216 + summary: "[Inference API] Deprecate elser service" + area: Machine Learning + type: deprecation + issues: [] + deprecation: + area: REST API + title: "[Inference API] Deprecate elser service" + details: The `elser` service of the inference API will be removed in an upcoming release. Please use the elasticsearch service instead. + impact: "In the current version there is no impact. In a future version, users of the `elser` service will no longer be able to use it, and will be required to use the `elasticsearch` service to access elser through the inference API." + notable: false + essSettingChange: false + - pr: 113237 + summary: Retry throttled snapshot deletions + area: Snapshot/Restore + type: bug + issues: [] + - pr: 113413 + summary: Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null. + area: Infra/REST API + type: bug + issues: + - 113413 + - pr: 113425 + summary: Add `ensureGreen` test method for use with `adminClient` + area: Infra/Metrics + type: enhancement + issues: [] + - pr: 113462 + summary: Suppress merge-on-recovery for older indices + area: CRUD + type: enhancement + issues: [] + - pr: 113482 + summary: The 'persian' analyzer has stemmer by default + area: Analysis + type: breaking + issues: + - 113050 + breaking: + area: Analysis + title: The 'persian' analyzer has stemmer by default + details: Lucene 10 has added a final stemming step to its PersianAnalyzer that Elasticsearch exposes as 'persian' analyzer. Existing indices will keep the old non-stemming behaviour while new indices will see the updated behaviour with added stemming. Users that wish to maintain the non-stemming behaviour need to define their own analyzer as outlined in https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer. Users that wish to use the new stemming behaviour for existing indices will have to reindex their data. + impact: Indexing with the 'persian' analyzer will produce slightly different tokens. Users should check if this impacts their search results. If they wish to maintain the legacy non-stemming behaviour they can define their own analyzer equivalent as explained in https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer. + notable: false + essSettingChange: false + - pr: 113561 + summary: Add link to Circuit Breaker "Data too large" exception message + area: Infra/Circuit Breakers + type: enhancement + issues: [] + - pr: 113614 + summary: The 'german2' stemmer is now an alias for the 'german' snowball stemmer + area: Analysis + type: breaking + issues: [] + breaking: + area: Analysis + title: The "german2" snowball stemmer is now an alias for the "german" stemmer + details: 'Lucene 10 has merged the improved "german2" snowball language stemmer with the "german" stemmer. For Elasticsearch, "german2" is now a deprecated alias for "german". This may results in slightly different tokens being generated for terms with umlaut substitution (like "ue" for "ü" etc...)' + impact: Replace usages of "german2" with "german" in analysis configuration. Old indices that use the "german" stemmer should be reindexed if possible. + notable: false + essSettingChange: false + - pr: 113827 + summary: Add Optional Source Filtering to Source Loaders + area: Mapping + type: enhancement + issues: [] + - pr: 114002 + summary: Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts + area: Infra/Scripting + type: enhancement + issues: [] + - pr: 114124 + summary: The Korean dictionary for Nori has been updated + area: Analysis + type: breaking + issues: [] + breaking: + area: Analysis + title: The Korean dictionary for Nori has been updated + details: Lucene 10 ships with an updated Korean dictionary (mecab-ko-dic-2.1.1). For details see https://github.com/apache/lucene/issues/11452. Users experiencing changes in search behaviour on existing data are advised to reindex. + impact: The change is small and should generally provide better analysis results. Existing indices for full-text use cases should be reindexed though. + notable: false + essSettingChange: false + - pr: 114146 + summary: Snowball stemmers have been upgraded + area: Analysis + type: breaking + issues: [] + breaking: + area: Analysis + title: Snowball stemmers have been upgraded + details: Lucene 10 ships with an upgrade of its Snowball stemmers. For details see https://github.com/apache/lucene/issues/13209. Users using Snowball stemmers that are experiencing changes in search behaviour on existing data are advised to reindex. + impact: "The upgrade should generally provide improved stemming results. Small changes in token analysis can lead to mismatches with previously index data, so existing indices using Snowball stemmers as part of their analysis chain should be reindexed." + notable: false + essSettingChange: false + - pr: 114177 + summary: "Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]`" + area: Infra/Metrics + type: bug + issues: [] + - pr: 114202 + summary: Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting + area: Snapshot/Restore + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting + details: The `xpack.searchable.snapshot.allocate_on_rolling_restart` setting was created as an escape-hatch just in case relying on the `cluster.routing.allocation.enable=primaries` setting for allocating searchable snapshots during rolling restarts had some unintended side-effects. It has been deprecated since 8.2.0. + impact: Remove `xpack.searchable.snapshot.allocate_on_rolling_restart` from your settings if present. + notable: false + essSettingChange: false + - pr: 114207 + summary: Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting + area: Allocation + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting + details: "Prior to 7.8, whenever a cluster had only a single data node, the watermarks would not be respected. In order to change this in 7.8+ in a backwards compatible way, we introduced the `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` node setting. The setting was deprecated in 7.14 and was made to accept only true in 8.0" + impact: No known end user impact + notable: false + essSettingChange: false + - pr: 114231 + summary: Remove cluster state from `/_cluster/reroute` response + area: Allocation + type: breaking + issues: + - 88978 + breaking: + area: REST API + title: Remove cluster state from `/_cluster/reroute` response + details: The `POST /_cluster/reroute` API no longer returns the cluster state in its response. The `?metric` query parameter to this API now has no effect and its use will be forbidden in a future version. + impact: Cease usage of the `?metric` query parameter when calling the `POST /_cluster/reroute` API. + notable: false + essSettingChange: false + - pr: 114303 + summary: Fix TDigestState.read CB leaks + area: ES|QL + type: bug + issues: + - 114194 + - pr: 114317 + summary: "ESQL: CATEGORIZE as a `BlockHash`" + area: ES|QL + type: enhancement + issues: [] + - pr: 114445 + summary: Wrap jackson exception on malformed json string + area: Infra/Core + type: bug + issues: + - 114142 + - pr: 114566 + summary: Use Azure blob batch API to delete blobs in batches + area: Distributed + type: enhancement + issues: [] + - pr: 114618 + summary: Add a new index setting to skip recovery source when synthetic source is enabled + area: Logs + type: enhancement + issues: [] + - pr: 114623 + summary: Preserve thread context when waiting for segment generation in RTG + area: CRUD + type: bug + issues: [] + - pr: 114741 + summary: Upgrade to Lucene 10 + area: Search + type: upgrade + issues: [] + - pr: 114813 + summary: Retry `S3BlobContainer#getRegister` on all exceptions + area: Snapshot/Restore + type: enhancement + issues: [] + - pr: 114837 + summary: Add warning headers for ingest pipelines containing special characters + area: Ingest Node + type: bug + issues: + - 104411 + - pr: 114879 + summary: Add refresh `.security` index call between security migrations + area: Security + type: enhancement + issues: [] + - pr: 114914 + summary: Adding chunking settings to `IbmWatsonxService` + area: Machine Learning + type: enhancement + issues: [] + - pr: 115020 + summary: Adding endpoint creation validation for all task types to remaining services + area: Machine Learning + type: enhancement + issues: [] + - pr: 115241 + summary: "[Security Solution] Add `create_index` to `kibana_system` role for index/DS `.logs-endpoint.action.responses-*`" + area: Authorization + type: enhancement + issues: [] + - pr: 115314 + summary: Only aggregations require at least one shard request + area: Search + type: enhancement + issues: [] + - pr: 115383 + summary: Only publish desired balance gauges on master + area: Allocation + type: enhancement + issues: [] + - pr: 115393 + summary: Remove deprecated local attribute from alias APIs + area: Indices APIs + type: breaking + issues: [] + breaking: + area: REST API + title: Remove deprecated local attribute from alias APIs + details: "The following APIs no longer accept the `?local` query parameter: `GET /_alias`, `GET /_aliases`, `GET /_alias/{name}`, `HEAD /_alias/{name}`, `GET /{index}/_alias`, `HEAD /{index}/_alias`, `GET /{index}/_alias/{name}`, `HEAD /{index}/_alias/{name}`, `GET /_cat/aliases`, and `GET /_cat/aliases/{alias}`. This parameter has been deprecated and ignored since version 8.12." + impact: Cease usage of the `?local` query parameter when calling the listed APIs. + notable: false + essSettingChange: false + - pr: 115616 + summary: Fix double lookup failure on ESQL + area: ES|QL + type: bug + issues: + - 111398 + - pr: 115721 + summary: Change Reindexing metrics unit from millis to seconds + area: Reindex + type: enhancement + issues: [] + - pr: 115779 + summary: Don't allow secure settings in YML config (109115) + area: Infra/Settings + type: bug + issues: + - 109115 + - pr: 115831 + summary: Increase minimum threshold in shard balancer + area: Allocation + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Minimum shard balancer threshold is now 1.0 + details: "Earlier versions of {es} accepted any non-negative value for `cluster.routing.allocation.balance.threshold`, but values smaller than `1.0` do not make sense and have been ignored since version 8.6.1. From 9.0.0 these nonsensical values are now forbidden." + impact: Do not set `cluster.routing.allocation.balance.threshold` to a value less than `1.0`. + notable: false + essSettingChange: false + - pr: 115836 + summary: Catch and handle disconnect exceptions in search + area: Search + type: bug + issues: [] + - pr: 115876 + summary: Inference duration and error metrics + area: Machine Learning + type: enhancement + issues: [] + - pr: 115923 + summary: Apply more strict parsing of actions in bulk API + area: Indices APIs + type: breaking + issues: [] + breaking: + area: REST API + title: Apply more strict parsing of actions in bulk API + details: "Previously, the following classes of malformed input were deprecated but not rejected in the action lines of the a bulk request: missing closing brace; additional keys after the action (which were ignored); additional data after the closing brace (which was ignored). They will now be considered errors and rejected." + impact: Users must provide well-formed input when using the bulk API. (They can request REST API compatibility with v8 to get the previous behaviour back as an interim measure.) + notable: false + essSettingChange: false + - pr: 115930 + summary: Inconsistency in the _analyzer api when the index is not included + area: Search + type: bug + issues: [] + - pr: 115938 + summary: Wait for up to 2 seconds for yellow status before starting search + area: Machine Learning + type: bug + issues: + - 107777 + - 105955 + - 107815 + - 112191 + - pr: 116026 + summary: Change Elasticsearch timeouts to 429 response instead of 5xx + area: Infra/Core + type: breaking + issues: [] + breaking: + area: REST API + title: Change most Elasticsearch timeouts to 429 response instead of 5xx + details: "When a timeout occurs in most REST requests, whether via a per-request timeout, or a system default, the request would return a 5xx response code. The response code from those APIs when a timeout occurs is now 429." + impact: Adjust any code relying on retrying on 5xx responses for timeouts to look for a 429 response code and inspect the response to determine whether a timeout occured. + notable: false + essSettingChange: false + - pr: 116043 + summary: Support partial sort fields in TopN pushdown + area: ES|QL + type: enhancement + issues: + - 114515 + - pr: 116060 + summary: Fix leak in `DfsQueryPhase` and introduce search disconnect stress test + area: Search + type: bug + issues: + - 115056 + - pr: 116077 + summary: Remove `ecs` option on `user_agent` processor + area: Ingest Node + type: breaking + issues: [] + breaking: + area: Ingest + title: Remove `ecs` option on `user_agent` processor + details: The `user_agent` ingest processor no longer accepts the `ecs` option. (It was previously deprecated and ignored.) + impact: Users should stop using the `ecs` option when creating instances of the `user_agent` ingest processor. The option will be removed from existing processors stored in the cluster state on upgrade. + notable: false + essSettingChange: false + - pr: 116112 + summary: Remove ignored fallback option on GeoIP processor + area: Ingest Node + type: breaking + issues: [] + breaking: + area: Ingest + title: Remove ignored fallback option on GeoIP processor + details: The option fallback_to_default_databases on the geoip ingest processor has been removed. (It was deprecated and ignored since 8.0.0.) + impact: Customers should stop remove the noop fallback_to_default_databases option on any geoip ingest processors. + notable: false + essSettingChange: false + - pr: 116115 + summary: Allow http unsafe buffers by default + area: Network + type: enhancement + issues: [] + - pr: 116174 + summary: Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations + area: Aggregations + type: bug + issues: + - 115777 + - pr: 116259 + summary: Fix `_type` deprecation on simulate pipeline API + area: Ingest Node + type: deprecation + issues: [] + deprecation: + area: REST API + title: Document `_type` deprecated on simulate pipeline API + details: "Passing a document with a `_type` property is deprecated in the `/_ingest/pipeline/{id}/_simulate` and `/_ingest/pipeline/_simulate` APIs." + impact: "Users should already have stopped using mapping types, which were deprecated in {es} 7. This deprecation warning will fire if they specify mapping types on documents pass to the simulate pipeline API." + notable: false + essSettingChange: false + - pr: 116388 + summary: Add support for partial shard results + area: EQL + type: enhancement + issues: [] + - pr: 116531 + summary: "Add a standard deviation aggregating function: STD_DEV" + area: ES|QL + type: enhancement + issues: [] + - pr: 116663 + summary: KNN vector rescoring for quantized vectors + area: Vector Search + type: feature + issues: [] + - pr: 116687 + summary: Add LogsDB option to route on sort fields + area: Logs + type: enhancement + issues: [] + - pr: 116692 + summary: Remove all mentions of eis and gateway and deprecate flags that do + area: Machine Learning + type: enhancement + issues: [] + - pr: 116739 + summary: Change default Docker image to be based on UBI minimal instead of Ubuntu + area: Infra/Core + type: enhancement + issues: [] + - pr: 116755 + summary: Smarter field caps with subscribable listener + area: ES|QL + type: enhancement + issues: [] + - pr: 116765 + summary: Metrics for incremental bulk splits + area: Distributed + type: enhancement + issues: [] + - pr: 116868 + summary: Run `TransportGetComponentTemplateAction` on local node + area: Indices APIs + type: enhancement + issues: [] + - pr: 116904 + summary: Add a not-master state for desired balance + area: Allocation + type: enhancement + issues: [] + - pr: 116922 + summary: Always check if index mode is logsdb + area: Logs + type: bug + issues: [] + - pr: 116943 + summary: Remove support for deprecated `force_source` highlighting parameter + area: Highlighting + type: breaking + issues: [] + breaking: + area: REST API + title: Remove support for deprecated `force_source` highlighting parameter + details: The deprecated highlighting `force_source` parameter is no longer supported. + impact: Users should remove usages of the `force_source` parameter from their search requests. + notable: false + essSettingChange: false + - pr: 116964 + summary: "Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions" + area: ES|QL + type: feature + issues: + - 104875 + - pr: 116970 + summary: Remove legacy params from range query + area: Search + type: breaking + issues: [] + breaking: + area: REST API + title: Remove legacy params from range query + details: "The deprecated range query parameters `to`, `from`, `include_lower`, and `include_upper` are no longer supported." + impact: "Users should use `lt`, `lte`, `gt`, and `gte` query parameters instead." + notable: false + essSettingChange: false + - pr: 116996 + summary: Initial work on `ReindexDatastreamIndexAction` + area: Data streams + type: enhancement + issues: [] + - pr: 117095 + summary: Add version prefix to Inference Service API path + area: Inference + type: enhancement + issues: [] + - pr: 117148 + summary: Preserve thread context when waiting for segment generation in RTG + area: CRUD + type: bug + issues: [] + - pr: 117176 + summary: Integrate IBM watsonx to Inference API for re-ranking task + area: Experiences + type: enhancement + issues: [] + - pr: 117199 + summary: Speed up bit compared with floats or bytes script operations + area: Vector Search + type: enhancement + issues: [] + - pr: 117214 + summary: Returning ignored fields in the simulate ingest API + area: Ingest Node + type: enhancement + issues: [] + - pr: 117229 + summary: "In this pr, a 400 error is returned when _source / _seq_no / _feature / _nested_path / _field_names is requested, rather a 5xx" + area: Search + type: bug + issues: [] + - pr: 117230 + summary: Make various alias retrieval APIs wait for cluster to unblock + area: Distributed + type: enhancement + issues: [] + - pr: 117243 + summary: Bump major version for feature migration system indices + area: Infra/Core + type: upgrade + issues: [] + - pr: 117246 + summary: LOOKUP JOIN using field-caps for field mapping + area: ES|QL + type: enhancement + issues: [] + - pr: 117265 + summary: Async search responses have CCS metadata while searches are running + area: ES|QL + type: enhancement + issues: [] + - pr: 117287 + summary: Fixing bug setting index when parsing Google Vertex AI results + area: Machine Learning + type: bug + issues: [] + - pr: 117303 + summary: Remove HTTP content copies + area: Network + type: enhancement + issues: [] + - pr: 117359 + summary: Term query for ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 117451 + summary: ST_EXTENT aggregation + area: ES|QL + type: feature + issues: + - 104659 + - pr: 117519 + summary: Remove `data_frame_transforms` roles + area: Transform + type: breaking + issues: [] + breaking: + area: Transform + title: Remove `data_frame_transforms` roles + details: "`data_frame_transforms_admin` and `data_frame_transforms_user` were deprecated in Elasticsearch 7 and are being removed in Elasticsearch 9. `data_frame_transforms_admin` is now `transform_admin`. `data_frame_transforms_user` is now `transform_user`. Users must call the `_update` API to replace the permissions on the Transform before the Transform can be started." + impact: "Transforms created with either the `data_frame_transforms_admin` or the `data_frame_transforms_user` role will fail to start. The Transform will remain in a `stopped` state, and its health will be red while displaying permission failures." + notable: false + essSettingChange: false + - pr: 117530 + summary: Expose operation and request counts separately in repository stats + area: Snapshot/Restore + type: enhancement + issues: + - 104443 + - pr: 117555 + summary: Expand type compatibility for match function and operator + area: ES|QL + type: feature + issues: [] + - pr: 117572 + summary: Address and remove any references of RestApiVersion version 7 + area: Search + type: enhancement + issues: [] + - pr: 117581 + summary: Make reserved built-in roles queryable + area: Authorization + type: enhancement + issues: [] + - pr: 117583 + summary: Removing index alias creation for deprecated transforms notification index + area: Machine Learning + type: deprecation + issues: [] + deprecation: + area: Transform + title: Removing index alias creation for deprecated transforms notification index + details: "As part of the migration from 7.x to 8.x, the `.data-frame-notifications-1` index was deprecated and replaced with the `.transform-notifications-000002` index. The index is no longer created by default, all writes are directed to the new index, and any clusters with the deprecated index will have an alias created to ensure that reads are still retrieving data that was written to the index before the migration to 8.x. This change removes the alias from the deprecated index in 9.x. Any clusters with the alias present will retain it, but it will not be created on new clusters." + impact: No known end user impact. + notable: false + essSettingChange: false + - pr: 117589 + summary: Add Inference Unified API for chat completions for OpenAI + area: Machine Learning + type: enhancement + issues: [] + - pr: 117606 + summary: Remove deprecated sort from reindex operation within dataframe analytics procedure + area: Machine Learning + type: enhancement + issues: [] + - pr: 117618 + summary: SearchStatesIt failures reported by CI + area: Search + type: bug + issues: + - 116617 + - 116618 + - pr: 117643 + summary: Drop null columns in text formats + area: ES|QL + type: bug + issues: + - 116848 + - pr: 117655 + summary: Add nulls support to Categorize + area: ES|QL + type: enhancement + issues: [] + - pr: 117701 + summary: Watcher history index has too many indexed fields - + area: Watcher + type: bug + issues: + - 71479 + - pr: 117731 + summary: Add cluster level reduction + area: ES|QL + type: enhancement + issues: [] + - pr: 117748 + summary: Add IMDSv2 support to `repository-s3` + area: Snapshot/Restore + type: enhancement + issues: + - 105135 + - pr: 117750 + summary: "`CrossClusterIT` `testCancel` failure" + area: Search + type: bug + issues: + - 108061 + - pr: 117778 + summary: "[Connector APIs] Enforce index prefix for managed connectors" + area: Extract&Transform + type: feature + issues: [] + - pr: 117831 + summary: Fix/QueryBuilderBWCIT_muted_test + area: Search + type: bug + issues: [] + - pr: 117839 + summary: Add match support for `semantic_text` fields + area: Search + type: enhancement + issues: [] + - pr: 117858 + summary: Create upgrade mode + area: Transform + type: enhancement + issues: [] + - pr: 117898 + summary: Limit size of query + area: ES|QL + type: bug + issues: [] + - pr: 117917 + summary: Add option to store `sparse_vector` outside `_source` + area: Mapping + type: feature + issues: [] + - pr: 117933 + summary: Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` + area: Infra/Logging + type: bug + issues: + - 83251 + breaking: + area: Logging + title: Deprecation logging value change for "data_stream.dataset" and "event.dataset" + details: |- + This change modifies the "data_stream.dataset" and "event.dataset" value for deprecation logging + to use the value `elasticsearch.deprecation` instead of `deprecation.elasticsearch`. This is now + consistent with other values where the name of the service is the first part of the key. + impact: |- + If you are directly consuming deprecation logs for "data_stream.dataset" and "event.dataset" and filtering on + this value, you will need to update your filters to use `elasticsearch.deprecation` instead of + `deprecation.elasticsearch`. + notable: false + essSettingChange: false + - pr: 117939 + summary: Adding default endpoint for Elastic Rerank + area: Machine Learning + type: enhancement + issues: [] + - pr: 117949 + summary: Move `SlowLogFieldProvider` instantiation to node construction + area: Infra/Logging + type: bug + issues: [] + - pr: 117963 + summary: "`SearchServiceTests.testParseSourceValidation` failure" + area: Search + type: bug + issues: [] + - pr: 117989 + summary: ESQL Add esql hash function + area: ES|QL + type: enhancement + issues: [] + - pr: 117994 + summary: Even better(er) binary quantization + area: Vector Search + type: enhancement + issues: [] + - pr: 118016 + summary: Propagate status codes from shard failures appropriately + area: Search + type: enhancement + issues: + - 118482 + - pr: 118025 + summary: Update sparse text embeddings API route for Inference Service + area: Inference + type: enhancement + issues: [] + - pr: 118027 + summary: Esql compare nanos and millis + area: ES|QL + type: enhancement + issues: + - 116281 + - pr: 118035 + summary: Include hidden indices in `DeprecationInfoAction` + area: Indices APIs + type: bug + issues: + - 118020 + - pr: 118058 + summary: Grant necessary Kibana application privileges to `reporting_user` role + area: Authorization + type: enhancement + issues: [] + - pr: 118064 + summary: Add Highlighter for Semantic Text Fields + area: Highlighting + type: feature + issues: [] + - pr: 118094 + summary: Update ASM 9.7 -> 9.7.1 to support JDK 24 + area: Infra/Core + type: upgrade + issues: [] + - pr: 118102 + summary: "ESQL: Enterprise license enforcement for CCS" + area: ES|QL + type: enhancement + issues: [] + - pr: 118103 + summary: Remove any references to org.elasticsearch.core.RestApiVersion#V_7 + area: Infra/Core + type: breaking + issues: [] + breaking: + area: REST API + title: Remove any references to org.elasticsearch.core.RestApiVersion#V_7 + details: This PR removes all references to V_7 in the Rest API. V7 features marked for deprecation have been removed. + impact: This change is breaking for any external plugins/clients that rely on the V_7 enum or deprecated version 7 functionality + notable: false + essSettingChange: false + - pr: 118104 + summary: Remove old `_knn_search` tech preview API in v9 + area: Vector Search + type: breaking + issues: [] + breaking: + area: REST API + title: Remove old `_knn_search` tech preview API in v9 + details: "The original, tech-preview api for vector search, `_knn_search`, has been removed in v9. For all vector search operations, you should utilize the `_search` endpoint." + impact: The `_knn_search` API is now inaccessible without providing a compatible-with flag for v8. + notable: false + essSettingChange: false + - pr: 118114 + summary: Enable physical plan verification + area: ES|QL + type: enhancement + issues: [] + - pr: 118122 + summary: "ES|QL: Partial result on demand for async queries" + area: ES|QL + type: enhancement + issues: [] + - pr: 118143 + summary: Infrastructure for assuming cluster features in the next major version + area: Infra/Core + type: feature + issues: [] + - pr: 118173 + summary: ES|QL categorize with multiple groupings + area: Machine Learning + type: feature + issues: [] + - pr: 118177 + summary: Fixing bedrock event executor terminated cache issue + area: Machine Learning + type: bug + issues: + - 117916 + - pr: 118188 + summary: Check for early termination in Driver + area: ES|QL + type: enhancement + issues: [] + - pr: 118192 + summary: Remove `client.type` setting + area: Infra/Core + type: breaking + issues: + - 104574 + breaking: + area: Cluster and node setting + title: Remove `client.type` setting + details: The node setting `client.type` has been ignored since the node client was removed in 8.0. The setting is now removed. + impact: Remove the `client.type` setting from `elasticsearch.yml` + notable: false + essSettingChange: false + - pr: 118194 + summary: Retry on `ClusterBlockException` on transform destination index + area: Machine Learning + type: enhancement + issues: [] + - pr: 118266 + summary: Prevent data nodes from sending stack traces to coordinator when `error_trace=false` + area: Search + type: enhancement + issues: [] + - pr: 118267 + summary: Adding get migration reindex status + area: Data streams + type: enhancement + issues: [] + - pr: 118291 + summary: Adding a migration reindex cancel API + area: Data streams + type: enhancement + issues: [] + - pr: 118324 + summary: Allow the data type of `null` in filters + area: ES|QL + type: bug + issues: + - 116351 + - pr: 118353 + summary: Epoch Millis Rounding Down and Not Up 2 + area: Infra/Core + type: bug + issues: [] + - pr: 118366 + summary: |- + Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password + will prevent node from starting + area: Authentication + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: -| Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password will prevent node from starting + details: "-| For LDAP or AD authentication realms, setting a bind DN (via the `xpack.security.authc.realms.ldap.*.bind_dn` or `xpack.security.authc.realms.active_directory.*.bind_dn` realm settings) without a bind password is a misconfiguration that may prevent successful authentication to the node. Nodes will fail to start if a bind DN is specified without a password." + impact: "-| If you have a bind DN configured for an LDAP or AD authentication realm, set a bind password for {ref}/ldap-realm.html#ldap-realm-configuration[LDAP] or {ref}/active-directory-realm.html#ad-realm-configuration[Active Directory]. Configuring a bind DN without a password prevents the misconfigured node from starting." + notable: false + essSettingChange: false + - pr: 118375 + summary: Check for presence of error object when validating streaming responses from integrations in the inference API + area: Machine Learning + type: enhancement + issues: [] + - pr: 118410 + summary: Push down filter passed lookup join + area: ES|QL + type: enhancement + issues: [] + - pr: 118435 + summary: "`_score` should not be a reserved attribute in ES|QL" + area: ES|QL + type: enhancement + issues: + - 118460 + - pr: 118474 + summary: Esql bucket function for date nanos + area: ES|QL + type: enhancement + issues: + - 118031 + - pr: 118484 + summary: Remove date histogram boolean support + area: Aggregations + type: breaking + issues: [] + breaking: + area: Aggregations + title: Remove date histogram boolean support + details: "Elasticsearch no longer allows running Date Histogram aggregations over boolean fields. Instead, use Terms aggregation for boolean fields." + impact: "We expect the impact to be minimal, as this never produced good results, and has been deprecated for years." + notable: false + essSettingChange: false + - pr: 118544 + summary: ESQL - Remove restrictions for disjunctions in full text functions + area: ES|QL + type: enhancement + issues: [] + - pr: 118562 + summary: Update data stream deprecations warnings to new format and filter searchable snapshots from response + area: Data streams + type: enhancement + issues: [] + - pr: 118585 + summary: Add a generic `rescorer` retriever based on the search request's rescore functionality + area: Ranking + type: feature + issues: + - 118327 + - pr: 118602 + summary: Limit memory usage of `fold` + area: ES|QL + type: bug + issues: [] + - pr: 118617 + summary: Add support for `sparse_vector` queries against `semantic_text` fields + area: Search + type: enhancement + issues: [] + - pr: 118619 + summary: Optional named arguments for function in map + area: EQL + type: enhancement + issues: [] + - pr: 118634 + summary: "Add undeclared Azure settings, modify test to exercise them" + area: Snapshot/Restore + type: bug + issues: [] + - pr: 118652 + summary: Add Jina AI API to do inference for Embedding and Rerank models + area: Machine Learning + type: enhancement + issues: [] + - pr: 118669 + summary: "[Connector API] Support soft-deletes of connectors" + area: Extract&Transform + type: feature + issues: [] + - pr: 118671 + summary: Adjust `random_score` default field to `_seq_no` field + area: Search + type: breaking + issues: [] + breaking: + area: Search + title: Adjust `random_score` default field to `_seq_no` field + details: "When providing a 'seed' parameter to a 'random_score' function in the 'function_score' query but NOT providing a 'field', the default 'field' is switched from '_id' to '_seq_no'." + impact: The random scoring and ordering may change when providing a 'seed' and not providing a 'field' to a 'random_score' function. + notable: false + essSettingChange: false + - pr: 118674 + summary: Ignore failures from renormalizing buckets in read-only index + area: Machine Learning + type: enhancement + issues: [] + - pr: 118681 + summary: "`ConnectTransportException` returns retryable BAD_GATEWAY" + area: Network + type: enhancement + issues: + - 118320 + - pr: 118697 + summary: Esql implicit casting for date nanos + area: ES|QL + type: enhancement + issues: + - 118476 + - pr: 118704 + summary: Avoid updating settings version in `MetadataMigrateToDataStreamService` when settings have not changed + area: Data streams + type: bug + issues: [] + - pr: 118774 + summary: Apply default k for knn query eagerly + area: Vector Search + type: bug + issues: [] + - pr: 118802 + summary: ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape + area: ES|QL + type: enhancement + issues: [] + - pr: 118804 + summary: Add new experimental `rank_vectors` mapping for late-interaction second order ranking + area: Vector Search + type: feature + issues: [] + highlight: + notable: true + title: rank_vectors field type is now available for late-interaction ranking + body: | + [`rank_vectors`](../reference/elasticsearch/mapping-reference/rank-vectors.md) is a new field type released as an experimental feature in Elasticsearch 9.0. It is designed to be used with dense vectors and allows for late-interaction second order ranking. + + Late-interaction models are powerful rerankers. While their size and overall cost doesn’t lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali. + pr: 118804 + - pr: 118823 + summary: Fix attribute set equals + area: ES|QL + type: bug + issues: [] + - pr: 118825 + summary: "Remove support for type, fields, `copy_to` and boost in metadata field definition" + area: Mapping + type: breaking + issues: [] + breaking: + area: Mapping + title: "Remove support for type, fields, copy_to and boost in metadata field definition" + details: "The type, fields, copy_to and boost parameters are no longer supported in metadata field definition starting with version 9." + impact: "Users providing type, fields, copy_to or boost as part of metadata field definition should remove them from their mappings." + notable: false + essSettingChange: false + - pr: 118858 + summary: Lookup join on multiple join fields not yet supported + area: ES|QL + type: enhancement + issues: [] + - pr: 118870 + summary: Rewrite TO_UPPER/TO_LOWER comparisons + area: ES|QL + type: enhancement + issues: + - 118304 + - pr: 118871 + summary: "[Elastic Inference Service] Add ElasticInferenceService Unified ChatCompletions Integration" + area: Inference + type: enhancement + issues: [] + - pr: 118890 + summary: Add action to create index from a source index + area: Data streams + type: enhancement + issues: [] + - pr: 118931 + summary: Add a `LicenseAware` interface for licensed Nodes + area: ES|QL + type: enhancement + issues: + - 117405 + - pr: 118938 + summary: Hash functions + area: ES|QL + type: enhancement + issues: [] + - pr: 118941 + summary: Allow archive and searchable snapshots indices in N-2 version + area: Recovery + type: enhancement + issues: [] + - pr: 118959 + summary: Allow kibana_system user to manage .reindexed-v8-internal.alerts indices + area: Authorization + type: enhancement + issues: [] + - pr: 118968 + summary: Configure index sorting through index settings for logsdb + area: Logs + type: enhancement + issues: + - 118686 + - pr: 119001 + summary: Add support for specifying reindexing script for system index migration + area: Infra/Core + type: enhancement + issues: [] + - pr: 119003 + summary: Add a `replicate_for` option to the ILM `searchable_snapshot` action + area: ILM+SLM + type: enhancement + issues: [] + - pr: 119007 + summary: Block-writes cannot be added after read-only + area: Data streams + type: bug + issues: + - 119002 + - pr: 119011 + summary: Add support for knn vector queries on `semantic_text` fields + area: Search + type: enhancement + issues: [] + - pr: 119054 + summary: "[Security Solution] allows `kibana_system` user to manage .reindexed-v8-* Security Solution indices" + area: Authorization + type: enhancement + issues: [] + - pr: 119067 + summary: Metrics for indexing failures due to version conflicts + area: CRUD + type: feature + issues: [] + - pr: 119072 + summary: Turn `_source` meta fieldmapper's mode attribute into a no-op + area: Mapping + type: breaking + issues: + - 118596 + breaking: + area: Mapping + title: Turn `_source` meta fieldmapper's mode attribute into a no-op + details: The `mode` mapping attribute of `_source` metadata field mapper has been turned into a no-op. Instead the `index.mapping.source.mode` index setting should be used to configure source mode. + impact: Configuring the `mode` attribute for the `_source` meta field mapper will have no effect on indices created with Elasticsearch 9.0.0 or later. Note that `_source.mode` configured on indices before upgrading to 9.0.0 or later will remain efficive after upgrading. + notable: false + essSettingChange: false + - pr: 119227 + summary: Remove unfreeze REST endpoint + area: Indices APIs + type: breaking + issues: [] + breaking: + area: REST API + title: Remove unfreeze REST endpoint + details: "The `/{index}/_unfreeze` REST endpoint is no longer supported. This API was deprecated, and the corresponding `/{index}/_freeze` endpoint was removed in 8.0." + impact: "None, since it is not possible to have a frozen index in a version which is readable by Elasticsearch 9.0" + notable: false + essSettingChange: false + - pr: 119250 + summary: Add rest endpoint for `create_from_source_index` + area: Data streams + type: enhancement + issues: [] + - pr: 119265 + summary: Fix `AbstractShapeGeometryFieldMapperTests` + area: ES|QL + type: bug + issues: + - 119201 + - pr: 119291 + summary: Register mustache size limit setting + area: Infra/Scripting + type: bug + issues: [] + - pr: 119296 + summary: Fix writing for LOOKUP status + area: ES|QL + type: bug + issues: + - 119086 + - pr: 119308 + summary: Upgrade to Lucene 10.1.0 + area: Search + type: upgrade + issues: [] + - pr: 119310 + summary: Remove ChunkedToXContentBuilder + area: Network + type: bug + issues: + - 118647 + - pr: 119348 + summary: Auto-migrate `max_page_search_size` + area: Transform + type: enhancement + issues: [] + - pr: 119474 + summary: Add ES|QL cross-cluster query telemetry collection + area: ES|QL + type: enhancement + issues: [] + - pr: 119476 + summary: Fix TopN row size estimate + area: ES|QL + type: bug + issues: + - 106956 + - pr: 119503 + summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. + area: Search + type: bug + issues: + - 117042 + - pr: 119504 + summary: Optimized index sorting for OTel logs + area: Data streams + type: enhancement + issues: [] + - pr: 119536 + summary: Fix ROUND() with unsigned longs throwing in some edge cases + area: ES|QL + type: bug + issues: [] + - pr: 119542 + summary: Wait while index is blocked + area: Transform + type: enhancement + issues: [] + - pr: 119543 + summary: "[Inference API] Fix unique ID message for inference ID matches trained model ID" + area: Machine Learning + type: bug + issues: + - 111312 + - pr: 119564 + summary: Http stream activity tracker and exceptions handling + area: Network + type: enhancement + issues: [] + - pr: 119575 + summary: Fix realtime get of nested fields with synthetic source + area: Mapping + type: bug + issues: + - 119553 + - pr: 119580 + summary: Do not serialize `EsIndex` in plan + area: ES|QL + type: enhancement + issues: [] + - pr: 119621 + summary: Enable node-level reduction by default + area: ES|QL + type: enhancement + issues: [] + - pr: 119679 + summary: Support mTLS for the Elastic Inference Service integration inside the inference API + area: Machine Learning + type: feature + issues: [] + - pr: 119691 + summary: Fix `bbq_hnsw` merge file cleanup on random IO exceptions + area: Vector Search + type: bug + issues: + - 119392 + - pr: 119730 + summary: Enable KQL function as a tech preview + area: ES|QL + type: enhancement + issues: [] + - pr: 119743 + summary: POC mark read-only + area: Engine + type: enhancement + issues: [] + - pr: 119749 + summary: Strengthen encryption for elasticsearch-keystore tool to AES 256 + area: Infra/CLI + type: enhancement + issues: [] + - pr: 119772 + summary: ESQL Support IN operator for Date nanos + area: ES|QL + type: enhancement + issues: + - 118578 + - pr: 119780 + summary: Add index and reindex request settings to speed up reindex + area: Data streams + type: enhancement + issues: [] + - pr: 119792 + summary: Make semantic text part of the text family + area: Search + type: enhancement + issues: [] + - pr: 119798 + summary: "Add a `PostAnalysisAware,` distribute verification" + area: ES|QL + type: enhancement + issues: [] + - pr: 119830 + summary: Run `TransportGetComposableIndexTemplate` on local node + area: Indices APIs + type: enhancement + issues: [] + - pr: 119831 + summary: Run `TransportClusterGetSettingsAction` on local node + area: Infra/Settings + type: enhancement + issues: [] + - pr: 119837 + summary: Run `TransportGetIndexTemplateAction` on local node + area: Indices APIs + type: enhancement + issues: [] + - pr: 119846 + summary: Drop support for brackets from METADATA syntax + area: ES|QL + type: deprecation + issues: + - 115401 + deprecation: + area: ES|QL + title: Drop support for brackets from METADATA syntax + details: Please describe the details of this change for the release notes. You can use asciidoc. + impact: Please describe the impact of this change to users + notable: false + essSettingChange: false + - pr: 119863 + summary: Restrict Connector APIs to manage/monitor_connector privileges + area: Extract&Transform + type: breaking + issues: [] + breaking: + area: REST API + title: Restrict Connector APIs to manage/monitor_connector privileges + details: "Connector APIs now enforce the manage_connector and monitor_connector privileges (introduced in 8.15), replacing the previous reliance on index-level permissions for .elastic-connectors and .elastic-connectors-sync-jobs in API calls." + impact: Connector APIs now require manage_connector and monitor_connector privileges + notable: false + essSettingChange: false + - pr: 119886 + summary: Initial support for unmapped fields + area: ES|QL + type: feature + issues: [] + - pr: 119889 + summary: Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` + area: ES|QL + type: enhancement + issues: [] + - pr: 119893 + summary: Add enterprise license check for Inference API actions + area: Machine Learning + type: enhancement + issues: [] + - pr: 119898 + summary: Resolve/cluster allows querying for cluster info only (no index expression required) + area: CCS + type: enhancement + issues: [] + - pr: 119922 + summary: "[Inference API] fix spell words: covertToString to convertToString" + area: Machine Learning + type: enhancement + issues: [] + - pr: 119926 + summary: Deprecated tracing.apm.* settings got removed. + area: Infra/Metrics + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Deprecated tracing.apm.* settings got removed. + details: "Deprecated `tracing.apm.*` settings got removed, use respective `telemetry.*` / `telemetry.tracing.*` settings instead." + impact: 9.x nodes will refuse to start if any such setting (including secret settings) is still present. + notable: false + essSettingChange: false + - pr: 119968 + summary: Reset relocation/allocation failure counter on node join/shutdown + area: Allocation + type: enhancement + issues: [] + - pr: 120020 + summary: Resume Driver on cancelled or early finished + area: ES|QL + type: enhancement + issues: [] + - pr: 120038 + summary: Run template simulation actions on local node + area: Ingest Node + type: enhancement + issues: [] + - pr: 120042 + summary: Match dot prefix of migrated DS backing index with the source index + area: Data streams + type: bug + issues: [] + - pr: 120055 + summary: Optimize loading mappings when determining synthetic source usage and whether host.name can be sorted on. + area: Logs + type: enhancement + issues: [] + - pr: 120064 + summary: Change the auditor to write via an alias + area: Machine Learning + type: upgrade + issues: [] + - pr: 120084 + summary: Improve how reindex data stream index action handles api blocks + area: Data streams + type: enhancement + issues: [] + - pr: 120087 + summary: Include `clusterApplyListener` in long cluster apply warnings + area: Cluster Coordination + type: enhancement + issues: [] + - pr: 120108 + summary: Remove the ability to read frozen indices + area: Indices APIs + type: breaking + issues: [] + breaking: + area: Index setting + title: Remove the ability to read frozen indices + details: The ability to read frozen indices has been removed. (Frozen indices are no longer useful due to improvements in heap memory usage. The ability to freeze indices was removed in 8.0.) + impact: Users must unfreeze any frozen indices before upgrading. + notable: false + essSettingChange: false + - pr: 120128 + summary: Add Multi-Field Support for Semantic Text Fields + area: Relevance + type: feature + issues: [] + - pr: 120142 + summary: Limit `ByteSizeUnit` to 2 decimals + area: Infra/Core + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Limit `ByteSizeUnit` to 2 decimals + details: "In the past, byte values like `1.25 mb` were allowed but deprecated. Now, values with up to two decimal places are allowed, unless the unit is bytes, in which case no decimals are allowed. Values with too many decimal places result in an error." + impact: "Values with more than two decimal places, like `0.123 mb` will be rejected as an error, where in the past, they'd be accepted with a deprecation warning." + notable: false + essSettingChange: false + - pr: 120143 + summary: Esql - support date nanos in date format function + area: ES|QL + type: enhancement + issues: + - 109994 + - pr: 120163 + summary: Filter deprecated settings when making dest index + area: Data streams + type: enhancement + issues: [] + - pr: 120192 + summary: Extend `TranslationAware` to all pushable expressions + area: ES|QL + type: enhancement + issues: [] + - pr: 120193 + summary: "Do not capture `ClusterChangedEvent` in `IndicesStore` call to #onClusterStateShardsClosed" + area: Store + type: bug + issues: [] + - pr: 120198 + summary: Bump `TrialLicenseVersion` to allow starting new trial on 9.0 + area: License + type: enhancement + issues: [] + - pr: 120200 + summary: "[Connector API] Support hard deletes with new URL param in delete endpoint" + area: Extract&Transform + type: feature + issues: [] + - pr: 120207 + summary: Make `requests_per_second` configurable to throttle reindexing + area: Data streams + type: enhancement + issues: [] + - pr: 120222 + summary: Adding linear retriever to support weighted sums of sub-retrievers + area: Search + type: enhancement + issues: [] + - pr: 120231 + summary: Add sanity check to `ReindexDatastreamIndexAction` + area: Data streams + type: enhancement + issues: [] + - pr: 120244 + summary: Ignore closed indices for reindex + area: Data streams + type: enhancement + issues: [] + - pr: 120250 + summary: "Retry internally when CAS upload is throttled [GCS]" + area: Snapshot/Restore + type: enhancement + issues: + - 116546 + - pr: 120267 + summary: Set allow_partial_search_results=true by default + area: EQL + type: breaking + issues: [] + breaking: + area: REST API + title: Set allow_partial_search_results=true by default + details: "Before this change, in case of shard failures, EQL queries always returned an error. With this change, they will keep running and will return partial results." + impact: "EQL queries that would previously fail due to shard failures, will now succeed and return partial results. The previous defaults can be restored by setting `xpack.eql.default_allow_partial_results` cluster setting to `false` or setting with `allow_partial_search_results` to `false` in the query request." + notable: false + essSettingChange: false + - pr: 120271 + summary: Optimize indexing points with index and doc values set to true + area: Geo + type: enhancement + issues: [] + - pr: 120291 + summary: ESQL - Allow full text functions disjunctions for non-full text functions + area: ES|QL + type: feature + issues: [] + - pr: 120334 + summary: Introduce `IndexSettingDeprecatedInV8AndRemovedInV9` Setting property + area: Infra/Settings + type: enhancement + issues: [] + - pr: 120340 + summary: Add support for `extended_stats` + area: Transform + type: enhancement + issues: [] + - pr: 120343 + summary: Support some stats on aggregate_metric_double + area: ES|QL + type: enhancement + issues: + - 110649 + - pr: 120354 + summary: Move scoring in ES|QL out of snapshot + area: ES|QL + type: enhancement + issues: [] + - pr: 120355 + summary: Ensure cluster string could be quoted + area: ES|QL + type: enhancement + issues: [] + - pr: 120360 + summary: ESQL - Add Match function options + area: ES|QL + type: feature + issues: [] + - pr: 120370 + summary: "Merge field mappers when updating mappings with [subobjects:false]" + area: Mapping + type: bug + issues: + - 120216 + - pr: 120392 + summary: Test/107515 restore template with match only text mapper it fail + area: Search + type: bug + issues: + - 107515 + - pr: 120400 + summary: "[Inference API] Add node-local rate limiting for the inference API" + area: Machine Learning + type: feature + issues: [] + - pr: 120405 + summary: Automatically rollover legacy ml indices + area: Machine Learning + type: upgrade + issues: [] + - pr: 120445 + summary: Run `GetPipelineTransportAction` on local node + area: Ingest Node + type: enhancement + issues: [] + - pr: 120458 + summary: Do not recommend increasing `max_shards_per_node` + area: Health + type: bug + issues: [] + - pr: 120487 + summary: Fix cat_component_templates documentation + area: CAT APIs + type: bug + issues: [] + - pr: 120494 + summary: Update grammar to rely on `indexPattern` instead of identifier in join target + area: ES|QL + type: enhancement + issues: [] + - pr: 120505 + summary: introduce new categories for deprecated resources in deprecation API + area: Indices APIs + type: enhancement + issues: [] + - pr: 120538 + summary: "Revert unwanted ES|QL lexer changes from PR #120354" + area: ES|QL + type: bug + issues: [] + - pr: 120542 + summary: "Feat: add a user-configurable timeout parameter to the `_resolve/cluster` API" + area: Search + type: enhancement + issues: [] + - pr: 120546 + summary: Restrict agent entitlements to the system classloader unnamed module + area: Infra/Plugins + type: bug + issues: [] + - pr: 120547 + summary: Consistent mapping for OTel log and event bodies + area: Data streams + type: enhancement + issues: [] + - pr: 120548 + summary: Add `remove_index_block` arg to `_create_from` api + area: Indices APIs + type: enhancement + issues: [] + - pr: 120551 + summary: Set default reranker for text similarity reranker to Elastic reranker + area: Ranking + type: enhancement + issues: [] + - pr: 120573 + summary: Optimize `IngestDocument` `FieldPath` allocation + area: Ingest Node + type: enhancement + issues: [] + - pr: 120591 + summary: Increase field limit for OTel metrics to 10 000 + area: TSDB + type: enhancement + issues: [] + - pr: 120617 + summary: Fix queries with document level security on lookup indexes + area: ES|QL + type: bug + issues: + - 120509 + - pr: 120629 + summary: Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As A Warning + area: Data streams + type: enhancement + issues: [] + - pr: 120642 + summary: Defer unpromotable shard refreshes until index refresh blocks are cleared + area: Engine + type: enhancement + issues: [] + - pr: 120643 + summary: Remove index blocks by default in `create_from` + area: Indices APIs + type: enhancement + issues: [] + - pr: 120645 + summary: Esql Support date nanos on date diff function + area: ES|QL + type: enhancement + issues: + - 109999 + - pr: 120662 + summary: Fix broken yaml test `30_create_from` + area: Indices APIs + type: bug + issues: [] + - pr: 120722 + summary: Migrate stream to core error parsing + area: Machine Learning + type: enhancement + issues: [] + - pr: 120725 + summary: |- + A new query parameter `?include_source_on_error` was added for create / index, update and bulk REST APIs to control + if to include the document source in the error response in case of parsing errors. The default value is `true`. + area: Infra/REST API + type: enhancement + issues: [] + - pr: 120727 + summary: Esql - Support date nanos in date extract function + area: ES|QL + type: enhancement + issues: + - 110000 + - pr: 120748 + summary: Removing support for types field in watcher search + area: Watcher + type: breaking + issues: [] + breaking: + area: REST API + title: Removing support for types field in watcher search + details: "Previously, setting the `input.search.request.types` field in the payload when creating a watcher to an empty array was allowed, although it resulted in a deprecation warning and had no effect (and any value other than an empty array would result in an error). Now, support for this field is entirely removed, and the empty array will also result in an error." + impact: Users should stop setting this field (which did not have any effect anyway). + notable: false + essSettingChange: false + - pr: 120752 + summary: Refresh source index before reindexing data stream index + area: Data streams + type: bug + issues: + - 120314 + - pr: 120753 + summary: Optimize `IngestDocMetadata` `isAvailable` + area: Ingest Node + type: enhancement + issues: [] + - pr: 120799 + summary: Remove duplicate code in ESIntegTestCase + area: Search + type: bug + issues: [] + - pr: 120806 + summary: Deprecate certificate based remote cluster security model + area: Security + type: deprecation + issues: [] + deprecation: + area: Authorization + title: Deprecate certificate based remote cluster security model + details: "-| <> is deprecated and will be removed in a future major version. Users are encouraged to <>. The <> is preferred way to configure remote clusters, as it allows to follow security best practices when setting up remote cluster connections and defining fine-grained access control." + impact: "-| If you have configured remote clusters with certificate-based security model, you should <>. Configuring a remote cluster using <>, generates a warning in the deprecation logs." + notable: false + essSettingChange: false + - pr: 120807 + summary: Remove INDEX_REFRESH_BLOCK after index becomes searchable + area: CRUD + type: enhancement + issues: [] + - pr: 120813 + summary: Change Semantic Text To Act Like A Normal Text Field + area: Search + type: breaking + issues: [] + breaking: + area: Search + title: Change Semantic Text To Act Like A Normal Text Field + details: "The previous semantic_text format used a complex subfield structure in _source to store the embeddings. This complicated interactions/integrations with semantic_text fields and _source in general. This new semantic_text format treats it as a normal text field, where the field's value in _source is the value assigned by the user." + impact: "Users who parsed the subfield structure of the previous semantic_text format in _source will need to update their parsing logic. The new format does not directly expose the chunks and embeddings generated from the input text. The new format will be applied to all new indices, any existing indices will continue to use the previous format." + notable: true + essSettingChange: false + - pr: 120821 + summary: "[Deprecation] Add `transform_ids` to outdated index" + area: Transform + type: enhancement + issues: [] + - pr: 120824 + summary: Optimize some per-document hot paths in the geoip processor + area: Ingest Node + type: enhancement + issues: [] + - pr: 120833 + summary: Optimize `IngestCtxMap` construction + area: Ingest Node + type: enhancement + issues: [] + - pr: 120842 + summary: Remove Elastic Inference Service feature flag and deprecated setting + area: Inference + type: enhancement + issues: [] + - pr: 120852 + summary: Correct line and column numbers of missing named parameters + area: ES|QL + type: bug + issues: [] + - pr: 120883 + summary: Add `?master_timeout` to `POST /_ilm/migrate_to_data_tiers` + area: Indices APIs + type: bug + issues: [] + - pr: 120913 + summary: Automatically rollover legacy .ml-anomalies indices + area: Machine Learning + type: upgrade + issues: [] + - pr: 120937 + summary: Use the system index descriptor in the snapshot blob cache cleanup task + area: Snapshot/Restore + type: bug + issues: + - 120518 + - pr: 120959 + summary: Remove unnecessary entitlement + area: Infra/Plugins + type: bug + issues: [] + - pr: 120974 + summary: Tweak `copy_to` handling in synthetic `_source` to account for nested objects + area: Mapping + type: bug + issues: + - 120831 + - pr: 120997 + summary: Allow `SSHA-256` for API key credential hash + area: Authentication + type: enhancement + issues: [] + - pr: 121048 + summary: Updating Inference Update API documentation to have the correct PUT method + area: Machine Learning + type: bug + issues: [] + - pr: 121049 + summary: Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. + area: Logs + type: breaking + issues: + - 106489 + breaking: + area: Logs + title: Conditionally enable logsdb by default + details: |- + Logsdb will be enabled by default for data streams matching with logs-*-* pattern. + If upgrading from 8.x to 9.x and data streams matching with log-*-* do exist, + then Logsdb will not be enabled by default. + impact: |- + Logsdb reduce storage footprint in Elasticsearch for logs, but there are side effects + to be taken into account that are described in the Logsdb docs: + https://www.elastic.co/guide/en/elasticsearch/reference/current/logs-data-stream.html#upgrade-to-logsdb-notes + notable: true + essSettingChange: false + - pr: 121074 + summary: Implement a `MetricsAware` interface + area: ES|QL + type: enhancement + issues: [] + - pr: 121105 + summary: Mark bbq indices as GA and add rolling upgrade integration tests + area: Vector Search + type: feature + issues: [] + - pr: 121109 + summary: Fix propagation of dynamic mapping parameter when applying `copy_to` + area: Mapping + type: bug + issues: + - 113049 + - pr: 121124 + summary: Run `TransportGetEnrichPolicyAction` on local node + area: Ingest Node + type: enhancement + issues: [] + - pr: 121156 + summary: Remove redundant sorts from execution plan + area: ES|QL + type: bug + issues: [] + - pr: 121193 + summary: Enable LOOKUP JOIN in non-snapshot builds + area: ES|QL + type: enhancement + issues: + - 121185 + highlight: + notable: true + title: "ES|QL LOOKUP JOIN is now available in technical preview" + body: |- + [LOOKUP JOIN](../reference/query-languages/esql/esql-commands.md) is now available in technical preview. LOOKUP JOIN combines data from your ES|QL queries with matching records from a lookup index, enabling you to: + + - Enrich your search results with reference data + - Speed up root-cause analysis and security investigations + - Join data across indices without complex queries + - Reduce operational overhead when correlating events + pr: 121193 + - pr: 121196 + summary: Fix geoip databases index access after system feature migration + area: Ingest Node + type: bug + issues: [] + - pr: 121207 + summary: "[Inference API] Put back legacy EIS URL setting" + area: Inference + type: bug + issues: [] + - pr: 121325 + summary: "`ReindexDataStreamIndex` bug in assertion caused by reference equality" + area: Data streams + type: bug + issues: [] + - pr: 121392 + summary: Include data streams when converting an existing resource to a system resource + area: Infra/Core + type: bug + issues: [] + - pr: 121396 + summary: Change format for Unified Chat + area: Machine Learning + type: bug + issues: [] + - pr: 121552 + summary: Fix a bug in TOP + area: ES|QL + type: bug + issues: [] + - pr: 121556 + summary: Enable New Semantic Text Format Only On Newly Created Indices + area: Mapping + type: bug + issues: [] + - pr: 121568 + summary: Analyze API to return 400 for wrong custom analyzer + area: Analysis + type: bug + issues: + - 121443 + - pr: 121667 + summary: Add deprecation warning for flush API + area: Machine Learning + type: deprecation + issues: + - 121506 + deprecation: + area: REST API + title: Add deprecation warning for flush API + details: "The anomaly detection job flush API is deprecated since it is only required for the post data API, which was deprecated since 7.11.0." + impact: "This should have a minimal impact on users as the flush API is only required for the post data API, which was deprecated since 7.11.0." + notable: false + essSettingChange: false + - pr: 121720 + summary: Skip fetching _inference_fields field in legacy semantic_text format + area: Search + type: bug + issues: [] + - pr: 121727 + summary: Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` + area: Downsampling + type: bug + issues: + - 119696 + - 96076 + - pr: 121731 + summary: Remove TLSv1.1 from default protocols + area: TLS + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Remove TLSv1.1 from default protocols + details: "TLSv1.1 is no longer enabled by default. Prior to version 9.0, Elasticsearch would attempt to enable TLSv1.1 if the JDK supported it. In most cases, including all cases where Elasticsearch 8 was running with the bundled JDK, the JDK would not support TLSv1.1, so that protocol would not be available in Elasticsearch. However, if Elasticsearch was running on an old JDK or a JDK that have been reconfigured to support TLSv1.1, then the protocol would automatically be available within Elasticsearch. As of Elasticsearch 9.0, this is no longer true. If you wish to enable TLSv1.1 then you must enable it within the JDK and also enable it within Elasticsearch by using the `ssl.supported_protocols` setting." + impact: "Most users will not be impacted. If your Elastisearch 8 cluster was using a custom JDK and you relied on TLSv1.1, then you will need to explicitly enable TLSv1.1 within Elasticsearch (as well as enabling it within your JDK)" + notable: false + essSettingChange: false + - pr: 121787 + summary: Added optional parameters to QSTR ES|QL function + area: Search + type: feature + issues: + - 120933 + - pr: 121793 + summary: ES|QL - Add scoring for full text functions disjunctions + area: ES|QL + type: enhancement + issues: [] + - pr: 121821 + summary: Fix get all inference endponts not returning multiple endpoints sharing model deployment + area: Machine Learning + type: bug + issues: [] + - pr: 121843 + summary: Fix async stop sometimes not properly collecting result + area: ES|QL + type: bug + issues: + - 121249 + - pr: 121850 + summary: Take named parameters for identifier and pattern out of snapshot + area: ES|QL + type: enhancement + issues: [] + - pr: 121971 + summary: Do not fetch reserved roles from native store when Get Role API is called + area: Authorization + type: enhancement + issues: [] + - pr: 122074 + summary: "If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias" + area: Transform + type: bug + issues: + - 121913 + - pr: 122246 + summary: Ensure removal of index blocks does not leave key with null value + area: Data streams + type: bug + issues: [] + - pr: 122250 + summary: "ESQL: Align `RENAME` behavior with `EVAL` for sequential processing" + area: ES|QL + type: enhancement + issues: + - 121739 + - pr: 122257 + summary: Revive inlinestats + area: ES|QL + type: bug + issues: [] + - pr: 122278 + summary: Fix serialising the inference update request + area: Machine Learning + type: bug + issues: [] + - pr: 122293 + summary: Add enterprise license check to inference action for semantic text fields + area: Machine Learning + type: bug + issues: [] + - pr: 122326 + summary: System Index Migration Failure Results in a Non-Recoverable State + area: Infra/Core + type: bug + issues: [] + - pr: 122357 + summary: Handle search timeout in `SuggestPhase` + area: Search + type: bug + issues: + - 122186 + - pr: 122409 + summary: Allow setting the `type` in the reroute processor + area: Ingest Node + type: enhancement + issues: + - 121553 + - pr: 122427 + summary: Improve size limiting string message + area: Infra/Core + type: enhancement + issues: [] + - pr: 122431 + summary: Upgrade AWS SDK to v1.12.746 + area: Snapshot/Restore + type: upgrade + issues: [] + - pr: 122575 + summary: Return an empty suggestion when suggest phase times out + area: Suggesters + type: bug + issues: + - 122548 + - pr: 122601 + summary: Implicit numeric casting for CASE/GREATEST/LEAST + area: ES|QL + type: bug + issues: + - 121890 + - pr: 122606 + summary: Avoid serializing empty `_source` fields in mappings + area: Mapping + type: bug + issues: [] + - pr: 122653 + summary: Knn vector rescoring to sort score docs + area: Vector Search + type: bug + issues: + - 119711 + - pr: 122731 + summary: Fork post-snapshot-delete cleanup off master thread + area: Snapshot/Restore + type: bug + issues: [] + - pr: 122762 + summary: "ESQL: Remove estimated row size assertion" + area: ES|QL + type: bug + issues: + - 121535 + - pr: 122886 + summary: Add support to VALUES aggregation for spatial types + area: ES|QL + type: bug + issues: + - 122413 + - pr: 122905 + summary: Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover + area: Data streams + type: bug + issues: [] + - pr: 122938 + summary: Fix geoip databases index access after system feature migration (again) + area: Ingest Node + type: bug + issues: [] + - pr: 122951 + summary: Updates the deprecation info API to not warn about system indices and data streams + area: Indices APIs + type: bug + issues: [] + - pr: 122960 + summary: Deprecate Behavioral Analytics CRUD apis + area: Search + type: deprecation + issues: [] + deprecation: + area: Search + title: Deprecate Behavioral Analytics CRUD apis + details: "Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. The APIs will still work for now, but will emit warning headers that the API has been deprecated." + impact: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. + notable: false + essSettingChange: false + - pr: 123010 + summary: Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` + area: Engine + type: bug + issues: + - 122974 + - pr: 123076 + summary: Retry on streaming errors + area: Machine Learning + type: bug + issues: [] + - pr: 123155 + summary: Add `ElasticInferenceServiceCompletionServiceSettings` + area: Machine Learning + type: bug + issues: [] + - pr: 123272 + summary: Set Connect Timeout to 5s + area: Machine Learning + type: bug + issues: [] + - pr: 123290 + summary: Fix Driver status iterations and `cpuTime` + area: ES|QL + type: enhancement + issues: + - 122967 + - pr: 123296 + summary: Avoid over collecting in Limit or Lucene Operator + area: ES|QL + type: bug + issues: [] + - pr: 123381 + summary: Push down `StartsWith` and `EndsWith` functions to Lucene + area: ES|QL + type: enhancement + issues: + - 123067 + - pr: 123427 + summary: Reduce iteration complexity for plan traversal + area: ES|QL + type: bug + issues: [] + - pr: 123492 + summary: Fix function registry concurrency issues on constructor + area: ES|QL + type: bug + issues: + - 123430 + - pr: 123589 + summary: Revive some more of inlinestats functionality + area: ES|QL + type: bug + issues: [] + - pr: 123600 + summary: Drop `TLS_RSA` cipher support for JDK 24 + area: TLS + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Drop `TLS_RSA` cipher support for JDK 24 + details: "This change removes `TLS_RSA` ciphers from the list of default supported ciphers, for Elasticsearch deployments running on JDK 24." + impact: "The dropped ciphers are `TLS_RSA_WITH_AES_256_GCM_SHA384`, `TLS_RSA_WITH_AES_128_GCM_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA`, and `TLS_RSA_WITH_AES_128_CBC_SHA`. TLS connections to Elasticsearch using these ciphers will no longer work. Please configure your clients to use one of supported cipher suites." + notable: false + essSettingChange: false + - pr: 123728 + summary: Allow skip shards with `_tier` and `_index` in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 123743 + summary: Adjust exception thrown when unable to load hunspell dict + area: Analysis + type: bug + issues: [] + - pr: 123761 + summary: Have create index return a bad request on poor formatting + area: Infra/Core + type: bug + issues: [] + - pr: 124001 + summary: Use a must boolean statement when pushing down to Lucene when scoring is also needed + area: ES|QL + type: bug + issues: + - 123967 + - pr: 124048 + summary: Handle long overflow in dates + area: Search + type: bug + issues: + - 112483 + - pr: 124149 + summary: Retry ILM async action after reindexing data stream + area: Data streams + type: enhancement + issues: [] + - pr: 124225 + summary: "[Inference API] Fix output stream ordering in `InferenceActionProxy`" + area: Machine Learning + type: bug + issues: [] + - pr: 124335 + summary: Change the order of the optimization rules + area: ES|QL + type: bug + issues: [] + - pr: 124363 + summary: Set cause on create index request in create from action + area: Data streams + type: enhancement + issues: [] + - pr: 124424 + summary: Lazy collection copying during node transform + area: ES|QL + type: bug + issues: [] + - pr: 124446 + summary: "ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate`" + area: ES|QL + type: bug + issues: + - 124311 + - pr: 124451 + summary: Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. + area: Downsampling + type: bug + issues: [] + - pr: 124477 + summary: Improve downsample performance by buffering docids and do bulk processing + area: Downsampling + type: enhancement + issues: [] + - pr: 124527 + summary: Avoid potentially throwing calls to Task#getDescription in model download + area: Machine Learning + type: bug + issues: [] + - pr: 124540 + summary: "ES|QL: Fix scoring for full text functions" + area: ES|QL + type: bug + issues: [] + - pr: 124604 + summary: Fix geoip databases index access after system feature migration (take 3) + area: Ingest Node + type: bug + issues: [] + - pr: 124611 + summary: Reuse child `outputSet` inside the plan where possible + area: ES|QL + type: enhancement + issues: [] + - pr: 124651 + summary: Fix system data streams to be restorable from a snapshot + area: Infra/Core + type: bug + issues: + - 89261 + - pr: 124662 + summary: Let MLTQuery throw IAE when no analyzer is set + area: Search + type: bug + issues: + - 124562 + - pr: 124669 + summary: Release semantic_text as a GA feature + area: Mapping + type: feature + issues: [] + highlight: + notable: true + title: The semantic_text field type is now GA + body: "[`semantic_text`](../reference/elasticsearch/mapping-reference/semantic-text.md) is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time." + pr: 124669 + - pr: 124739 + summary: Improve rolling up metrics + area: Downsampling + type: enhancement + issues: [] + - pr: 124782 + summary: "Aggs: Let terms run in global ords mode no match" + area: Aggregations + type: bug + issues: [] + - pr: 124784 + summary: Merge template mappings properly during validation + area: Mapping + type: bug + issues: + - 123372 + - pr: 124843 + summary: Ignore _JAVA_OPTIONS + area: Infra/CLI + type: enhancement + issues: [] + - pr: 124865 + summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." + area: Infra/Core + type: upgrade + issues: [] + - pr: 124873 + summary: Restore V8 REST compatibility around highlight `force_source` parameter + area: Highlighting + type: bug + issues: [] + - pr: 124884 + summary: System data streams are not being upgraded in the feature migration API + area: Infra/Core + type: bug + issues: + - 122949 + - pr: 124931 + summary: This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. + area: Snapshot/Restore + type: bug + issues: [] + - pr: 124936 + summary: Indicate when errors represent timeouts + area: Infra/REST API + type: enhancement + issues: [] + - pr: 125023 + summary: Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` + area: Machine Learning + type: bug + issues: [] + - pr: 125103 + summary: Fix LTR query feature with phrases (and two-phase) queries + area: Ranking + type: bug + issues: [] + - pr: 125159 + summary: Update bundled JDK to Java 24 + area: Packaging + type: upgrade + issues: [] + - pr: 125171 + summary: Reindex data stream indices on different nodes + area: Data streams + type: enhancement + issues: [] + - pr: 125345 + summary: ESQL - date nanos range bug? + area: ES|QL + type: bug + issues: + - 125439 + - pr: 125352 + summary: Fix NPE in rolling over unknown target and return 404 + area: Indices APIs + type: bug + issues: [] + - pr: 125370 + summary: Set default similarity for Cohere model to cosine + area: Machine Learning + type: bug + issues: + - 122878 + - pr: 125404 + summary: Check if the anomaly results index has been rolled over + area: Machine Learning + type: upgrade + issues: [] + - pr: 125446 + summary: Fix Semantic Text 8.x Upgrade Bug + area: Mapping + type: bug + issues: [] + - pr: 125595 + summary: Esql - Fix lucene push down behavior when a range contains nanos and millis + area: ES|QL + type: bug + issues: [] + - pr: 125606 + summary: Rename deprecation index template + area: Infra/Logging + type: breaking + issues: + - 125445 + breaking: + area: Logging + title: Rename deprecation index template + details: "The deprecation datastream contains log entries for deprecations that occured while Elasticsearch is running. The deprecation log entries had a mismatch in their dataset name. In order to avoid changing the dataset name in the existing datastream, a new datastream now exists." + impact: "If querying for deprecations previously using the `.logs-deprecation.elasticsearch-default` datastream, you should now use the `.logs-elasticsearch.deprecation-default` datastream." + notable: false + essSettingChange: false + - pr: 125636 + summary: Make `numberOfChannels` consistent with layout map by removing duplicated `ChannelSet` + area: ES|QL + type: bug + issues: [] + - pr: 125650 + summary: Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` + area: Search + type: bug + issues: + - 125483 + - pr: 125659 + summary: Non existing synonyms sets do not fail shard recovery for indices + area: Analysis + type: bug + issues: + - 125603 + - pr: 125666 + summary: Minor-Fixes Support 7x segments as archive in 8x / 9x + area: Search + type: bug + issues: [] + - pr: 125716 + summary: Return appropriate error on null dims update instead of npe + area: Vector Search + type: bug + issues: [] + - pr: 125732 + summary: Log stack traces on data nodes before they are cleared for transport + area: Search + type: bug + issues: [] + - pr: 125764 + summary: Fix `ReplaceMissingFieldsWithNull` + area: ES|QL + type: bug + issues: + - 126036 + - 121754 + - 126030 + - pr: 125881 + summary: Fixes a invalid warning from being issued when restoring a system data stream from a snapshot. + area: Data streams + type: bug + issues: [] + - pr: 125916 + summary: Re-enable parallel collection for field sorted top hits + area: Search + type: bug + issues: [] + - pr: 126077 + summary: Preventing `ConcurrentModificationException` when updating settings for more than one index + area: Indices APIs + type: bug + issues: [] + - pr: 126191 + summary: Fix NPE for missing Content Type header in OIDC Authenticator + area: Authentication + type: bug + issues: [] diff --git a/docs/release-notes/changelog-bundles/9.0.1.yml b/docs/release-notes/changelog-bundles/9.0.1.yml new file mode 100644 index 0000000000000..92394783847ec --- /dev/null +++ b/docs/release-notes/changelog-bundles/9.0.1.yml @@ -0,0 +1,148 @@ +version: 9.0.1 +released: false +generated: 2025-05-02T19:34:24.151692Z +changelogs: + - pr: 125694 + summary: LTR score bounding + area: Ranking + type: bug + issues: [] + - pr: 125922 + summary: Fix text structure NPE when fields in list have null value + area: Machine Learning + type: bug + issues: [] + - pr: 126273 + summary: Fix LTR rescorer with model alias + area: Ranking + type: bug + issues: [] + - pr: 126310 + summary: Add Issuer to failed SAML Signature validation logs when available + area: Security + type: enhancement + issues: + - 111022 + - pr: 126342 + summary: Enable sort optimization on float and `half_float` + area: Search + type: enhancement + issues: [] + - pr: 126583 + summary: Cancel expired async search task when a remote returns its results + area: CCS + type: bug + issues: [] + - pr: 126605 + summary: Fix equality bug in `WaitForIndexColorStep` + area: ILM+SLM + type: bug + issues: [] + - pr: 126614 + summary: Fix join masking eval + area: ES|QL + type: bug + issues: [] + - pr: 126637 + summary: Improve resiliency of `UpdateTimeSeriesRangeService` + area: TSDB + type: bug + issues: [] + - pr: 126686 + summary: Fix race condition in `RestCancellableNodeClient` + area: Task Management + type: bug + issues: + - 88201 + - pr: 126729 + summary: Use terminal reader in keystore add command + area: Infra/CLI + type: bug + issues: + - 98115 + - pr: 126778 + summary: Fix bbq quantization algorithm but for differently distributed components + area: Vector Search + type: bug + issues: [] + - pr: 126783 + summary: Fix shard size of initializing restored shard + area: Allocation + type: bug + issues: + - 105331 + - pr: 126806 + summary: Workaround max name limit imposed by Jackson 2.17 + area: Infra/Core + type: bug + issues: [] + - pr: 126850 + summary: "[otel-data] Bump plugin version to release _metric_names_hash changes" + area: Data streams + type: bug + issues: [] + - pr: 126852 + summary: "Validation checks on paths allowed for 'files' entitlements. Restrict the paths we allow access to, forbidding plugins to specify/request entitlements for reading or writing to specific protected directories." + area: Infra/Core + type: enhancement + issues: [] + - pr: 126858 + summary: Leverage threadpool schedule for inference api to avoid long running thread + area: Machine Learning + type: bug + issues: + - 126853 + - pr: 126884 + summary: Rare terms aggregation false **positive** fix + area: Aggregations + type: bug + issues: [] + - pr: 126889 + summary: Rework uniquify to not use iterators + area: Infra/Core + type: bug + issues: + - 126883 + - pr: 126911 + summary: Fix `vec_caps` to test for OS support too (on x64) + area: Vector Search + type: bug + issues: + - 126809 + - pr: 126930 + summary: Adding missing `onFailure` call for Inference API start model request + area: Machine Learning + type: bug + issues: [] + - pr: 126990 + summary: "Fix: consider case sensitiveness differences in Windows/Unix-like filesystems for files entitlements" + area: Infra/Core + type: bug + issues: + - 127047 + - pr: 127146 + summary: Fix sneaky bug in single value query + area: ES|QL + type: bug + issues: [] + - pr: 127225 + summary: Fix count optimization with pushable union types + area: ES|QL + type: bug + issues: + - 127200 + - pr: 127353 + summary: Updating tika to 2.9.3 + area: Ingest Node + type: upgrade + issues: [] + - pr: 127414 + summary: Fix npe when using source confirmed text query against missing field + area: Search + type: bug + issues: [] + - pr: 127527 + summary: "No, line noise isn't a valid ip" + area: ES|QL + type: bug + issues: [] diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index b835466a91452..1cb9fe15363b7 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -16,6 +16,13 @@ To give you insight into what deprecated features you’re using, {{es}}: % ## Next version [elasticsearch-nextversion-deprecations] +```{applies_to} +stack: coming 9.0.1 +``` +## 9.0.1 [elasticsearch-9.0.1-deprecations] + +No deprecations in this version. + ## 9.0.0 [elasticsearch-900-deprecations] ES|QL: diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index c94ab727dcaea..b38ee35860e73 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -1,6 +1,7 @@ --- navigation_title: "Elasticsearch" mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html --- @@ -20,19 +21,90 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * +```{applies_to} +stack: coming 9.0.1 +``` +## 9.0.1 [elasticsearch-9.0.1-release-notes] + +### Features and enhancements [elasticsearch-9.0.1-features-enhancements] + +Infra/Core: +* Validation checks on paths allowed for 'files' entitlements. Restrict the paths we allow access to, forbidding plugins to specify/request entitlements for reading or writing to specific protected directories. [#126852](https://github.com/elastic/elasticsearch/pull/126852) + +Ingest Node: +* Updating tika to 2.9.3 [#127353](https://github.com/elastic/elasticsearch/pull/127353) + +Search: +* Enable sort optimization on float and `half_float` [#126342](https://github.com/elastic/elasticsearch/pull/126342) + +Security: +* Add Issuer to failed SAML Signature validation logs when available [#126310](https://github.com/elastic/elasticsearch/pull/126310) (issue: [#111022](https://github.com/elastic/elasticsearch/issues/111022)) + +### Fixes [elasticsearch-9.0.1-fixes] + +Aggregations: +* Rare terms aggregation false **positive** fix [#126884](https://github.com/elastic/elasticsearch/pull/126884) + +Allocation: +* Fix shard size of initializing restored shard [#126783](https://github.com/elastic/elasticsearch/pull/126783) (issue: [#105331](https://github.com/elastic/elasticsearch/issues/105331)) + +CCS: +* Cancel expired async search task when a remote returns its results [#126583](https://github.com/elastic/elasticsearch/pull/126583) + +Data streams: +* [otel-data] Bump plugin version to release _metric_names_hash changes [#126850](https://github.com/elastic/elasticsearch/pull/126850) + +ES|QL: +* Fix count optimization with pushable union types [#127225](https://github.com/elastic/elasticsearch/pull/127225) (issue: [#127200](https://github.com/elastic/elasticsearch/issues/127200)) +* Fix join masking eval [#126614](https://github.com/elastic/elasticsearch/pull/126614) +* Fix sneaky bug in single value query [#127146](https://github.com/elastic/elasticsearch/pull/127146) +* No, line noise isn't a valid ip [#127527](https://github.com/elastic/elasticsearch/pull/127527) + +ILM+SLM: +* Fix equality bug in `WaitForIndexColorStep` [#126605](https://github.com/elastic/elasticsearch/pull/126605) + +Infra/CLI: +* Use terminal reader in keystore add command [#126729](https://github.com/elastic/elasticsearch/pull/126729) (issue: [#98115](https://github.com/elastic/elasticsearch/issues/98115)) + +Infra/Core: +* Fix: consider case sensitiveness differences in Windows/Unix-like filesystems for files entitlements [#126990](https://github.com/elastic/elasticsearch/pull/126990) (issue: [#127047](https://github.com/elastic/elasticsearch/issues/127047)) +* Rework uniquify to not use iterators [#126889](https://github.com/elastic/elasticsearch/pull/126889) (issue: [#126883](https://github.com/elastic/elasticsearch/issues/126883)) +* Workaround max name limit imposed by Jackson 2.17 [#126806](https://github.com/elastic/elasticsearch/pull/126806) + +Machine Learning: +* Adding missing `onFailure` call for Inference API start model request [#126930](https://github.com/elastic/elasticsearch/pull/126930) +* Fix text structure NPE when fields in list have null value [#125922](https://github.com/elastic/elasticsearch/pull/125922) +* Leverage threadpool schedule for inference api to avoid long running thread [#126858](https://github.com/elastic/elasticsearch/pull/126858) (issue: [#126853](https://github.com/elastic/elasticsearch/issues/126853)) + +Ranking: +* Fix LTR rescorer with model alias [#126273](https://github.com/elastic/elasticsearch/pull/126273) +* LTR score bounding [#125694](https://github.com/elastic/elasticsearch/pull/125694) + +Search: +* Fix npe when using source confirmed text query against missing field [#127414](https://github.com/elastic/elasticsearch/pull/127414) + +TSDB: +* Improve resiliency of `UpdateTimeSeriesRangeService` [#126637](https://github.com/elastic/elasticsearch/pull/126637) + +Task Management: +* Fix race condition in `RestCancellableNodeClient` [#126686](https://github.com/elastic/elasticsearch/pull/126686) (issue: [#88201](https://github.com/elastic/elasticsearch/issues/88201)) + +Vector Search: +* Fix `vec_caps` to test for OS support too (on x64) [#126911](https://github.com/elastic/elasticsearch/pull/126911) (issue: [#126809](https://github.com/elastic/elasticsearch/issues/126809)) +* Fix bbq quantization algorithm but for differently distributed components [#126778](https://github.com/elastic/elasticsearch/pull/126778) + + ## 9.0.0 [elasticsearch-900-release-notes] ### Highlights [elasticsearch-900-highlights] ::::{dropdown} rank_vectors field type is now available for late-interaction ranking - [`rank_vectors`](../reference/elasticsearch/mapping-reference/rank-vectors.md) is a new field type released as an experimental feature in Elasticsearch 9.0. It is designed to be used with dense vectors and allows for late-interaction second order ranking. Late-interaction models are powerful rerankers. While their size and overall cost doesn’t lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali. :::: ::::{dropdown} ES|QL LOOKUP JOIN is now available in technical preview - [LOOKUP JOIN](../reference/query-languages/esql/esql-commands.md) is now available in technical preview. LOOKUP JOIN combines data from your ES|QL queries with matching records from a lookup index, enabling you to: - Enrich your search results with reference data