Make better-fors a preview feature in 3.7 (#22776)

Makes the SIP-62 better fors a preview feature in Scala 3.7. Shall be
stabilised no earlier then Scala 3.8

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -1953,9 +1953,9 @@ object desugar {
     /** Create tree for for-comprehension `<for (enums) do body>` or
      *   `<for (enums) yield body>` where mapName and flatMapName are chosen
      *  corresponding to whether this is a for-do or a for-yield.
-     *  If sourceVersion >= 3.7 are enabled, the creation performs the following rewrite rules:
+     *  If betterFors are enabled, the creation performs the following rewrite rules:
      *
-     *  1. if sourceVersion >= 3.7:
+     *  1. if betterFors is enabled:
      *
      *    for () do E  ==>  E
      *      or
@@ -1986,13 +1986,13 @@ object desugar {
      *      ==>
      *    for (P <- G.withFilter (P => E); ...) ...
      *
-     *  6. For any N, if sourceVersion >= 3.7:
+     *  6. For any N, if betterFors is enabled:
      *
      *    for (P <- G; P_1 = E_1; ... P_N = E_N; P1 <- G1; ...) ...
      *      ==>
      *    G.flatMap (P => for (P_1 = E_1; ... P_N = E_N; ...))
      *
-     *  7. For any N, if sourceVersion >= 3.7:
+     *  7. For any N, if betterFors is enabled:
      *
      *    for (P <- G; P_1 = E_1; ... P_N = E_N) ...
      *      ==>
@@ -2013,7 +2013,7 @@ object desugar {
      *    If any of the P_i are variable patterns, the corresponding `x_i @ P_i` is not generated
      *    and the variable constituting P_i is used instead of x_i
      *
-     *  9. For any N, if sourceVersion >= 3.7:
+     *  9. For any N, if betterFors is enabled:
      *
      *    for (P_1 = E_1; ... P_N = E_N; ...)
      *      ==>
@@ -2157,15 +2157,15 @@ object desugar {
           case _ => false
 
       def markTrailingMap(aply: Apply, gen: GenFrom, selectName: TermName): Unit =
-        if sourceVersion.isAtLeast(`3.7`)
+        if sourceVersion.enablesBetterFors
           && selectName == mapName
           && gen.checkMode != GenCheckMode.Filtered // results of withFilter have the wrong type
           && (deepEquals(gen.pat, body) || deepEquals(body, Tuple(Nil)))
         then
           aply.putAttachment(TrailingForMap, ())
 
       enums match {
-        case Nil if sourceVersion.isAtLeast(`3.7`) => body
+        case Nil if sourceVersion.enablesBetterFors => body
         case (gen: GenFrom) :: Nil =>
           val aply = Apply(rhsSelect(gen, mapName), makeLambda(gen, body))
           markTrailingMap(aply, gen, mapName)
@@ -2174,7 +2174,7 @@ object desugar {
           val cont = makeFor(mapName, flatMapName, rest, body)
           Apply(rhsSelect(gen, flatMapName), makeLambda(gen, cont))
         case (gen: GenFrom) :: rest
-        if sourceVersion.isAtLeast(`3.7`)
+        if sourceVersion.enablesBetterFors
           && rest.dropWhile(_.isInstanceOf[GenAlias]).headOption.forall(e => e.isInstanceOf[GenFrom]) // possible aliases followed by a generator or end of for
           && !rest.takeWhile(_.isInstanceOf[GenAlias]).exists(a => isNestedGivenPattern(a.asInstanceOf[GenAlias].pat)) =>
           val cont = makeFor(mapName, flatMapName, rest, body)
@@ -2202,9 +2202,9 @@ object desugar {
           makeFor(mapName, flatMapName, vfrom1 :: rest1, body)
         case (gen: GenFrom) :: test :: rest =>
           val filtered = Apply(rhsSelect(gen, nme.withFilter), makeLambda(gen, test))
-          val genFrom = GenFrom(gen.pat, filtered, if sourceVersion.isAtLeast(`3.7`) then GenCheckMode.Filtered else GenCheckMode.Ignore)
+          val genFrom = GenFrom(gen.pat, filtered, if sourceVersion.enablesBetterFors then GenCheckMode.Filtered else GenCheckMode.Ignore)
           makeFor(mapName, flatMapName, genFrom :: rest, body)
-        case GenAlias(_, _) :: _ if sourceVersion.isAtLeast(`3.7`) =>
+        case GenAlias(_, _) :: _ if sourceVersion.enablesBetterFors =>
           val (valeqs, rest) = enums.span(_.isInstanceOf[GenAlias])
           val pats = valeqs.map { case GenAlias(pat, _) => pat }
           val rhss = valeqs.map { case GenAlias(_, rhs) => rhs }

compiler/src/dotty/tools/dotc/config/SourceVersion.scala

@@ -3,6 +3,8 @@ package dotc
 package config
 
 import core.Decorators.*
+import core.Contexts.*
+import Feature.isPreviewEnabled
 import util.Property
 
 enum SourceVersion:
@@ -35,6 +37,7 @@ enum SourceVersion:
   def enablesClauseInterleaving = isAtLeast(`3.6`)
   def enablesNewGivens = isAtLeast(`3.6`)
   def enablesNamedTuples = isAtLeast(`3.7`)
+  def enablesBetterFors(using Context) = isAtLeast(`3.7`) && isPreviewEnabled
 
 object SourceVersion extends Property.Key[SourceVersion]:
   def defaultSourceVersion = `3.7`

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -2956,7 +2956,7 @@ object Parsers {
     /** Enumerators ::= Generator {semi Enumerator | Guard}
      */
     def enumerators(): List[Tree] =
-      if sourceVersion.isAtLeast(`3.7`) then
+      if sourceVersion.enablesBetterFors then
         aliasesUntilGenerator() ++ enumeratorsRest()
       else
         generator() :: enumeratorsRest()

compiler/test/dotty/tools/debug/DebugTests.scala

@@ -18,7 +18,8 @@ class DebugTests:
     implicit val testGroup: TestGroup = TestGroup("debug")
     CompilationTest.aggregateTests(
       compileFile("tests/debug-custom-args/eval-explicit-nulls.scala", TestConfiguration.explicitNullsOptions),
-      compileFilesInDir("tests/debug", TestConfiguration.defaultOptions)
+      compileFilesInDir("tests/debug", TestConfiguration.defaultOptions),
+      compileFilesInDir("tests/debug-preview", TestConfiguration.defaultOptions.and("-preview"))
     ).checkDebug()
 
 object DebugTests extends ParallelTesting:

docs/_docs/reference/changed-features/better-fors.md → docs/_docs/reference/preview/better-fors.md

@@ -1,10 +1,10 @@
 ---
 layout: doc-page
 title: "Better fors"
-nightlyOf: https://docs.scala-lang.org/scala3/reference/changed-features/better-fors.html
+nightlyOf: https://docs.scala-lang.org/scala3/reference/preview/better-fors.html
 ---
 
-Starting in Scala `3.7`, the usability of `for`-comprehensions is improved.
+Starting in Scala `3.7` under `-preview` mode, the usability of `for`-comprehensions is improved.
 
 The biggest user facing change is the new ability to start `for`-comprehensions with aliases. This means that the following previously invalid code is now valid:
 

docs/_docs/reference/preview/overview.md

@@ -0,0 +1,24 @@
+---
+layout: doc-page
+title: "Preview"
+nightlyOf: https://docs.scala-lang.org/scala3/reference/preview/overview.html
+---
+
+## Preview language features
+
+New Scala language features or standard library APIs are initially introduced as experimental, but once they become fully implemented and accepted by the [SIP](https://docs.scala-lang.org/sips/) these can become a preview features.
+
+Preview language features and APIs are guaranteed to be standardized in some next Scala minor release, but allow the compiler team to introduce small, possibly binary incompatible, changes based on the community feedback.
+These can be used by early adopters who can accept the possibility of binary compatibility breakage. For instance, preview features could be used in some internal tool or application. On the other hand, preview features are discouraged in publicly available libraries.
+
+More information about preview featues can be found in [preview defintions guide](../other-new-features/preview-defs.md)
+
+### `-preview` compiler flag
+
+This flag enables the use of all preview language feature in the project.
+
+
+## List of available preview features
+
+* [`better-fors`](./better-fors.md): Enables new for-comprehension behaviour under SIP-62 under `-source:3.7` or later
+

docs/sidebar.yml

@@ -116,7 +116,6 @@ subsection:
           - page: reference/changed-features/lazy-vals-init.md
           - page: reference/changed-features/main-functions.md
           - page: reference/changed-features/interpolation-escapes.md
-          - page: reference/changed-features/better-fors.md
       - title: Dropped Features
         index: reference/dropped-features/dropped-features.md
         subsection:
@@ -140,6 +139,11 @@ subsection:
           - page: reference/dropped-features/nonlocal-returns.md
           - page: reference/dropped-features/this-qualifier.md
           - page: reference/dropped-features/wildcard-init.md
+      - title: Preview Features
+        directory: preview
+        index: reference/preview/overview.md
+        subsection:
+          - page: reference/preview/better-fors.md
       - title: Experimental Features
         directory: experimental
         index: reference/experimental/overview.md

library/src/scala/runtime/stdLibPatches/language.scala

@@ -140,7 +140,7 @@ object language:
      * @see [[https://github.com/scala/improvement-proposals/pull/79]]
      */
     @compileTimeOnly("`betterFors` can only be used at compile time in import statements")
-    @deprecated("The `experimental.betterFors` language import is no longer needed since the feature is now standard", since = "3.7")
+    @deprecated("The `experimental.betterFors` language import no longer has any effect, the feature is being stablised and can be enabled using `-preview` flag", since = "3.7")
     object betterFors
 
     /** Experimental support for package object values

presentation-compiler/test/dotty/tools/pc/tests/tokens/SemanticTokensSuite.scala

@@ -1,10 +1,14 @@
 package dotty.tools.pc.tests.tokens
 
 import dotty.tools.pc.base.BaseSemanticTokensSuite
+import java.nio.file.Path
 
 import org.junit.Test
 
 class SemanticTokensSuite extends BaseSemanticTokensSuite:
+  // -preview required for `for-comprehension` test
+  override protected def scalacOptions(classpath: Seq[Path]): Seq[String] =
+    super.scalacOptions(classpath) ++ Seq("-preview")
 
   @Test def `class, object, var, val(readonly), method, type, parameter, String(single-line)` =
     check(

tests/debug-preview/eval-in-for-comprehension.check

@@ -0,0 +1,27 @@
+break Test$ 5 // in main
+eval list(0)
+result 1
+// TODO can we remove debug line in adapted methods?
+break Test$ 5 // in main$$anonfun$adapted$1
+break Test$ 6 // in main$$anonfun$1
+eval list(0)
+result 1
+eval x
+result 1
+break Test$ 7 // in main$$anonfun$1$$anonfun$1
+eval x + y
+result 2
+
+break Test$ 11 // in main$$anonfun$2
+eval x
+result 1
+
+break Test$ 13 // in main
+eval list(0)
+result 1
+break Test$ 13 // in main$$anonfun$4
+
+break Test$ 14 // in main
+eval list(0)
+result 1
+break Test$ 14 // in main$$anonfun$5

tests/debug-preview/eval-in-for-comprehension.scala

@@ -0,0 +1,14 @@
+object Test:
+  def main(args: Array[String]): Unit =
+    val list = List(1)
+    for
+      x <- list
+      y <- list
+      z = x + y
+    yield x
+    for
+      x <- list
+      if x == 1
+    yield x
+    for x <- list yield x
+    for x <- list do println(x)

tests/debug/eval-in-for-comprehension.check

@@ -8,9 +8,16 @@ eval list(0)
 result 1
 eval x
 result 1
+break Test$ 6 // in main$$anonfun$1$$anonfun$adapted$1
 break Test$ 7 // in main$$anonfun$1$$anonfun$1
 eval x + y
 result 2
+// TODO this line position does not make any sense
+break Test$ 6 // in main$$anonfun$1$$anonfun$1
+break Test$ 7 // in main$$anonfun$1$$anonfun$1
+break Test$ 6 // in main$$anonfun$1$$anonfun$2
+break Test$ 6 // in main$$anonfun$1$$anonfun$2
+break Test$ 7 // in main$$anonfun$1$$anonfun$2
 
 break Test$ 11 // in main$$anonfun$2
 eval x

tests/pos/better-fors-given.scala

@@ -1,3 +1,5 @@
+//> using options -preview
+
 @main def Test: Unit =
   for
     x <- Option(23 -> "abc")

tests/pos/better-fors-i21804.scala

@@ -1,4 +1,5 @@
-import scala.language.experimental.betterFors
+//> using options -preview
+// import scala.language.experimental.betterFors
 
 case class Container[A](val value: A) {
   def map[B](f: A => B): Container[B] = Container(f(value))

tests/run/better-fors-map-elim.scala

@@ -1,4 +1,5 @@
-import scala.language.experimental.betterFors
+//> using options -preview
+// import scala.language.experimental.betterFors
 
 class myOptionModule(doOnMap: => Unit) {
   sealed trait MyOption[+A] {

tests/run/better-fors.scala

@@ -1,3 +1,6 @@
+//> using options -preview
+// import scala.language.experimental.betterFors
+
 def for1 =
   for {
     a = 1

tests/run/fors.scala

@@ -1,3 +1,4 @@
+//> using options -preview
 //############################################################################
 // for-comprehensions (old and new syntax)
 //############################################################################

tests/semanticdb/expect/ForComprehension.expect.scala

@@ -3,43 +3,43 @@ package example
 class ForComprehension/*<-example::ForComprehension#*/ {
   for {
     a/*<-local0*/ <- List/*->scala::package.List.*/(1)
-    b/*<-local1*/ <- List/*->scala::package.List.*/(1)
+    b/*<-local1*//*->local1*/ <- List/*->scala::package.List.*/(1)
     if b/*->local1*/ >/*->scala::Int#`>`(+3).*/ 1
-    c/*<-local2*/ = a/*->local0*/ +/*->scala::Int#`+`(+4).*/ b/*->local1*/
+    c/*<-local2*//*->local2*/ = a/*->local0*/ +/*->scala::Int#`+`(+4).*/ b/*->local1*/
   } yield (a/*->local0*/, b/*->local1*/, c/*->local2*/)
   for {
-    a/*<-local3*/ <- List/*->scala::package.List.*/(1)
-    b/*<-local4*/ <- List/*->scala::package.List.*/(a/*->local3*/)
+    a/*<-local4*/ <- List/*->scala::package.List.*/(1)
+    b/*<-local5*/ <- List/*->scala::package.List.*/(a/*->local4*/)
     if (
-      a/*->local3*/,
-      b/*->local4*/
+      a/*->local4*/,
+      b/*->local5*/
     ) ==/*->scala::Any#`==`().*/ (1, 2)
     (
-      c/*<-local6*/,
-      d/*<-local7*/
-    ) <- List/*->scala::package.List.*/((a/*->local3*/, b/*->local4*/))
+      c/*<-local7*/,
+      d/*<-local8*/
+    ) <- List/*->scala::package.List.*/((a/*->local4*/, b/*->local5*/))
     if (
-      a/*->local3*/,
-      b/*->local4*/,
-      c/*->local6*/,
-      d/*->local7*/
+      a/*->local4*/,
+      b/*->local5*/,
+      c/*->local7*/,
+      d/*->local8*/
     ) ==/*->scala::Any#`==`().*/ (1, 2, 3, 4)
-    e/*<-local8*//*->local8*/ = (
-      a/*->local3*/,
-      b/*->local4*/,
-      c/*->local6*/,
-      d/*->local7*/
+    e/*<-local9*//*->local9*/ = (
+      a/*->local4*/,
+      b/*->local5*/,
+      c/*->local7*/,
+      d/*->local8*/
     )
-    if e/*->local8*/ ==/*->scala::Any#`==`().*/ (1, 2, 3, 4)
-    f/*<-local9*/ <- List/*->scala::package.List.*/(e/*->local8*/)
+    if e/*->local9*/ ==/*->scala::Any#`==`().*/ (1, 2, 3, 4)
+    f/*<-local10*/ <- List/*->scala::package.List.*/(e/*->local9*/)
   } yield {
     (
-      a/*->local3*/,
-      b/*->local4*/,
-      c/*->local6*/,
-      d/*->local7*/,
-      e/*->local8*/,
-      f/*->local9*/
+      a/*->local4*/,
+      b/*->local5*/,
+      c/*->local7*/,
+      d/*->local8*/,
+      e/*->local9*/,
+      f/*->local10*/
     )
   }
 }