Backports for 4.5 from 5.0 (j-mie6#233)

This PR backports a number of fixes from the 5.0 series. Fixes j-mie6#230,
fixes j-mie6#229, fixes j-mie6#226, and fixes j-mie6#227

Author: j-mie6

parsley/shared/src/main/scala/parsley/internal/machine/errors/DefuncError.scala

@@ -357,7 +357,9 @@ private [parsley] final class EmptyErrorWithReason(val presentationOffset: Int,
 }
 
 private [errors] final class TrivialMergedErrors private [errors] (val err1: TrivialDefuncError, val err2: TrivialDefuncError) extends TrivialDefuncError {
-    override final val flags = err1.flags & err2.flags
+    // FIXME: this is horrid, split out the flags at this point, we'll do 16-bit and 8-bit
+    override final val flags = scala.math.max(err1.entrenchedBy, err2.entrenchedBy) |
+                               (err1.flags & err2.flags & ~DefuncError.EntrenchedMask)
     assume(err1.underlyingOffset == err2.underlyingOffset, "two errors only merge when they have matching offsets")
     override val underlyingOffset = err1.underlyingOffset
     assume(err1.presentationOffset == err2.presentationOffset, "two errors only merge when they have matching offsets")
@@ -369,7 +371,9 @@ private [errors] final class TrivialMergedErrors private [errors] (val err1: Tri
 }
 
 private [errors] final class FancyMergedErrors private [errors] (val err1: FancyDefuncError, val err2: FancyDefuncError) extends FancyDefuncError {
-    override final val flags = err1.flags & err2.flags
+    // FIXME: this is horrid, split out the flags at this point, we'll do 16-bit and 8-bit
+    override final val flags = scala.math.max(err1.entrenchedBy, err2.entrenchedBy) |
+                               (err1.flags & err2.flags & ~DefuncError.EntrenchedMask)
     assume(err1.underlyingOffset == err2.underlyingOffset, "two errors only merge when they have matching offsets")
     override val underlyingOffset = err1.underlyingOffset
     assume(err1.presentationOffset == err2.presentationOffset, "two errors only merge when they have matching offsets")

parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala

@@ -11,15 +11,15 @@ import parsley.token.errors.LabelConfig
 import parsley.token.predicate
 
 import parsley.internal.collection.immutable.Trie
-import parsley.internal.errors.ExpectDesc
+import parsley.internal.errors.{ExpectDesc, ExpectItem}
 import parsley.internal.machine.Context
 import parsley.internal.machine.XAssert._
 import parsley.internal.machine.instructions.Instr
 
 private [token] abstract class Specific extends Instr {
     protected val specific: String
     protected val caseSensitive: Boolean
-    protected val expected: Iterable[ExpectDesc]
+    protected val expected: Iterable[ExpectItem]
     private [this] final val strsz = specific.length
     private [this] final val numCodePoints = specific.codePointCount(0, strsz)
 
@@ -65,12 +65,13 @@ private [token] abstract class Specific extends Instr {
 }
 
 private [internal] final class SoftKeyword(protected val specific: String, letter: CharPredicate, protected val caseSensitive: Boolean,
-                                           protected val expected: Iterable[ExpectDesc], expectedEnd: Iterable[ExpectDesc]) extends Specific {
+                                           protected val expected: Iterable[ExpectItem], protected val reason: Option[String],
+                                           expectedEnd: Iterable[ExpectDesc]) extends Specific {
     def this(specific: String, letter: predicate.CharPredicate, caseSensitive: Boolean, expected: LabelConfig, expectedEnd: String) = {
         this(if (caseSensitive) specific else specific.toLowerCase,
              letter.asInternalPredicate,
              caseSensitive,
-             expected.asExpectDescs, Some(new ExpectDesc(expectedEnd)))
+             expected.asExpectItems(specific), expected.asReason, Some(new ExpectDesc(expectedEnd)))
     }
 
     protected def postprocess(ctx: Context): Unit = {
@@ -90,9 +91,10 @@ private [internal] final class SoftKeyword(protected val specific: String, lette
 }
 
 private [internal] final class SoftOperator(protected val specific: String, letter: CharPredicate, ops: Trie[Unit],
-                                            protected val expected: Iterable[ExpectDesc], expectedEnd: Iterable[ExpectDesc]) extends Specific {
+                                            protected val expected: Iterable[ExpectItem], protected val reason: Option[String],
+                                            expectedEnd: Iterable[ExpectDesc]) extends Specific {
     def this(specific: String, letter: predicate.CharPredicate, ops: Trie[Unit], expected: LabelConfig, expectedEnd: String) = {
-        this(specific, letter.asInternalPredicate, ops, expected.asExpectDescs, Some(new ExpectDesc(expectedEnd)))
+        this(specific, letter.asInternalPredicate, ops, expected.asExpectItems(specific), expected.asReason, Some(new ExpectDesc(expectedEnd)))
     }
     protected val caseSensitive = true
     private val ends = ops.suffixes(specific)

parsley/shared/src/main/scala/parsley/token/errors/ConfigImplUntyped.scala

@@ -27,9 +27,9 @@ private [parsley] trait LabelOps {
     private [parsley] final def asExpectItems(raw: Char): Iterable[ExpectItem] = asExpectItems(s"$raw")
 }
 
-// TODO: reason extraction, maybe tie into errors?
-private [parsley] trait ExplainOps
-
+private [parsley] sealed trait ExplainOps {
+    private [parsley] def asReason: Option[String]
+}
 
 // Constraining Types
 /** This type can be used to configure ''both'' errors that make labels and those that make reasons.
@@ -52,6 +52,10 @@ trait LabelConfig extends LabelWithExplainConfig {
   */
 trait ExplainConfig extends LabelWithExplainConfig
 
+private [parsley] sealed trait Labeller {
+    private [parsley] def config(name: String): LabelConfig
+}
+
 private [errors] final class Label private[errors] (val labels: Seq[String]) extends LabelConfig {
     require(labels.forall(_.nonEmpty), "labels cannot be empty strings")
     private [parsley] final override def apply[A](p: Parsley[A]) = p.labels(labels: _*)
@@ -64,16 +68,18 @@ private [errors] final class Label private[errors] (val labels: Seq[String]) ext
         case _ => this
     }
     private [parsley] final override def orElse(config: LabelConfig) = this
+    private [parsley] final override def asReason: Option[String] = None
 }
 /** This object has a factory for configurations producing labels: labels may not be empty.
   * @since 4.1.0
   * @group labels
   */
-object Label {
+object Label extends Labeller {
     def apply(label: String): LabelConfig = if (label.isEmpty) Hidden else new Label(Seq(label))
     def apply(label1: String, label2: String, labels: String*): LabelConfig = new Label(label1 +: label2 +: labels)
     // this is required internally, will go in parsley 5
     private [parsley] def apply(labels: String*) = new Label(labels)
+    private [parsley] final def config(name: String) = Label(name)
 }
 
 /** This object configures labels by stating that it must be hidden.
@@ -87,6 +93,7 @@ object Hidden extends LabelConfig {
     private [parsley] final override def asExpectItems(@unused raw: String) = asExpectDescs
     private [parsley] final override def orElse(config: LabelWithExplainConfig) = this
     private [parsley] final override def orElse(config: LabelConfig) = this
+    private [parsley] final override def asReason: Option[String] = None
 }
 
 private [errors] final class Reason private[errors]  (val reason: String) extends ExplainConfig {
@@ -100,6 +107,7 @@ private [errors] final class Reason private[errors]  (val reason: String) extend
         case lr: LabelAndReason => new LabelAndReason(lr.labels, reason)
         case _ => this
     }
+    private [parsley] final override def asReason: Option[String] = Some(reason)
 }
 /** This object has a factory for configurations producing reasons: if the empty string is provided, this equivalent to [[NotConfigured `NotConfigured`]].
   * @since 4.1.0
@@ -117,6 +125,7 @@ private [errors] final class LabelAndReason private[errors] (val labels: Seq[Str
     private [parsley] final override def asExpectDescs(@unused otherwise: String) = asExpectDescs
     private [parsley] final override def asExpectItems(@unused raw: String) = asExpectDescs
     private [parsley] final override def orElse(config: LabelWithExplainConfig) = this
+    private [parsley] final override def asReason: Option[String] = Some(reason)
 }
 /** This object has a factory for configurations producing labels and reasons: if the empty label is provided, this equivalent to [[Hidden `Hidden`]] with no
   * reason; if the empty reason is provided this is equivalent to [[Label$ `Label`]].
@@ -135,11 +144,13 @@ object LabelAndReason {
   * @since 4.1.0
   * @group labels
   */
-object NotConfigured extends LabelConfig with ExplainConfig with LabelWithExplainConfig {
+object NotConfigured extends LabelConfig with ExplainConfig with LabelWithExplainConfig with Labeller {
     private [parsley] final override def apply[A](p: Parsley[A]) = p
     private [parsley] final override def asExpectDescs = None
     private [parsley] final override def asExpectDescs(otherwise: String) = Some(new ExpectDesc(otherwise))
     private [parsley] final override def asExpectItems(raw: String) = Some(new ExpectRaw(raw))
     private [parsley] final override def orElse(config: LabelWithExplainConfig) = config
     private [parsley] final override def orElse(config: LabelConfig) = config
+    private [parsley] final override def asReason: Option[String] = None
+    private [parsley] final def config(name: String) = this
 }

parsley/shared/src/main/scala/parsley/token/errors/ErrorConfig.scala

@@ -818,6 +818,11 @@ class ErrorConfig {
       * @group symbol
       */
     def labelSymbolOperator(symbol: String): LabelConfig = Label(symbol)
+    // To unify, or not to unify
+    private [parsley] def defaultSymbolKeyword: Labeller = Label
+    private [parsley] def defaultSymbolOperator: Labeller = Label
+    // Other?
+    private [parsley] def defaultSymbolPunctuation: Labeller = NotConfigured
     /** How the required end of a given keyword should be specified in an error.
       * @since 4.1.0
       * @note defaults to "end of symbol"

parsley/shared/src/main/scala/parsley/token/numeric/SignedInteger.scala

@@ -28,7 +28,7 @@ private [token] final class SignedInteger(desc: NumericDesc, unsigned: UnsignedI
     override def number: Parsley[BigInt] = err.labelIntegerSignedNumber.apply(_number)
 
     override protected [numeric] def bounded[T](number: Parsley[BigInt], bits: Bits, radix: Int, label: (ErrorConfig, Boolean) => LabelWithExplainConfig)
-                                               (implicit ev: CanHold[bits.self,T]): Parsley[T] = label(err, false) {
+                                               (implicit ev: CanHold[bits.self,T]): Parsley[T] = label(err, true) {
         err.filterIntegerOutOfBounds(bits.lowerSigned, bits.upperSigned, radix).collect(number) {
             case x if bits.lowerSigned <= x && x <= bits.upperSigned => ev.fromBigInt(x)
         }

parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala

@@ -8,7 +8,7 @@ package parsley.token.symbol
 import parsley.Parsley, Parsley.atomic
 import parsley.character.{char, string}
 import parsley.token.descriptions.{NameDesc, SymbolDesc}
-import parsley.token.errors.ErrorConfig
+import parsley.token.errors.{ErrorConfig, NotConfigured}
 
 import parsley.internal.deepembedding.singletons.token
 
@@ -22,22 +22,26 @@ private [token] class ConcreteSymbol(nameDesc: NameDesc, symbolDesc: SymbolDesc,
 
     override def apply(name: String): Parsley[Unit] = {
         require(name.nonEmpty, "Symbols may not be empty strings")
+        lazy val punctuationLabel = err.labelSymbolPunctuation.get(name).map {
+            case None => parsley.token.errors.Hidden
+            case Some(l) => parsley.token.errors.Label(l)
+        }.getOrElse(NotConfigured).orElse(err.defaultSymbolPunctuation.config(name))
         if (symbolDesc.hardKeywords(name))       softKeyword(name)
         else if (symbolDesc.hardOperators(name)) softOperator(name)
-        else                                     atomic(string(name)).void
+        else                                     punctuationLabel(atomic(string(name)).void)
     }
 
     override def apply(name: Char): Parsley[Unit] = char(name).void
 
     override def softKeyword(name: String): Parsley[Unit] = {
         require(name.nonEmpty, "Keywords may not be empty strings")
         new Parsley(new token.SoftKeyword(name, nameDesc.identifierLetter, symbolDesc.caseSensitive,
-                                          err.labelSymbolKeyword(name), err.labelSymbolEndOfKeyword(name)))
+                                          err.labelSymbolKeyword(name).orElse(err.defaultSymbolKeyword.config(name)), err.labelSymbolEndOfKeyword(name)))
     }
 
     override def softOperator(name: String): Parsley[Unit] = {
         require(name.nonEmpty, "Operators may not be empty strings")
         new Parsley(new token.SoftOperator(name, nameDesc.operatorLetter, symbolDesc.hardOperatorsTrie,
-                                           err.labelSymbolOperator(name), err.labelSymbolEndOfOperator(name)))
+                                           err.labelSymbolOperator(name).orElse(err.defaultSymbolOperator.config(name)), err.labelSymbolEndOfOperator(name)))
     }
 }

parsley/shared/src/main/scala/parsley/token/text/StringCharacter.scala

@@ -31,7 +31,7 @@ private [token] class RawCharacter(err: ErrorConfig) extends StringCharacter {
 
 private [token] class EscapableCharacter(desc: EscapeDesc, escapes: Escape, space: Parsley[_], err: ErrorConfig) extends StringCharacter {
     override def isRaw: Boolean = false
-    private lazy val escapeEmpty = err.labelStringEscapeEmpty(desc.emptyEscape.fold[Parsley[Char]](empty)(char))
+    private lazy val escapeEmpty = desc.emptyEscape.fold[Parsley[Char]](empty)(c => err.labelStringEscapeEmpty(char(c)))
     private lazy val escapeGap = {
         if (desc.gapsSupported) some(err.labelStringEscapeGap(space)) ~> err.labelStringEscapeGapEnd(desc.escBegin)
         else empty

parsley/shared/src/test/scala/parsley/internal/deepembedding/frontend/VisitorTests.scala

@@ -83,6 +83,8 @@ class VisitorTests extends ParsleyTest {
 
         override private[parsley] def apply[A](p: Parsley[A]): Parsley[A] =
             dontExecute()
+
+        override private[parsley] def asReason: Option[String] = None
     }
 
     private val dummyCaretWidth: CaretWidth = new FlexibleCaret(0)