diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 08ffd078..3d9997bb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,7 +50,7 @@ jobs: - "11" scala: - "2.12.12" - - "2.13.1" + - "2.13.6" testWithCoverageReport: runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 9002d4fa..cdf0de60 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ Extensions for the [Kryo serialization library](https://github.com/EsotericSoftw serializers and a set of classes to ease configuration of Kryo in systems like Hadoop, Storm, Akka, etc. +### Compatibility + +Serialization compatibility is **NOT** guaranteed between releases, and for this reason, we don't recommend using it for long-term storage. Serialization is highly dependent on scala version compatibility and on the underlying Kryo serializers, which take different approaches to compatibility. ### Building Chill diff --git a/chill-scala/src/main/scala-2.12-/com/twitter/chill/ScalaCollectionsRegistrarCompat.scala b/chill-scala/src/main/scala-2.12-/com/twitter/chill/AllScalaRegistrarCompat.scala similarity index 82% rename from chill-scala/src/main/scala-2.12-/com/twitter/chill/ScalaCollectionsRegistrarCompat.scala rename to chill-scala/src/main/scala-2.12-/com/twitter/chill/AllScalaRegistrarCompat.scala index 61c178db..8c4fde71 100644 --- a/chill-scala/src/main/scala-2.12-/com/twitter/chill/ScalaCollectionsRegistrarCompat.scala +++ b/chill-scala/src/main/scala-2.12-/com/twitter/chill/AllScalaRegistrarCompat.scala @@ -23,6 +23,10 @@ package com.twitter.chill * @see [[ScalaCollectionsRegistrar]] and [[AllScalaRegistrar]] for all the * provided registrations. */ -class ScalaCollectionsRegistrarCompat extends IKryoRegistrar { +private[chill] class AllScalaRegistrarCompat_0_9_5 extends IKryoRegistrar { + override def apply(newK: Kryo): Unit = () +} + +private[chill] class AllScalaRegistrarCompat extends IKryoRegistrar { override def apply(newK: Kryo): Unit = () } diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/ScalaCollectionsRegistrarCompat.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/AllScalaRegistrarCompat.scala similarity index 76% rename from chill-scala/src/main/scala-2.13+/com/twitter/chill/ScalaCollectionsRegistrarCompat.scala rename to chill-scala/src/main/scala-2.13+/com/twitter/chill/AllScalaRegistrarCompat.scala index 831449da..00f77b7a 100644 --- a/chill-scala/src/main/scala-2.13+/com/twitter/chill/ScalaCollectionsRegistrarCompat.scala +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/AllScalaRegistrarCompat.scala @@ -25,7 +25,15 @@ import scala.collection.immutable.Range * @see [[ScalaCollectionsRegistrar]] and [[AllScalaRegistrar]] for all the * provided registrations. */ -class ScalaCollectionsRegistrarCompat extends IKryoRegistrar { +private[chill] class AllScalaRegistrarCompat_0_9_5 extends IKryoRegistrar { override def apply(newK: Kryo): Unit = newK.register(classOf[Range.Exclusive]) + +} + +private[chill] class AllScalaRegistrarCompat extends IKryoRegistrar { + override def apply(newK: Kryo): Unit = + newK + .forConcreteTraversableClass(Vector[Any]()) + .forConcreteTraversableClass(Vector('a)) } diff --git a/chill-scala/src/main/scala/com/twitter/chill/ScalaKryoInstantiator.scala b/chill-scala/src/main/scala/com/twitter/chill/ScalaKryoInstantiator.scala index 5a04afb9..e751b0a7 100644 --- a/chill-scala/src/main/scala/com/twitter/chill/ScalaKryoInstantiator.scala +++ b/chill-scala/src/main/scala/com/twitter/chill/ScalaKryoInstantiator.scala @@ -183,7 +183,7 @@ class JavaWrapperCollectionRegistrar extends IKryoRegistrar { } /** Registrar for everything that was registered in chill 0.9.2 - included for backwards compatibility. */ -class AllScalaRegistrar_0_9_2 extends IKryoRegistrar { +final private[chill] class AllScalaRegistrar_0_9_2 extends IKryoRegistrar { def apply(k: Kryo): Unit = { new ScalaCollectionsRegistrar()(k) new JavaWrapperCollectionRegistrar()(k) @@ -207,26 +207,11 @@ class AllScalaRegistrar_0_9_2 extends IKryoRegistrar { } } -/** Registrar for everything that was registered in chill 0.10.0 */ -class AllScalaRegistrar_0_10_0 extends IKryoRegistrar { +/** Registrar for everything that was registered in chill 0.9.5 */ +final private[chill] class AllScalaRegistrar_0_9_5 extends IKryoRegistrar { def apply(k: Kryo): Unit = { new AllScalaRegistrar_0_9_2()(k) - new ScalaCollectionsRegistrarCompat()(k) - } -} - -/** - * Registers all the scala (and java) serializers we have. The registrations are designed to cover most of - * scala.collecion.immutable, so they can be used in long term persistence scenarios that run with - * setRegistrationRequired(true). - * - * When adding new serializers, add them to the end of the list, so compatibility is not broken needlessly - * for projects using chill for long term persistence - see com.twitter.chill.RegistrationIdsSpec. - */ -class AllScalaRegistrar extends IKryoRegistrar { - def apply(k: Kryo): Unit = { - new AllScalaRegistrar_0_10_0()(k) - + new AllScalaRegistrarCompat_0_9_5()(k) k.registerClasses( Seq( classOf[Array[Byte]], @@ -289,3 +274,18 @@ class AllScalaRegistrar extends IKryoRegistrar { .forConcreteTraversableClass(Map(1 -> 2).keySet) } } + +/** + * Registers all the scala (and java) serializers we have. The registrations are designed to cover most of + * scala.collecion.immutable, so they can be used in long term persistence scenarios that run with + * setRegistrationRequired(true). + * + * When adding new serializers, add them to the end of the list, so compatibility is not broken needlessly + * for projects using chill for long term persistence - see com.twitter.chill.RegistrationIdsSpec. + */ +class AllScalaRegistrar extends IKryoRegistrar { + def apply(k: Kryo): Unit = { + new AllScalaRegistrar_0_9_5()(k) + new AllScalaRegistrarCompat()(k) + } +} diff --git a/chill-scala/src/test/scala-2.12-/com/twitter/chill/RegistrationIdsSpecData.scala b/chill-scala/src/test/scala-2.12-/com/twitter/chill/RegistrationIdsSpecData.scala index 1d749510..4b6ba8b5 100644 --- a/chill-scala/src/test/scala-2.12-/com/twitter/chill/RegistrationIdsSpecData.scala +++ b/chill-scala/src/test/scala-2.12-/com/twitter/chill/RegistrationIdsSpecData.scala @@ -17,7 +17,7 @@ limitations under the License. package com.twitter.chill object RegistrationIdsSpecData { - val Entries_0_10_0 = + val Entries_0_9_5 = """0 -> int |1 -> class java.lang.String |2 -> float @@ -110,11 +110,8 @@ object RegistrationIdsSpecData { |89 -> class java.util.Collections$UnmodifiableSet |90 -> class java.util.Collections$UnmodifiableSortedMap |91 -> class java.util.Collections$UnmodifiableSortedSet - |92 -> class com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure""".stripMargin.linesIterator - .mkString("\n") - - val RecentEntries = - """93 -> class [B + |92 -> class com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure + |93 -> class [B |94 -> class [S |95 -> class [I |96 -> class [J @@ -172,6 +169,8 @@ object RegistrationIdsSpecData { |148 -> class scala.collection.immutable.MapLike$ImmutableDefaultKeySet""".stripMargin.linesIterator .mkString("\n") + val RecentEntries = "" + val CurrentEntries = - (Entries_0_10_0.linesIterator ++ RecentEntries.linesIterator).mkString("\n") + (Entries_0_9_5.linesIterator ++ RecentEntries.linesIterator).mkString("\n") } diff --git a/chill-scala/src/test/scala-2.13+/com/twitter/chill/RegistrationIdsSpecData.scala b/chill-scala/src/test/scala-2.13+/com/twitter/chill/RegistrationIdsSpecData.scala index b647967f..2d44f0fb 100644 --- a/chill-scala/src/test/scala-2.13+/com/twitter/chill/RegistrationIdsSpecData.scala +++ b/chill-scala/src/test/scala-2.13+/com/twitter/chill/RegistrationIdsSpecData.scala @@ -17,7 +17,8 @@ limitations under the License. package com.twitter.chill object RegistrationIdsSpecData { - val Entries_0_10_0 = + + val Entries_0_9_5 = """0 -> int |1 -> class java.lang.String |2 -> float @@ -111,10 +112,8 @@ object RegistrationIdsSpecData { |90 -> class java.util.Collections$UnmodifiableSortedMap |91 -> class java.util.Collections$UnmodifiableSortedSet |92 -> class com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure - |93 -> class scala.collection.immutable.Range$Exclusive""".stripMargin.linesIterator.mkString("\n") - - val RecentEntries = - """94 -> class [B + |93 -> class scala.collection.immutable.Range$Exclusive + |94 -> class [B |95 -> class [S |96 -> class [I |97 -> class [J @@ -166,6 +165,11 @@ object RegistrationIdsSpecData { |143 -> class scala.collection.immutable.MapOps$ImmutableKeySet""".stripMargin.linesIterator .mkString("\n") + val RecentEntries = + """144 -> class scala.collection.immutable.Vector0$ + |145 -> class scala.collection.immutable.Vector1""".stripMargin.linesIterator + .mkString("\n") + val CurrentEntries = - (Entries_0_10_0.linesIterator ++ RecentEntries.linesIterator).mkString("\n") + (Entries_0_9_5.linesIterator ++ RecentEntries.linesIterator).mkString("\n") } diff --git a/chill-scala/src/test/scala-2.13+/com/twitter/chill/SerializedExamplesData.scala b/chill-scala/src/test/scala-2.13+/com/twitter/chill/SerializedExamplesData.scala index 16527c25..773c7de4 100644 --- a/chill-scala/src/test/scala-2.13+/com/twitter/chill/SerializedExamplesData.scala +++ b/chill-scala/src/test/scala-2.13+/com/twitter/chill/SerializedExamplesData.scala @@ -53,7 +53,8 @@ object SerializedExamplesData { 15 -> ("EQECBA==" -> Some(2)), 16 -> ("EgECBA==" -> Left(2)), 17 -> ("EwECBA==" -> Right(2)), - 18 -> ("FAEBAgQ=" -> Vector(2)), + // 18 -> ("FAEBAgQ=" -> Vector(2)), + // new vector classes in 2.13 see 144, 145 19 -> ("FQEBAgQ=" -> Set(2)), 20 -> ("FgECAgQCBg==" -> Set(2, 3)), 21 -> ("FwEDAgQCBgII" -> Set(2, 3, 4)), @@ -65,9 +66,9 @@ object SerializedExamplesData { 27 -> ("HQEEJwECBAIGJwECCAIKJwECDAIOJwECEAIS" -> Map(2 -> 3, 4 -> 5, 6 -> 7, 8 -> 9)), 28 -> ("HgEA" -> HashMap.empty[Any, Any]), 29 -> ("HwEMAAwIBgI=" -> new Range.Inclusive(3, 6, 1)), - 30 -> ("IAEBAgoAAQgBAHNjYWxhLm1hdGguTnVtZXJpYyRJbnRJc0ludGVncmFspAEBAAMCBAIC" -> + 30 -> ("IAECAgoAAAEAAQBzY2FsYS5tYXRoLk51bWVyaWMkSW50SXNJbnRlZ3JhbKQBAQADAgQCAg==" -> new NumericRange.Inclusive[Int](2, 5, 1)), - 31 -> ("IQEBAgoAAAYBAHNjYWxhLm1hdGguTnVtZXJpYyRJbnRJc0ludGVncmFspAEBAAMCBAIC" -> + 31 -> ("IQECAgoAAAAAAQBzY2FsYS5tYXRoLk51bWVyaWMkSW50SXNJbnRlZ3JhbKQBAQADAgQCAg==" -> new NumericRange.Exclusive[Int](2, 5, 1)), 32 -> ("IgECAgYCCg==" -> scala.collection.mutable.BitSet(3, 5)), 33 -> ("IwEBJwECBgIK" -> scala.collection.mutable.HashMap(3 -> 5)), @@ -197,10 +198,12 @@ object SerializedExamplesData { 140 -> ("jgEBCg==" -> new VolatileByteRef(10)), 141 -> ("jwEBAQBqYXZhLm1hdGguQmlnRGVjaW1h7AECAgA=" -> math.BigDecimal(2)), 142 -> ("kAEBAA==" -> (Queue.empty[Any], true)), - 143 -> ("kQEBAQIC" -> (Map(1 -> 2).keySet, true)) + 143 -> ("kQEBAQIC" -> (Map(1 -> 2).keySet, true)), + 144 -> ("kgEBAA==" -> Vector.empty[Int]), + 145 -> ("kwEBAQIC" -> Vector(1)) ) - val SpecialCasesNotInExamplesMap: Seq[Int] = Seq(9, 71, 84, 91, 92, 119) + val SpecialCasesNotInExamplesMap: Seq[Int] = Seq(9, 18, 71, 84, 91, 92, 119) val OmitExamplesInScalaVersion: Map[String, Seq[Int]] = Map.empty } diff --git a/chill-scala/src/test/scala/com/twitter/chill/RegistrationIdsSpec.scala b/chill-scala/src/test/scala/com/twitter/chill/RegistrationIdsSpec.scala index 8f80e4fb..5cf7d23b 100644 --- a/chill-scala/src/test/scala/com/twitter/chill/RegistrationIdsSpec.scala +++ b/chill-scala/src/test/scala/com/twitter/chill/RegistrationIdsSpec.scala @@ -38,13 +38,13 @@ class RegistrationIdsSpec extends AnyWordSpec with Matchers { | |For the ScalaKryoInstantiators, the registered classes""".stripMargin .should { - val compatibility = classOf[AllScalaRegistrar_0_10_0].getSimpleName + val compatibility = classOf[AllScalaRegistrar_0_9_5].getSimpleName (s"be as expected for the backward compatibility layer $compatibility,\n" + " i.e. contain the list of registrations defined in this test.").in { val k = new KryoBase - new AllScalaRegistrar_0_10_0().apply(k) - if (registeredEntries(k) != Entries_0_10_0) printMessageFor(k, compatibility) - assert(registeredEntries(k) == Entries_0_10_0) + new AllScalaRegistrar_0_9_5().apply(k) + if (registeredEntries(k) != Entries_0_9_5) printMessageFor(k, compatibility) + assert(registeredEntries(k) == Entries_0_9_5) } val current = classOf[AllScalaRegistrar].getSimpleName