Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
- "11"
scala:
- "2.12.12"
- "2.13.1"
- "2.13.6"
testWithCoverageReport:
runs-on: ubuntu-latest
steps:
Expand Down
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ Extensions for the [Kryo serialization library](https://github.com/EsotericSoftw
serializers and a set of classes to ease configuration of Kryo in systems like Hadoop, Storm,
Akka, etc.

### Compatibility
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@johnynek added this section. what do you think?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circling back on this. Does this seem ok?


Serialization compatibility is **NOT** guaranteed between releases, and for this reason, we don't recommend using it for long-term storage. Serialization is highly dependent on scala version compatibility and on the underlying Kryo serializers, which take different approaches to compatibility.

### Building Chill

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ package com.twitter.chill
* @see [[ScalaCollectionsRegistrar]] and [[AllScalaRegistrar]] for all the
* provided registrations.
*/
class ScalaCollectionsRegistrarCompat extends IKryoRegistrar {
private[chill] class AllScalaRegistrarCompat_0_9_5 extends IKryoRegistrar {
override def apply(newK: Kryo): Unit = ()
}

private[chill] class AllScalaRegistrarCompat extends IKryoRegistrar {
override def apply(newK: Kryo): Unit = ()
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,15 @@ import scala.collection.immutable.Range
* @see [[ScalaCollectionsRegistrar]] and [[AllScalaRegistrar]] for all the
* provided registrations.
*/
class ScalaCollectionsRegistrarCompat extends IKryoRegistrar {
private[chill] class AllScalaRegistrarCompat_0_9_5 extends IKryoRegistrar {
override def apply(newK: Kryo): Unit =
newK.register(classOf[Range.Exclusive])

}

private[chill] class AllScalaRegistrarCompat extends IKryoRegistrar {
override def apply(newK: Kryo): Unit =
newK
.forConcreteTraversableClass(Vector[Any]())
.forConcreteTraversableClass(Vector('a))
}
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ class JavaWrapperCollectionRegistrar extends IKryoRegistrar {
}

/** Registrar for everything that was registered in chill 0.9.2 - included for backwards compatibility. */
class AllScalaRegistrar_0_9_2 extends IKryoRegistrar {
final private[chill] class AllScalaRegistrar_0_9_2 extends IKryoRegistrar {
def apply(k: Kryo): Unit = {
new ScalaCollectionsRegistrar()(k)
new JavaWrapperCollectionRegistrar()(k)
Expand All @@ -207,26 +207,11 @@ class AllScalaRegistrar_0_9_2 extends IKryoRegistrar {
}
}

/** Registrar for everything that was registered in chill 0.10.0 */
class AllScalaRegistrar_0_10_0 extends IKryoRegistrar {
/** Registrar for everything that was registered in chill 0.9.5 */
final private[chill] class AllScalaRegistrar_0_9_5 extends IKryoRegistrar {
def apply(k: Kryo): Unit = {
new AllScalaRegistrar_0_9_2()(k)
new ScalaCollectionsRegistrarCompat()(k)
}
}

/**
* Registers all the scala (and java) serializers we have. The registrations are designed to cover most of
* scala.collecion.immutable, so they can be used in long term persistence scenarios that run with
* setRegistrationRequired(true).
*
* When adding new serializers, add them to the end of the list, so compatibility is not broken needlessly
* for projects using chill for long term persistence - see com.twitter.chill.RegistrationIdsSpec.
*/
class AllScalaRegistrar extends IKryoRegistrar {
def apply(k: Kryo): Unit = {
new AllScalaRegistrar_0_10_0()(k)

new AllScalaRegistrarCompat_0_9_5()(k)
k.registerClasses(
Seq(
classOf[Array[Byte]],
Expand Down Expand Up @@ -289,3 +274,18 @@ class AllScalaRegistrar extends IKryoRegistrar {
.forConcreteTraversableClass(Map(1 -> 2).keySet)
}
}

/**
* Registers all the scala (and java) serializers we have. The registrations are designed to cover most of
* scala.collecion.immutable, so they can be used in long term persistence scenarios that run with
* setRegistrationRequired(true).
*
* When adding new serializers, add them to the end of the list, so compatibility is not broken needlessly
* for projects using chill for long term persistence - see com.twitter.chill.RegistrationIdsSpec.
*/
class AllScalaRegistrar extends IKryoRegistrar {
def apply(k: Kryo): Unit = {
new AllScalaRegistrar_0_9_5()(k)
new AllScalaRegistrarCompat()(k)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ limitations under the License.
package com.twitter.chill

object RegistrationIdsSpecData {
val Entries_0_10_0 =
val Entries_0_9_5 =
"""0 -> int
|1 -> class java.lang.String
|2 -> float
Expand Down Expand Up @@ -110,11 +110,8 @@ object RegistrationIdsSpecData {
|89 -> class java.util.Collections$UnmodifiableSet
|90 -> class java.util.Collections$UnmodifiableSortedMap
|91 -> class java.util.Collections$UnmodifiableSortedSet
|92 -> class com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure""".stripMargin.linesIterator
.mkString("\n")

val RecentEntries =
"""93 -> class [B
|92 -> class com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure
|93 -> class [B
|94 -> class [S
|95 -> class [I
|96 -> class [J
Expand Down Expand Up @@ -172,6 +169,8 @@ object RegistrationIdsSpecData {
|148 -> class scala.collection.immutable.MapLike$ImmutableDefaultKeySet""".stripMargin.linesIterator
.mkString("\n")

val RecentEntries = ""

val CurrentEntries =
(Entries_0_10_0.linesIterator ++ RecentEntries.linesIterator).mkString("\n")
(Entries_0_9_5.linesIterator ++ RecentEntries.linesIterator).mkString("\n")
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ limitations under the License.
package com.twitter.chill

object RegistrationIdsSpecData {
val Entries_0_10_0 =

val Entries_0_9_5 =
"""0 -> int
|1 -> class java.lang.String
|2 -> float
Expand Down Expand Up @@ -111,10 +112,8 @@ object RegistrationIdsSpecData {
|90 -> class java.util.Collections$UnmodifiableSortedMap
|91 -> class java.util.Collections$UnmodifiableSortedSet
|92 -> class com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure
|93 -> class scala.collection.immutable.Range$Exclusive""".stripMargin.linesIterator.mkString("\n")

val RecentEntries =
"""94 -> class [B
|93 -> class scala.collection.immutable.Range$Exclusive
|94 -> class [B
|95 -> class [S
|96 -> class [I
|97 -> class [J
Expand Down Expand Up @@ -166,6 +165,11 @@ object RegistrationIdsSpecData {
|143 -> class scala.collection.immutable.MapOps$ImmutableKeySet""".stripMargin.linesIterator
.mkString("\n")

val RecentEntries =
"""144 -> class scala.collection.immutable.Vector0$
|145 -> class scala.collection.immutable.Vector1""".stripMargin.linesIterator
.mkString("\n")

val CurrentEntries =
(Entries_0_10_0.linesIterator ++ RecentEntries.linesIterator).mkString("\n")
(Entries_0_9_5.linesIterator ++ RecentEntries.linesIterator).mkString("\n")
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ object SerializedExamplesData {
15 -> ("EQECBA==" -> Some(2)),
16 -> ("EgECBA==" -> Left(2)),
17 -> ("EwECBA==" -> Right(2)),
18 -> ("FAEBAgQ=" -> Vector(2)),
// 18 -> ("FAEBAgQ=" -> Vector(2)),
// new vector classes in 2.13 see 144, 145
19 -> ("FQEBAgQ=" -> Set(2)),
20 -> ("FgECAgQCBg==" -> Set(2, 3)),
21 -> ("FwEDAgQCBgII" -> Set(2, 3, 4)),
Expand All @@ -65,9 +66,9 @@ object SerializedExamplesData {
27 -> ("HQEEJwECBAIGJwECCAIKJwECDAIOJwECEAIS" -> Map(2 -> 3, 4 -> 5, 6 -> 7, 8 -> 9)),
28 -> ("HgEA" -> HashMap.empty[Any, Any]),
29 -> ("HwEMAAwIBgI=" -> new Range.Inclusive(3, 6, 1)),
30 -> ("IAEBAgoAAQgBAHNjYWxhLm1hdGguTnVtZXJpYyRJbnRJc0ludGVncmFspAEBAAMCBAIC" ->
30 -> ("IAECAgoAAAEAAQBzY2FsYS5tYXRoLk51bWVyaWMkSW50SXNJbnRlZ3JhbKQBAQADAgQCAg==" ->
new NumericRange.Inclusive[Int](2, 5, 1)),
31 -> ("IQEBAgoAAAYBAHNjYWxhLm1hdGguTnVtZXJpYyRJbnRJc0ludGVncmFspAEBAAMCBAIC" ->
31 -> ("IQECAgoAAAAAAQBzY2FsYS5tYXRoLk51bWVyaWMkSW50SXNJbnRlZ3JhbKQBAQADAgQCAg==" ->
new NumericRange.Exclusive[Int](2, 5, 1)),
32 -> ("IgECAgYCCg==" -> scala.collection.mutable.BitSet(3, 5)),
33 -> ("IwEBJwECBgIK" -> scala.collection.mutable.HashMap(3 -> 5)),
Expand Down Expand Up @@ -197,10 +198,12 @@ object SerializedExamplesData {
140 -> ("jgEBCg==" -> new VolatileByteRef(10)),
141 -> ("jwEBAQBqYXZhLm1hdGguQmlnRGVjaW1h7AECAgA=" -> math.BigDecimal(2)),
142 -> ("kAEBAA==" -> (Queue.empty[Any], true)),
143 -> ("kQEBAQIC" -> (Map(1 -> 2).keySet, true))
143 -> ("kQEBAQIC" -> (Map(1 -> 2).keySet, true)),
144 -> ("kgEBAA==" -> Vector.empty[Int]),
145 -> ("kwEBAQIC" -> Vector(1))
)

val SpecialCasesNotInExamplesMap: Seq[Int] = Seq(9, 71, 84, 91, 92, 119)
val SpecialCasesNotInExamplesMap: Seq[Int] = Seq(9, 18, 71, 84, 91, 92, 119)

val OmitExamplesInScalaVersion: Map[String, Seq[Int]] = Map.empty
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@ class RegistrationIdsSpec extends AnyWordSpec with Matchers {
|
|For the ScalaKryoInstantiators, the registered classes""".stripMargin
.should {
val compatibility = classOf[AllScalaRegistrar_0_10_0].getSimpleName
val compatibility = classOf[AllScalaRegistrar_0_9_5].getSimpleName
(s"be as expected for the backward compatibility layer $compatibility,\n" +
" i.e. contain the list of registrations defined in this test.").in {
val k = new KryoBase
new AllScalaRegistrar_0_10_0().apply(k)
if (registeredEntries(k) != Entries_0_10_0) printMessageFor(k, compatibility)
assert(registeredEntries(k) == Entries_0_10_0)
new AllScalaRegistrar_0_9_5().apply(k)
if (registeredEntries(k) != Entries_0_9_5) printMessageFor(k, compatibility)
assert(registeredEntries(k) == Entries_0_9_5)
}

val current = classOf[AllScalaRegistrar].getSimpleName
Expand Down