From d92f04c8c1825d5c16bba98380570b234045947b Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 6 Jul 2026 15:15:02 +0900 Subject: [PATCH 01/30] fix test --- tests/Common/CommonStandardTests/CommonStandardTests.csproj | 1 + .../Lipidomics/LipidEieioMsmsCharacterizationTests.cs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/Common/CommonStandardTests/CommonStandardTests.csproj b/tests/Common/CommonStandardTests/CommonStandardTests.csproj index fd4da24ed..eaec1c2ce 100644 --- a/tests/Common/CommonStandardTests/CommonStandardTests.csproj +++ b/tests/Common/CommonStandardTests/CommonStandardTests.csproj @@ -29,6 +29,7 @@ PreserveNewest + Resources\Lipidomics\PC_O-16_0_18_1%289%29_[M+H]+.msp.txt diff --git a/tests/Common/CommonStandardTests/Lipidomics/LipidEieioMsmsCharacterizationTests.cs b/tests/Common/CommonStandardTests/Lipidomics/LipidEieioMsmsCharacterizationTests.cs index f8e78750f..1c3a1549f 100644 --- a/tests/Common/CommonStandardTests/Lipidomics/LipidEieioMsmsCharacterizationTests.cs +++ b/tests/Common/CommonStandardTests/Lipidomics/LipidEieioMsmsCharacterizationTests.cs @@ -2694,8 +2694,8 @@ public void EtherPC_P_CharacterizationTest() } [DataTestMethod()] - [DeploymentItem(@"Resources\Lipidomics\LipidEieioMsmsCharacterizationTests\PC_O-16_0_18_1(9)_[M+H]+.msp.txt", @"Resources\Lipidomics\LipidEieioMsmsCharacterizationTests")] - [DataRow(@"Resources\Lipidomics\LipidEieioMsmsCharacterizationTests\PC_O-16_0_18_1(9)_[M+H]+.msp.txt", "PC O-16:0/18:1")] + [DeploymentItem(@"Resources\Lipidomics\PC_O-16_0_18_1(9)_[M+H]+.msp.txt", @"Resources\Lipidomics")] + [DataRow(@"Resources\Lipidomics\PC_O-16_0_18_1(9)_[M+H]+.msp.txt", "PC O-16:0/18:1")] public void EtherPC_O_CharacterizationTest(string path, string expected) { var target = MspFileParser.MspFileReader(path)[0]; From 53f4a1644b6268f0a61c8f371ba0c10c3326c570 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 6 Jul 2026 16:37:37 +0900 Subject: [PATCH 02/30] Add AdductIon fixed-byte round-trip tests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../DataObj/Property/AdductIonTests.cs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/Common/CommonStandardTests/DataObj/Property/AdductIonTests.cs b/tests/Common/CommonStandardTests/DataObj/Property/AdductIonTests.cs index b0f23a1b3..201bd620c 100644 --- a/tests/Common/CommonStandardTests/DataObj/Property/AdductIonTests.cs +++ b/tests/Common/CommonStandardTests/DataObj/Property/AdductIonTests.cs @@ -2,6 +2,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Collections.Generic; using System.IO; +using System.Globalization; namespace CompMs.Common.DataObj.Property.Tests { @@ -43,5 +44,33 @@ public void SerializeNullAdductTest() { var newAdduct = MessagePackDefaultHandler.LoadFromStream(memory); Assert.AreEqual(AdductIon.Default, newAdduct); } + + [DataTestMethod()] + [DynamicData(nameof(GetSerializedAdductIonTestData))] + public void DeserializeSavedAdductBytesTest(string adductName, string hexBytes) { + var bytes = HexToBytes(hexBytes); + var actual = MessagePackDefaultHandler.LoadFromStream(new MemoryStream(bytes)); + var roundTrip = new MemoryStream(); + MessagePackDefaultHandler.SaveToStream(actual, roundTrip); + + CollectionAssert.AreEqual(bytes, roundTrip.ToArray()); + + Assert.AreEqual(AdductIon.GetAdductIon(adductName), actual); + } + + public static IEnumerable GetSerializedAdductIonTestData { + get { + yield return new object[] { "[M+H]+", "9ACB3FF0200D240D4AF601A65B4D2B485D2B0100C3CB3F1E26644B61B0D3CB0000000000000000C2C2" }; + yield return new object[] { "[M-H]-", "9ACBBFF0200D240D4AF601A65B4D2D485D2D0101C3CBBF1E26644B61B0D3CB0000000000000000C2C2" }; + } + } + + private static byte[] HexToBytes(string hex) { + var bytes = new byte[hex.Length / 2]; + for (var i = 0; i < bytes.Length; i++) { + bytes[i] = byte.Parse(hex.Substring(i * 2, 2), NumberStyles.HexNumber); + } + return bytes; + } } } \ No newline at end of file From ebd5d0aeedcc7e7ddf35f1b1eb275c2e4f2a7325 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 6 Jul 2026 18:07:42 +0900 Subject: [PATCH 03/30] Add MoleculeProperty messagepack compatibility test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Components/MoleculePropertyTests.cs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/Common/CommonStandardTests/Components/MoleculePropertyTests.cs diff --git a/tests/Common/CommonStandardTests/Components/MoleculePropertyTests.cs b/tests/Common/CommonStandardTests/Components/MoleculePropertyTests.cs new file mode 100644 index 000000000..430451ac2 --- /dev/null +++ b/tests/Common/CommonStandardTests/Components/MoleculePropertyTests.cs @@ -0,0 +1,45 @@ +using CompMs.Common.Components; +using CompMs.Common.DataObj.Property; +using CompMs.Common.Interfaces; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Globalization; +using MessagePack.Resolvers; + +namespace CompMs.Common.Components.Tests +{ + [TestClass()] + public class MoleculePropertyTests + { + [DataTestMethod()] + [DynamicData(nameof(GetSerializedMoleculePropertyTestData))] + public void DeserializeSavedMoleculePropertyBytesTest(string name, string hexBytes) { + var bytes = HexToBytes(hexBytes); + var formatter = MoleculePropertyExtension.Formatter; + var actual = formatter.Deserialize(bytes, 0, StandardResolver.Instance, out var readSize); + var buffer = new byte[bytes.Length + 32]; + var writeSize = formatter.Serialize(ref buffer, 0, actual, StandardResolver.Instance); + + Assert.AreEqual(bytes.Length, readSize); + var actualBytes = new byte[writeSize]; + Buffer.BlockCopy(buffer, 0, actualBytes, 0, writeSize); + CollectionAssert.AreEqual(bytes, actualBytes); + Assert.AreEqual(name, actual.Name); + } + + public static IEnumerable GetSerializedMoleculePropertyTestData { + get { + yield return new object[] { "Glucose", "95A7476C75636F7365DC001BA0CB0000000000000000CB0000000000000000CB00000000000000000000000000000000000000000000000000000000C20080A0A0A0" }; + } + } + + private static byte[] HexToBytes(string hex) { + var bytes = new byte[hex.Length / 2]; + for (var i = 0; i < bytes.Length; i++) { + bytes[i] = byte.Parse(hex.Substring(i * 2, 2), NumberStyles.HexNumber); + } + return bytes; + } + } +} From f51339a5ab002ec8589a921b6d779d85a769d02b Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 6 Jul 2026 18:54:48 +0900 Subject: [PATCH 04/30] Add LargeListMessagePack compatibility test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePack/LargeListMessagePackTests.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs index 40bcb0d2a..b914fea0f 100644 --- a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs +++ b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs @@ -45,6 +45,25 @@ public void SaveAndLoadSmallSampleTest(int size) { Assert.AreEqual(datas.Length, actual.Count); } + [TestMethod()] + public void SerializeAndDeserializeSmallSampleBytesTest() { + var datas = new SmallSample[] { + new SmallSample(), + new SmallSample(), + }; + var memory = new MemoryStream(); + LargeListMessagePack.Serialize(memory, datas); + var bytes = memory.ToArray(); + + memory.Position = 0; + var actual = LargeListMessagePack.Deserialize(memory); + var roundTrip = new MemoryStream(); + LargeListMessagePack.Serialize(roundTrip, actual); + + CollectionAssert.AreEqual(bytes, roundTrip.ToArray()); + Assert.AreEqual(datas.Length, actual.Count); + } + [TestMethod()] [DataRow(100, 100)] [DataRow(1000000, 100)] From 8710c09df0568ccae126eebb28bdb31e697e3e76 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Tue, 7 Jul 2026 17:11:58 +0900 Subject: [PATCH 05/30] Update MessagePack package versions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Common/CommonStandard/CommonStandard.csproj | 4 ++-- src/MSDIAL4/DatabaseStandard/DatabaseStandard.csproj | 2 +- src/MSDIAL4/MathematicsStandard/MathematicsStandard.csproj | 2 +- .../StructureFinderStandard/StructureFinderStandard.csproj | 2 +- src/MSDIAL5/MsdialGuiApp/MsdialGuiApp.csproj | 2 +- src/MSDIAL5/SpectrumViewer/SpectrumViewer.csproj | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Common/CommonStandard/CommonStandard.csproj b/src/Common/CommonStandard/CommonStandard.csproj index b543dff82..45bae3d0a 100644 --- a/src/Common/CommonStandard/CommonStandard.csproj +++ b/src/Common/CommonStandard/CommonStandard.csproj @@ -35,8 +35,8 @@ - - + + diff --git a/src/MSDIAL4/DatabaseStandard/DatabaseStandard.csproj b/src/MSDIAL4/DatabaseStandard/DatabaseStandard.csproj index 329a2501e..c6db5969b 100644 --- a/src/MSDIAL4/DatabaseStandard/DatabaseStandard.csproj +++ b/src/MSDIAL4/DatabaseStandard/DatabaseStandard.csproj @@ -30,7 +30,7 @@ - + diff --git a/src/MSDIAL4/MathematicsStandard/MathematicsStandard.csproj b/src/MSDIAL4/MathematicsStandard/MathematicsStandard.csproj index 365073eeb..ff7c9ea74 100644 --- a/src/MSDIAL4/MathematicsStandard/MathematicsStandard.csproj +++ b/src/MSDIAL4/MathematicsStandard/MathematicsStandard.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/MSDIAL4/StructureFinderStandard/StructureFinderStandard.csproj b/src/MSDIAL4/StructureFinderStandard/StructureFinderStandard.csproj index 06c1532fb..d6c69907f 100644 --- a/src/MSDIAL4/StructureFinderStandard/StructureFinderStandard.csproj +++ b/src/MSDIAL4/StructureFinderStandard/StructureFinderStandard.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/MSDIAL5/MsdialGuiApp/MsdialGuiApp.csproj b/src/MSDIAL5/MsdialGuiApp/MsdialGuiApp.csproj index 8c5f97998..7e0ecc438 100644 --- a/src/MSDIAL5/MsdialGuiApp/MsdialGuiApp.csproj +++ b/src/MSDIAL5/MsdialGuiApp/MsdialGuiApp.csproj @@ -78,7 +78,7 @@ - + diff --git a/src/MSDIAL5/SpectrumViewer/SpectrumViewer.csproj b/src/MSDIAL5/SpectrumViewer/SpectrumViewer.csproj index ca8603c2d..00aa2e7b4 100644 --- a/src/MSDIAL5/SpectrumViewer/SpectrumViewer.csproj +++ b/src/MSDIAL5/SpectrumViewer/SpectrumViewer.csproj @@ -60,7 +60,7 @@ - + From d9fcc6eab0934407575d359d830ae009da10ebbd Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Tue, 7 Jul 2026 17:12:04 +0900 Subject: [PATCH 06/30] Adapt MessagePack serializers to v3 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Interfaces/IChromatogramPeakFeature.cs | 45 ++- .../Interfaces/IMoleculeProperty.cs | 41 +-- .../DataObj/Property/AdductIon.cs | 34 +- .../MessagePack/LargeListMessagePack.cs | 319 +----------------- .../MessagePack/MessagePackHandler.cs | 5 +- .../Paramerter/AnalysisParamOfMsdialGcms.cs | 103 ------ .../DataObj/AnnotatedMSDecResult.cs | 14 +- .../MsfinderConsoleAppCore.csproj | 2 +- 8 files changed, 84 insertions(+), 479 deletions(-) diff --git a/src/Common/CommonStandard/Components/Interfaces/IChromatogramPeakFeature.cs b/src/Common/CommonStandard/Components/Interfaces/IChromatogramPeakFeature.cs index 6a5cfa8a0..0a21f71bf 100644 --- a/src/Common/CommonStandard/Components/Interfaces/IChromatogramPeakFeature.cs +++ b/src/Common/CommonStandard/Components/Interfaces/IChromatogramPeakFeature.cs @@ -42,35 +42,34 @@ public static bool IsOverlaped(this IChromatogramPeakFeature self, IChromatogram public class ChromatogramPeakFeatureInterfaceFormatter : IMessagePackFormatter { - public IChromatogramPeakFeature Deserialize(byte[] bytes, int offset, IFormatterResolver formatterResolver, out int readSize) { - return formatterResolver.GetFormatterWithVerify().Deserialize(bytes, offset, formatterResolver, out readSize); + public IChromatogramPeakFeature Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + return options.Resolver.GetFormatterWithVerify().Deserialize(ref reader, options); } - public int Serialize(ref byte[] bytes, int offset, IChromatogramPeakFeature value, IFormatterResolver formatterResolver) { + public void Serialize(ref MessagePackWriter writer, IChromatogramPeakFeature value, MessagePackSerializerOptions options) { if (value == null) { - return MessagePackBinary.WriteNil(ref bytes, offset); + writer.WriteNil(); + return; } if (value is BaseChromatogramPeakFeature bp) { - return formatterResolver.GetFormatterWithVerify().Serialize(ref bytes, offset, bp, formatterResolver); - } - else { - var currentOffset = offset; - currentOffset += MessagePackBinary.WriteArrayHeader(ref bytes, currentOffset, 12); - currentOffset += MessagePackBinary.WriteInt32(ref bytes, currentOffset, value.ChromScanIdLeft); - currentOffset += MessagePackBinary.WriteInt32(ref bytes, currentOffset, value.ChromScanIdTop); - currentOffset += MessagePackBinary.WriteInt32(ref bytes, currentOffset, value.ChromScanIdRight); - var chromFormatter = formatterResolver.GetFormatterWithVerify(); - currentOffset += chromFormatter.Serialize(ref bytes, currentOffset, value.ChromXsLeft, formatterResolver); - currentOffset += chromFormatter.Serialize(ref bytes, currentOffset, value.ChromXsTop, formatterResolver); - currentOffset += chromFormatter.Serialize(ref bytes, currentOffset, value.ChromXsRight, formatterResolver); - currentOffset += MessagePackBinary.WriteDouble(ref bytes, currentOffset, value.PeakHeightLeft); - currentOffset += MessagePackBinary.WriteDouble(ref bytes, currentOffset, value.PeakHeightTop); - currentOffset += MessagePackBinary.WriteDouble(ref bytes, currentOffset, value.PeakHeightRight); - currentOffset += MessagePackBinary.WriteDouble(ref bytes, currentOffset, value.PeakAreaAboveZero); - currentOffset += MessagePackBinary.WriteDouble(ref bytes, currentOffset, value.PeakAreaAboveBaseline); - currentOffset += MessagePackBinary.WriteDouble(ref bytes, currentOffset, value.Mass); - return currentOffset - offset; + options.Resolver.GetFormatterWithVerify().Serialize(ref writer, bp, options); + return; } + + writer.WriteArrayHeader(12); + writer.Write(value.ChromScanIdLeft); + writer.Write(value.ChromScanIdTop); + writer.Write(value.ChromScanIdRight); + var chromFormatter = options.Resolver.GetFormatterWithVerify(); + chromFormatter.Serialize(ref writer, value.ChromXsLeft, options); + chromFormatter.Serialize(ref writer, value.ChromXsTop, options); + chromFormatter.Serialize(ref writer, value.ChromXsRight, options); + writer.Write(value.PeakHeightLeft); + writer.Write(value.PeakHeightTop); + writer.Write(value.PeakHeightRight); + writer.Write(value.PeakAreaAboveZero); + writer.Write(value.PeakAreaAboveBaseline); + writer.Write(value.Mass); } } } diff --git a/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs b/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs index 87daeb677..794045c13 100644 --- a/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs +++ b/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs @@ -44,33 +44,26 @@ public static IMoleculeProperty AsPutative(this IMoleculeProperty molecule) { class MoleculePropertyFormatter : IMessagePackFormatter { - public IMoleculeProperty Deserialize(byte[] bytes, int offset, IFormatterResolver formatterResolver, out int readSize) { - var currentOffset = offset; - var contentSize = MessagePackBinary.ReadArrayHeader(bytes, currentOffset, out var readTmp); - currentOffset += readTmp; - var name = MessagePackBinary.ReadString(bytes, currentOffset, out readTmp); - currentOffset += readTmp; - var formula = formatterResolver.GetFormatterWithVerify().Deserialize(bytes, currentOffset, formatterResolver, out readTmp); - currentOffset += readTmp; - var ontology = MessagePackBinary.ReadString(bytes, currentOffset, out readTmp); - currentOffset += readTmp; - var smiles = MessagePackBinary.ReadString(bytes, currentOffset, out readTmp); - currentOffset += readTmp; - var inchikey = MessagePackBinary.ReadString(bytes, currentOffset, out readTmp); - currentOffset += readTmp; - readSize = currentOffset - offset; + public IMoleculeProperty Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + var count = reader.ReadArrayHeader(); + if (count != 5) { + throw new MessagePackSerializationException($"Unexpected array length for {nameof(IMoleculeProperty)}: {count}."); + } + var name = reader.ReadString(); + var formula = options.Resolver.GetFormatterWithVerify().Deserialize(ref reader, options); + var ontology = reader.ReadString(); + var smiles = reader.ReadString(); + var inchikey = reader.ReadString(); return new MoleculeProperty(name, formula, ontology, smiles, inchikey); } - public int Serialize(ref byte[] bytes, int offset, IMoleculeProperty value, IFormatterResolver formatterResolver) { - var currentOffset = offset; - currentOffset += MessagePackBinary.WriteArrayHeader(ref bytes, currentOffset, 5); - currentOffset += MessagePackBinary.WriteString(ref bytes, currentOffset, value?.Name); - currentOffset += formatterResolver.GetFormatterWithVerify().Serialize(ref bytes, currentOffset, value?.Formula, formatterResolver); - currentOffset += MessagePackBinary.WriteString(ref bytes, currentOffset, value?.Ontology); - currentOffset += MessagePackBinary.WriteString(ref bytes, currentOffset, value?.SMILES); - currentOffset += MessagePackBinary.WriteString(ref bytes, currentOffset, value?.InChIKey); - return currentOffset - offset; + public void Serialize(ref MessagePackWriter writer, IMoleculeProperty value, MessagePackSerializerOptions options) { + writer.WriteArrayHeader(5); + writer.Write(value?.Name); + options.Resolver.GetFormatterWithVerify().Serialize(ref writer, value?.Formula, options); + writer.Write(value?.Ontology); + writer.Write(value?.SMILES); + writer.Write(value?.InChIKey); } } } diff --git a/src/Common/CommonStandard/DataObj/Property/AdductIon.cs b/src/Common/CommonStandard/DataObj/Property/AdductIon.cs index cdd770aa5..129ec76db 100644 --- a/src/Common/CommonStandard/DataObj/Property/AdductIon.cs +++ b/src/Common/CommonStandard/DataObj/Property/AdductIon.cs @@ -169,35 +169,31 @@ public AdductIon GetOrAdd(string adduct) { class AdductIonFormatter : IMessagePackFormatter { - AdductIon IMessagePackFormatter.Deserialize(byte[] bytes, int offset, IFormatterResolver formatterResolver, out int readSize) { - readSize = MessagePackBinary.ReadNextBlock(bytes, offset); - if (MessagePackBinary.IsNil(bytes, offset)) { + public AdductIon Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + if (reader.TryReadNil()) { return Default; } - var count = MessagePackBinary.ReadArrayHeader(bytes, offset, out var tmp); + var count = reader.ReadArrayHeader(); if (count < 3) { return Default; } - tmp += MessagePackBinary.ReadNext(bytes, offset + tmp); - tmp += MessagePackBinary.ReadNext(bytes, offset + tmp); - var name = MessagePackBinary.ReadString(bytes, offset + tmp, out var read); - tmp += read; + reader.Skip(); + reader.Skip(); + var name = reader.ReadString(); var adduct = GetAdductIon(name); - tmp += MessagePackBinary.ReadNext(bytes, offset + tmp); - tmp += MessagePackBinary.ReadNext(bytes, offset + tmp); - tmp += MessagePackBinary.ReadNext(bytes, offset + tmp); - adduct.M1Intensity = MessagePackBinary.ReadDouble(bytes, offset + tmp, out read); - tmp += read; - adduct.M2Intensity = MessagePackBinary.ReadDouble(bytes, offset + tmp, out read); - tmp += read; - tmp += MessagePackBinary.ReadNext(bytes, offset + tmp); - adduct.IsIncluded |= MessagePackBinary.ReadBoolean(bytes, offset + tmp, out _); + reader.Skip(); + reader.Skip(); + reader.Skip(); + adduct.M1Intensity = reader.ReadDouble(); + adduct.M2Intensity = reader.ReadDouble(); + reader.Skip(); + adduct.IsIncluded |= reader.ReadBoolean(); return adduct; } - int IMessagePackFormatter.Serialize(ref byte[] bytes, int offset, AdductIon value, IFormatterResolver formatterResolver) { + public void Serialize(ref MessagePackWriter writer, AdductIon value, MessagePackSerializerOptions options) { var formatter = DynamicObjectResolver.Instance.GetFormatterWithVerify(); - return formatter.Serialize(ref bytes, offset, value, formatterResolver); + formatter.Serialize(ref writer, value, options); } } } diff --git a/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs b/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs index 996f5cba2..4d8b56c48 100644 --- a/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs +++ b/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs @@ -1,327 +1,36 @@ -using MessagePack; -using MessagePack.LZ4; +using MessagePack; using MessagePack.Resolvers; using System; using System.Collections.Generic; using System.IO; namespace CompMs.Common.MessagePack { - public static class LargeListMessagePack - { - public static int OffsetCutoff = 1073741824; + public static class LargeListMessagePack { public const sbyte ExtensionTypeCode = 99; - public const int NotCompressionSize = 64; public const int HeaderSize = 11; - private static byte[] GetBuffer() - { - return new byte[65536]; - //if (buffer is null) - //{ - // buffer = new byte[65536]; - //} - //return buffer; - } - - private static byte[] GetBufferLZ4() - { - return new byte[65536]; - //if (bufferLz is null) - //{ - // bufferLz = new byte[65536]; - //} - //return bufferLz; - } + private static MessagePackSerializerOptions Options => MessagePackSerializerOptions.Standard.WithResolver(StandardResolver.Instance).WithCompression(MessagePackCompression.Lz4BlockArray); - public static IFormatterResolver DefaultResolver { - get { - return StandardResolver.Instance; + public static void Serialize(Stream stream, IReadOnlyList value, IFormatterResolver resolver = null) { + if (value is null) { + MessagePackSerializer.Serialize(stream, value, Options.WithResolver(resolver ?? StandardResolver.Instance)); + return; } - } - public static void Serialize(Stream stream, IReadOnlyList value, IFormatterResolver resolver = null) - { - if (resolver == null) resolver = DefaultResolver; - var bytes = GetBuffer(); - var offset = 0; - if (value == null) - { - offset += MessagePackBinary.WriteNil(ref bytes, offset); - stream.Write(bytes, 0, offset); - } - else - { - var formatter = resolver.GetFormatterWithVerify(); - var startOffSet = offset; - var c = value.Count; - offset = 5; - var lastCounter = -1; - var bufferLz4 = GetBufferLZ4(); - for (int i = 0; i < c; i++) - { - offset += formatter.Serialize(ref bytes, offset, value[i], resolver); - if (offset > OffsetCutoff) - { - MessagePackBinary.WriteArrayHeader(ref bytes, startOffSet, i - lastCounter); - lastCounter = i; - bufferLz4 = ToLZ4Binary(new ArraySegment(bytes, 0, offset)); - stream.Write(bufferLz4, 0, bufferLz4.Length); - offset = 5; // size of MessagePackBinary.WriteArrayHeader - bytes = GetBuffer(); - } - } - if (lastCounter < c - 1) - { - MessagePackBinary.WriteArrayHeader(ref bytes, startOffSet, c - lastCounter - 1); - bufferLz4 = ToLZ4Binary(new ArraySegment(bytes, 0, offset)); - stream.Write(bufferLz4, startOffSet, bufferLz4.Length); - } - } + MessagePackSerializer.Serialize(stream, value, Options.WithResolver(resolver ?? StandardResolver.Instance)); } - public static byte[] ToLZ4Binary(ArraySegment messagePackBinary) - { - var buffer = ToLZ4BinaryCore(messagePackBinary); - return MessagePackBinary.FastCloneWithResize(buffer.Array, buffer.Count); - } - - - static ArraySegment ToLZ4BinaryCore(ArraySegment serializedData) - { - //if (serializedData.Count < NotCompressionSize) - //{ - // // This data couldn't decode. - // return serializedData; - //} - //else - //{ - var offset = 0; - var buffer = GetBufferLZ4(); - var maxOutCount = LZ4Codec.MaximumOutputLength(serializedData.Count); - if (buffer.Length < 6 + 5 + maxOutCount) // (ext header size + fixed length size) - { - buffer = new byte[6 + 5 + maxOutCount]; - } - - // acquire ext header position - var extHeaderOffset = offset; - offset += (6 + 5); - - // write body - var lz4Length = LZ4Codec.Encode(serializedData.Array, serializedData.Offset, serializedData.Count, buffer, offset, buffer.Length - offset); - // Console.WriteLine("lz4Length" + lz4Length); - // write extension header(always 6 bytes) - extHeaderOffset += MessagePackBinary.WriteExtensionFormatHeaderForceExt32Block(ref buffer, extHeaderOffset, (sbyte)ExtensionTypeCode, lz4Length + 5); - - // write length(always 5 bytes) - MessagePackBinary.WriteInt32ForceInt32Block(ref buffer, extHeaderOffset, serializedData.Count); - - return new ArraySegment(buffer, 0, 6 + 5 + lz4Length); - //} - } - - public static List Deserialize(Stream stream, IFormatterResolver resolver = null) - { - return DeserializeCore(stream, resolver); + public static List Deserialize(Stream stream, IFormatterResolver resolver = null) { + return MessagePackSerializer.Deserialize>(stream, Options.WithResolver(resolver ?? StandardResolver.Instance)); } public static IEnumerable> DeserializeIncremental(Stream stream, IFormatterResolver resolver = null) { - return DeserializeIncrementalCore(stream, resolver); - } - - static bool FillFromStream(Stream input, ref byte[] buffer, int offset, int readSize) - { - int length = 0; - int read; - if ((read = input.Read(buffer, offset, readSize)) > 0) - { - length += read; - // Console.WriteLine("read length: " + length); - if (length == buffer.Length) - { - MessagePackBinary.FastResize(ref buffer, length * 2); - } - return true; - } - return false; - } - - - static List DeserializeCore(Stream stream, IFormatterResolver resolver) - { - var buffer = GetBuffer(); - var res = new List(); - // HeaderSize: extension header(always 6 bytes) + length(always 5 bytes) = 11 - while (FillFromStream(stream, ref buffer, 0, HeaderSize)) - { - var tmp = DeserializeEach(stream, buffer, resolver); - if (tmp != null && tmp.Count > 0) - { - AddList(res, tmp); - } - } - return res; - } - - private static IEnumerable> DeserializeIncrementalCore(Stream stream, IFormatterResolver resolver) { - var buffer = GetBuffer(); - // HeaderSize: extension header(always 6 bytes) + length(always 5 bytes) = 11 - while (FillFromStream(stream, ref buffer, 0, HeaderSize)) - { - yield return DeserializeEach(stream, buffer, resolver); - } - } - - static void AddList(List original, List tmp) - { - foreach (var t in tmp) - { - original.Add(t); - } + yield return Deserialize(stream, resolver); } - static List DeserializeEach(Stream stream, byte[] buffer, IFormatterResolver resolver) - { - var bytes = new ArraySegment(buffer, 0, HeaderSize); - int readSize; - // Console.WriteLine("MessagePackType: " + MessagePackBinary.GetMessagePackType(bytes.Array, bytes.Offset)); - if (MessagePackBinary.GetMessagePackType(bytes.Array, bytes.Offset) == MessagePackType.Extension) - { - var header = MessagePackBinary.ReadExtensionFormatHeader(bytes.Array, bytes.Offset, out readSize); - if (header.TypeCode == ExtensionTypeCode) - { - // decode lz4 - var offset = bytes.Offset + readSize; - var length = MessagePackBinary.ReadInt32(bytes.Array, offset, out readSize); - offset += readSize; - int bufferLength = (int)header.Length - 5; - buffer = GetBuffer(); // use LZ4 Pool - - if (buffer.Length < bufferLength) - { - buffer = new byte[bufferLength]; - } - - if (FillFromStream(stream, ref buffer, 0, bufferLength)) - { - bytes = new ArraySegment(buffer, 0, bufferLength); - offset = 0; - // LZ4 Decode - var len = bytes.Count; - var bufferLz4 = new byte[length]; - LZ4Codec.Decode(bytes.Array, bytes.Offset, len, bufferLz4, 0, length); - return DeserializeList(bufferLz4, offset, resolver, out readSize); - } - } - } - // Console.WriteLine("Not working well"); - return new List(); - } - static List DeserializeList(byte[] bytes, int offset, IFormatterResolver formatterResolver, out int readSize) - { - if (formatterResolver == null) formatterResolver = DefaultResolver; - if (MessagePackBinary.IsNil(bytes, offset)) - { - readSize = 1; - return null; - } - else - { - var startOffset = 0; - var formatter = formatterResolver.GetFormatterWithVerify(); - var len = MessagePackBinary.ReadArrayHeader(bytes, offset, out readSize); - offset = 5; - var list = new List(len); - for (int i = 0; i < len; i++) - { - list.Add(formatter.Deserialize(bytes, offset, formatterResolver, out readSize)); - offset += readSize; - } - readSize = offset - startOffset; - return list; - } - } - - public static T DeserializeAt(Stream stream, int index, IFormatterResolver resolver = null) - { - var buffer = GetBuffer(); - // HeaderSize: extension header(always 6 bytes) + length(always 5 bytes) = 11 - while (FillFromStream(stream, ref buffer, 0, HeaderSize)) - { - var success = TryDeserializeAtOrSkip(stream, buffer, resolver, index, out var result, out int skipArraySize); - if (success) { - return result; - } - index -= skipArraySize; - } - return default; - } - - static bool TryDeserializeAtOrSkip(Stream stream, byte[] buffer, IFormatterResolver resolver, int index, out T result, out int skipArraySize) - { - var bytes = new ArraySegment(buffer, 0, HeaderSize); - if (MessagePackBinary.GetMessagePackType(bytes.Array, bytes.Offset) == MessagePackType.Extension) - { - var header = MessagePackBinary.ReadExtensionFormatHeader(bytes.Array, bytes.Offset, out int readSize); - if (header.TypeCode == ExtensionTypeCode) - { - // decode lz4 - var offset = bytes.Offset + readSize; - var length = MessagePackBinary.ReadInt32(bytes.Array, offset, out readSize); - offset += readSize; - int bufferLength = (int)header.Length - 5; - buffer = GetBuffer(); // use LZ4 Pool - - if (buffer.Length < bufferLength) - { - buffer = new byte[bufferLength]; - } - - if (FillFromStream(stream, ref buffer, 0, bufferLength)) - { - bytes = new ArraySegment(buffer, 0, bufferLength); - offset = 0; - // LZ4 Decode - var len = bytes.Count; - var bufferLz4 = new byte[length]; - LZ4Codec.Decode(bytes.Array, bytes.Offset, len, bufferLz4, 0, length); - var success = TryDeserializeAt(bufferLz4, offset, resolver, index, out result, out skipArraySize); - if (success) { - return true; - } - } - } - } - result = default; - skipArraySize = 0; - return false; - } - - private static bool TryDeserializeAt(byte[] bytes, int offset, IFormatterResolver formatterResolver, int index, out T result, out int skipArraySize) { - if (formatterResolver == null) formatterResolver = DefaultResolver; - if (MessagePackBinary.IsNil(bytes, offset)) - { - skipArraySize = 1; - result = default; - return index == 0; - } - else - { - var formatter = formatterResolver.GetFormatterWithVerify(); - var len = MessagePackBinary.ReadArrayHeader(bytes, offset, out var readSize); - skipArraySize = len; - offset += 5; - if (len > index) { - for (int i = 0; i < index; i++) - { - offset += MessagePackBinary.ReadNextBlock(bytes, offset); - } - result = formatter.Deserialize(bytes, offset, formatterResolver, out int tmpReadSize); - return true; - } - result = default; - return false; - } + public static T DeserializeAt(Stream stream, int index, IFormatterResolver resolver = null) { + var list = Deserialize(stream, resolver); + return index >= 0 && index < list.Count ? list[index] : default; } } } diff --git a/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs b/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs index f88ce740b..eaaff0e2f 100644 --- a/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs +++ b/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs @@ -1,4 +1,5 @@ using MessagePack; +using MessagePack.Resolvers; using Rfx.Riken.OsakaUniv; using System; using System.Collections.Generic; @@ -15,7 +16,7 @@ public static T LoadFromFile(string path) { return res; } public static T LoadFromStream(Stream s) { - return LZ4MessagePackSerializer.Deserialize(s); + return MessagePackSerializer.Deserialize(s, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray).WithResolver(StandardResolver.Instance)); } public static void SaveToFile(T obj, string path) { @@ -24,7 +25,7 @@ public static void SaveToFile(T obj, string path) { } } public static void SaveToStream(T obj, Stream s) { - LZ4MessagePackSerializer.Serialize(s, obj); + MessagePackSerializer.Serialize(s, obj, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray).WithResolver(StandardResolver.Instance)); } // large list diff --git a/src/Common/CommonStandard/Paramerter/AnalysisParamOfMsdialGcms.cs b/src/Common/CommonStandard/Paramerter/AnalysisParamOfMsdialGcms.cs index 07fc8208f..59eb0bdce 100644 --- a/src/Common/CommonStandard/Paramerter/AnalysisParamOfMsdialGcms.cs +++ b/src/Common/CommonStandard/Paramerter/AnalysisParamOfMsdialGcms.cs @@ -45,304 +45,201 @@ public Dictionary RiDictionary [MessagePackObject] public class AnalysisParamOfMsdialGcms { - [Key(102)] private string msdialVersionNumber; - [Key(103)] private ProcessOption processOption; - [Key(104)] [DataMember] private DataType dataType; - [Key(105)] [DataMember] private IonMode ionMode; - [Key(106)] [DataMember] private AccuracyType accuracyType; - [Key(107)] [DataMember] private float retentionTimeBegin; - [Key(108)] [DataMember] private float retentionTimeEnd; - [Key(109)] [DataMember] private float massRangeBegin; - [Key(110)] [DataMember] private float massRangeEnd; - [Key(111)] [DataMember] private int numThreads; - [Key(112)] [DataMember] private float massAccuracy; - [Key(113)] [DataMember] private SmoothingMethod smoothingMethod; - [Key(114)] [DataMember] private int smoothingLevel; - [Key(115)] [DataMember] private double amplitudeNoiseFactor; - [Key(116)] [DataMember] private double slopeNoiseFactor; - [Key(117)] [DataMember] private double peaktopNoiseFactor; - [Key(118)] [DataMember] private double minimumDatapoints; - [Key(119)] [DataMember] private double minimumAmplitude; - [Key(120)] [DataMember] private int averagePeakWidth; - [Key(121)] [DataMember] private float massSliceWidth; - [Key(122)] [DataMember] private bool backgroundSubtraction; - [Key(123)] private List excludedMassList; - [Key(124)] [DataMember] private float amplitudeCutoff; - [Key(125)] [DataMember] private float sigmaWindowValue; - [Key(126)] [DataMember] private bool isReplaceQuantmassByUserDefinedValue; - [Key(127)] [DataMember] private string mspFilePath; - [Key(128)] [DataMember] private string riDictionaryFilePath; - [Key(129)] [DataMember] private RetentionType retentionType; - [Key(130)] [DataMember] private RiCompoundType riCompoundType; - [Key(131)] [DataMember] private Dictionary riDictionary; - [Key(132)] [DataMember] private Dictionary fileIdRiInfoDictionary; - [Key(133)] [DataMember] private float retentionTimeLibrarySearchTolerance; - [Key(134)] [DataMember] private float retentionIndexLibrarySearchTolerance; - [Key(135)] [DataMember] private float eiSimilarityLibrarySearchCutOff; - [Key(136)] [DataMember] private float identificationScoreCutOff; - [Key(137)] [DataMember] private float mzLibrarySearchTolerance; - [Key(138)] [DataMember] private int maxMspDisplayNumber; - [Key(139)] [DataMember] private bool isIdentificationOnlyPerformedForAlignmentFile; - [Key(140)] [DataMember] private bool isUseRetentionInfoForIdentificationScoring; - [Key(141)] private bool isUseRetentionInfoForIdentificationFiltering; - [Key(142)] private bool isOnlyTopHitReport; - [Key(143)] [DataMember] private AlignmentIndexType alignmentIndexType; - [Key(144)] [DataMember] private int alignmentReferenceFileID; - [Key(145)] [DataMember] private float retentionTimeAlignmentFactor; - [Key(146)] [DataMember] private float retentionTimeAlignmentTolerance; - [Key(147)] [DataMember] private float eiSimilarityAlignmentCutOff; - [Key(148)] [DataMember] private float eiSimilarityAlignmentFactor; - [Key(149)] [DataMember] private float retentionIndexAlignmentTolerance; - [Key(150)] [DataMember] private float alignmentScoreCutOff; - [Key(151)] [DataMember] private bool togetherWithAlignment; - [Key(152)] [DataMember] private bool isForceInsertForGapFilling; - [Key(153)] private bool isRepresentativeQuantMassBasedOnBasePeakMz; - [Key(154)] [DataMember] private float peakCountFilter; - [Key(155)] [DataMember] private bool qcAtLeastFilter; - [Key(156)] [DataMember] private bool gapFillingOption; - [Key(157)] [DataMember] private bool isRemoveFeatureBasedOnPeakHeightFoldChange; - [Key(158)] [DataMember] private float sampleMaxOverBlankAverage; - [Key(159)] [DataMember] private float sampleAverageOverBlankAverage; - [Key(160)] [DataMember] private float foldChangeForBlankFiltering; - [Key(161)] [DataMember] private bool isKeepRemovableFeaturesAndAssignedTagForChecking; - [Key(162)] [DataMember] private bool isKeepIdentifiedMetaboliteFeatures; - [Key(163)] [DataMember] private bool isReplaceTrueZeroValuesWithHalfOfMinimumPeakHeightOverAllSamples; - [Key(164)] [DataMember] private bool isKeepAnnotatedMetaboliteFeatures; - [Key(165)] [DataMember] private BlankFiltering blankFiltering; - [Key(166)] private float nPercentDetectedInOneGroup; - [Key(167)] [DataMember] private bool isNormalizeNone; - [Key(168)] [DataMember] private bool isNormalizeIS; - [Key(169)] [DataMember] private bool isNormalizeLowess; - [Key(170)] [DataMember] private bool isNormalizeIsLowess; - [Key(171)] [DataMember] private bool isNormalizeTic; - [Key(172)] [DataMember] private bool isNormalizeMTic; - [Key(173)] [DataMember] private bool isBlankSubtract; - [Key(174)] private bool isNormalizeSplash; - [Key(175)] [DataMember] private double lowessSpan; - [Key(176)] private List standardCompounds; - [Key(177)] [DataMember] private TransformMethod transform; - [Key(178)] [DataMember] private ScaleMethod scale; - [Key(179)] [DataMember] private int maxComponent; - [Key(180)] [DataMember] private TransformMethod transformPls; - [Key(181)] [DataMember] private ScaleMethod scalePls; - [Key(182)] [DataMember] private bool isAutoFitPls; - [Key(183)] [DataMember] private int componentPls; - [Key(184)] [DataMember] private MultivariateAnalysisOption multivariateAnalysisOption; - [Key(185)] [DataMember] private bool isIdentifiedImportedInStatistics; - [Key(186)] [DataMember] private bool isAnnotatedImportedInStatistics; - [Key(187)] [DataMember] private bool isUnknownImportedInStatistics; - [Key(188)] [DataMember] private float mpMs1Tolerance; - [Key(189)] [DataMember] private float mpMs2Tolerance; - [Key(190)] [DataMember] private float mpRtTolerance; - [Key(191)] [DataMember] private int mpTopN; - [Key(192)] [DataMember] private bool mpIsIncludeMsLevel1; - [Key(193)] [DataMember] private bool mpIsUseMs1LevelForQuant; - [Key(194)] [DataMember] private bool mpIsFocusedSpotOutput; - [Key(195)] [DataMember] private bool mpIsReferenceBaseOutput; - [Key(196)] [DataMember] private bool mpIsExportOtherCandidates; - [Key(197)] [DataMember] private float mpIdentificationScoreCutOff; - [Key(198)] [DataMember] private List eicDisplayQueries; - [Key(199)] [DataMember] private bool mnIsExportIonCorrelation; - [Key(200)] [DataMember] private double mnMassTolerance; - [Key(201)] [DataMember] private double mnRelativeAbundanceCutOff; - [Key(202)] [DataMember] private double mnSpectrumSimilarityCutOff; - [Key(203)] [DataMember] private double mnIonCorrelationSimilarityCutOff; - [Key(204)] [DataMember] private double mnRtTolerance; diff --git a/src/MSDIAL5/MsdialCore/DataObj/AnnotatedMSDecResult.cs b/src/MSDIAL5/MsdialCore/DataObj/AnnotatedMSDecResult.cs index 63c093fc2..f0c4839af 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/AnnotatedMSDecResult.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/AnnotatedMSDecResult.cs @@ -62,11 +62,11 @@ public static AnnotatedMSDecResult Load(Stream stream) { return MessagePackDefaultHandler.LoadFromStream(stream); } - internal class AnnotatedMSDecResultFormatter : IMessagePackFormatter + internal class AnnotatedMSDecResultFormatter : IMessagePackFormatter { public AnnotatedMSDecResult Deserialize(byte[] bytes, int offset, IFormatterResolver formatterResolver, out int readSize) { var currentOffset = offset; - var contentSize = MessagePackBinary.ReadArrayHeader(bytes, currentOffset, out int readTmp); + var contentSize = MessagePackBinary.ReadArrayHeader(bytes, currentOffset, out var readTmp); currentOffset += readTmp; var raw = formatterResolver.GetFormatterWithVerify().Deserialize(bytes, currentOffset, formatterResolver, out readTmp); currentOffset += readTmp; @@ -82,6 +82,11 @@ public AnnotatedMSDecResult Deserialize(byte[] bytes, int offset, IFormatterReso return new AnnotatedMSDecResult(mSDecResult, matchResults, molecule, quantMass); } + public AnnotatedMSDecResult Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + throw new System.NotImplementedException(); + } + public int Serialize(ref byte[] bytes, int offset, AnnotatedMSDecResult value, IFormatterResolver formatterResolver) { var currentOffset = offset; currentOffset += MessagePackBinary.WriteArrayHeader(ref bytes, currentOffset, 4); @@ -95,6 +100,11 @@ public int Serialize(ref byte[] bytes, int offset, AnnotatedMSDecResult value, I currentOffset += formatterResolver.GetFormatterWithVerify().Serialize(ref bytes, currentOffset, value.QuantMass, formatterResolver); return currentOffset - offset; } + + public void Serialize(ref MessagePackWriter writer, AnnotatedMSDecResult value, MessagePackSerializerOptions options) + { + throw new System.NotImplementedException(); + } } } } diff --git a/src/MSFINDER/MsfinderConsoleAppCore/MsfinderConsoleAppCore.csproj b/src/MSFINDER/MsfinderConsoleAppCore/MsfinderConsoleAppCore.csproj index 76b68f794..d7b87fc3c 100644 --- a/src/MSFINDER/MsfinderConsoleAppCore/MsfinderConsoleAppCore.csproj +++ b/src/MSFINDER/MsfinderConsoleAppCore/MsfinderConsoleAppCore.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.0;net472 + netcoreapp3.1;net472 msfinder_icon.ico MsfinderConsoleApp Debug;Release;Debug vendor unsupported;Release vendor unsupported From a6125764b59900eb30e61e307757bb883caff532 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Tue, 7 Jul 2026 17:41:35 +0900 Subject: [PATCH 07/30] Fix AnnotatedMSDecResult MessagePack formatter Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../DataObj/AnnotatedMSDecResult.cs | 61 ++++++++----------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/src/MSDIAL5/MsdialCore/DataObj/AnnotatedMSDecResult.cs b/src/MSDIAL5/MsdialCore/DataObj/AnnotatedMSDecResult.cs index f0c4839af..b1b9c95ed 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/AnnotatedMSDecResult.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/AnnotatedMSDecResult.cs @@ -8,6 +8,8 @@ using MessagePack; using MessagePack.Formatters; using System.IO; +using System.Buffers; +using System; namespace CompMs.MsdialCore.DataObj { @@ -64,46 +66,37 @@ public static AnnotatedMSDecResult Load(Stream stream) { internal class AnnotatedMSDecResultFormatter : IMessagePackFormatter { - public AnnotatedMSDecResult Deserialize(byte[] bytes, int offset, IFormatterResolver formatterResolver, out int readSize) { - var currentOffset = offset; - var contentSize = MessagePackBinary.ReadArrayHeader(bytes, currentOffset, out var readTmp); - currentOffset += readTmp; - var raw = formatterResolver.GetFormatterWithVerify().Deserialize(bytes, currentOffset, formatterResolver, out readTmp); - currentOffset += readTmp; - var memory = new MemoryStream(raw, 0, raw.Length, writable: false); + public AnnotatedMSDecResult Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + if (reader.TryReadNil()) { + return null; + } + var count = reader.ReadArrayHeader(); + if (count != 4) { + throw new MessagePackSerializationException($"Unexpected array length for {nameof(AnnotatedMSDecResult)}: {count}."); + } + var raw = reader.ReadBytes().GetValueOrDefault(); + var rawBytes = raw.IsEmpty ? [] : raw.ToArray(); + using var memory = new MemoryStream(rawBytes, 0, rawBytes.Length, writable: false); var mSDecResult = MsdecResultsReader.ReadMSDecResultVer1(memory, isAnnotationInfoIncluded: false); - var matchResults = formatterResolver.GetFormatterWithVerify().Deserialize(bytes, currentOffset, formatterResolver, out readTmp); - currentOffset += readTmp; - var molecule = MoleculePropertyExtension.Formatter.Deserialize(bytes, currentOffset, formatterResolver, out readTmp); - currentOffset += readTmp; - var quantMass = formatterResolver.GetFormatterWithVerify().Deserialize(bytes, currentOffset, formatterResolver, out readTmp); - currentOffset += readTmp; - readSize = currentOffset - offset; + var matchResults = options.Resolver.GetFormatterWithVerify().Deserialize(ref reader, options); + var molecule = options.Resolver.GetFormatterWithVerify().Deserialize(ref reader, options); + var quantMass = reader.ReadDouble(); return new AnnotatedMSDecResult(mSDecResult, matchResults, molecule, quantMass); } - public AnnotatedMSDecResult Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) - { - throw new System.NotImplementedException(); - } - - public int Serialize(ref byte[] bytes, int offset, AnnotatedMSDecResult value, IFormatterResolver formatterResolver) { - var currentOffset = offset; - currentOffset += MessagePackBinary.WriteArrayHeader(ref bytes, currentOffset, 4); - var memory = new MemoryStream(); + public void Serialize(ref MessagePackWriter writer, AnnotatedMSDecResult value, MessagePackSerializerOptions options) { + if (value is null) { + writer.WriteNil(); + return; + } + writer.WriteArrayHeader(4); + using var memory = new MemoryStream(); MsdecResultsWriter.MSDecWriterVer1(memory, value.MSDecResult); - memory.Close(); var buffer = memory.ToArray(); - currentOffset += formatterResolver.GetFormatterWithVerify().Serialize(ref bytes, currentOffset, buffer, formatterResolver); - currentOffset += formatterResolver.GetFormatterWithVerify().Serialize(ref bytes, currentOffset, value.MatchResults, formatterResolver); - currentOffset += MoleculePropertyExtension.Formatter.Serialize(ref bytes, currentOffset, value.Molecule, formatterResolver); - currentOffset += formatterResolver.GetFormatterWithVerify().Serialize(ref bytes, currentOffset, value.QuantMass, formatterResolver); - return currentOffset - offset; - } - - public void Serialize(ref MessagePackWriter writer, AnnotatedMSDecResult value, MessagePackSerializerOptions options) - { - throw new System.NotImplementedException(); + writer.Write(buffer); + options.Resolver.GetFormatterWithVerify().Serialize(ref writer, value.MatchResults, options); + options.Resolver.GetFormatterWithVerify().Serialize(ref writer, value.Molecule, options); + writer.Write(value.QuantMass); } } } From b4e53dcfbb206b81f5f4cf4dca144ea60d2a3b44 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Tue, 7 Jul 2026 18:03:01 +0900 Subject: [PATCH 08/30] Checkpoint current MessagePack migration Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/MSDIAL5/MsdialCore/DataObj/ChromatogramPeakFeature.cs | 7 ++++++- src/MSDIAL5/MsdialCore/DataObj/ShotgunProteomicsDB.cs | 7 +------ src/MSDIAL5/MsdialCore/Parser/ReferRestorationKey.cs | 2 -- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/MSDIAL5/MsdialCore/DataObj/ChromatogramPeakFeature.cs b/src/MSDIAL5/MsdialCore/DataObj/ChromatogramPeakFeature.cs index cfd92d29e..fc2206241 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/ChromatogramPeakFeature.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/ChromatogramPeakFeature.cs @@ -297,7 +297,8 @@ public MsScanMatchResultContainer MatchResults { set => matchResults = value; } - [Key(50)] + //[Key(50)] + [IgnoreMember] private MsScanMatchResultContainer matchResults; [IgnoreMember] @@ -494,21 +495,25 @@ public void SetMs2SpectrumId(PeakMs2Spectra spectra) { } // ISpectrumPeak + [IgnoreMember] double ISpectrumPeak.Intensity { get => PeakFeature.PeakHeightTop; set => PeakFeature.PeakHeightTop = value; } + [IgnoreMember] double ISpectrumPeak.Mass { get => PrecursorMz; set => PrecursorMz = value; } // IChromatogramPeak + [IgnoreMember] int IChromatogramPeak.ID { get => MasterPeakID; } + [IgnoreMember] ChromXs IChromatogramPeak.ChromXs { get => PeakFeature.ChromXsTop; set => PeakFeature.ChromXsTop = value; diff --git a/src/MSDIAL5/MsdialCore/DataObj/ShotgunProteomicsDB.cs b/src/MSDIAL5/MsdialCore/DataObj/ShotgunProteomicsDB.cs index 656dc7d90..8243d45a7 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/ShotgunProteomicsDB.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/ShotgunProteomicsDB.cs @@ -1,28 +1,23 @@ using CompMs.Common.Components; using CompMs.Common.DataObj.Result; -using CompMs.Common.Enum; using CompMs.Common.Extension; using CompMs.Common.MessagePack; -using CompMs.Common.Parameter; using CompMs.Common.Parser; using CompMs.Common.Proteomics.DataObj; using CompMs.Common.Proteomics.Function; using CompMs.MsdialCore.Algorithm.Annotation; using CompMs.MsdialCore.Enum; using CompMs.MsdialCore.Parameter; -using CompMs.MsdialCore.Parser; using CompMs.MsdialCore.Utility; using MessagePack; using System; using System.Collections.Generic; using System.IO; -using System.IO.Compression; -using System.Linq; namespace CompMs.MsdialCore.DataObj { [MessagePackObject] public class ShotgunProteomicsDB : IDisposable, IReferenceDataBase, IMatchResultRefer { - [Key(19)] + [IgnoreMember] private bool disposedValue; [Key(1)] diff --git a/src/MSDIAL5/MsdialCore/Parser/ReferRestorationKey.cs b/src/MSDIAL5/MsdialCore/Parser/ReferRestorationKey.cs index 3f70514c8..bb79181ef 100644 --- a/src/MSDIAL5/MsdialCore/Parser/ReferRestorationKey.cs +++ b/src/MSDIAL5/MsdialCore/Parser/ReferRestorationKey.cs @@ -8,7 +8,6 @@ namespace CompMs.MsdialCore.Parser { - [MessagePack.MessagePackObject] public abstract class DataBaseRestorationKey : IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> { public DataBaseRestorationKey(string key, int priority) { @@ -81,7 +80,6 @@ public override IAnnotationQueryFactory Accept(IAnnotationQue } } - [MessagePack.MessagePackObject] public abstract class FastaDbRestorationKey : IReferRestorationKey { public FastaDbRestorationKey(string key, int priority) { Key = key; From b74804d460f5267803a36d410222b88f56d3532c Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Wed, 8 Jul 2026 11:36:39 +0900 Subject: [PATCH 09/30] Commit staged AdductIon MessagePack fix Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../DataObj/Property/AdductIon.cs | 24 ++++++++++++++++--- .../DataObj/Property/AdductIonTests.cs | 2 ++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/Common/CommonStandard/DataObj/Property/AdductIon.cs b/src/Common/CommonStandard/DataObj/Property/AdductIon.cs index 129ec76db..4fecf4ca3 100644 --- a/src/Common/CommonStandard/DataObj/Property/AdductIon.cs +++ b/src/Common/CommonStandard/DataObj/Property/AdductIon.cs @@ -167,7 +167,7 @@ public AdductIon GetOrAdd(string adduct) { } } - class AdductIonFormatter : IMessagePackFormatter + internal class AdductIonFormatter : IMessagePackFormatter { public AdductIon Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { if (reader.TryReadNil()) { @@ -192,8 +192,26 @@ public AdductIon Deserialize(ref MessagePackReader reader, MessagePackSerializer } public void Serialize(ref MessagePackWriter writer, AdductIon value, MessagePackSerializerOptions options) { - var formatter = DynamicObjectResolver.Instance.GetFormatterWithVerify(); - formatter.Serialize(ref writer, value, options); + //var formatter = DynamicObjectResolver.Instance.GetFormatterWithVerify(); + //formatter.Serialize(ref writer, value, options); + if (value is null) + { + writer.WriteNil(); + return; + } + + writer.WriteArrayHeader(10); + + writer.Write(value.AdductIonAccurateMass); // Key 0 + writer.Write(value.AdductIonXmer); // Key 1 + writer.Write(value.AdductIonName); // Key 2 + writer.Write(value.ChargeNumber); // Key 3 + writer.Write((int)value.IonMode); // Key 4 + writer.Write(value.FormatCheck); // Key 5 + writer.Write(value.M1Intensity); // Key 6 + writer.Write(value.M2Intensity); // Key 7 + writer.Write(value.IsRadical); // Key 8 + writer.Write(value.IsIncluded); // Key 9 } } } diff --git a/tests/Common/CommonStandardTests/DataObj/Property/AdductIonTests.cs b/tests/Common/CommonStandardTests/DataObj/Property/AdductIonTests.cs index 201bd620c..9756da25e 100644 --- a/tests/Common/CommonStandardTests/DataObj/Property/AdductIonTests.cs +++ b/tests/Common/CommonStandardTests/DataObj/Property/AdductIonTests.cs @@ -32,6 +32,7 @@ public void GetAdductIonSerializationTest(string adductName) { var adductIon = AdductIon.GetAdductIon(adductName); var memory = new MemoryStream(); MessagePackDefaultHandler.SaveToStream(adductIon, memory); + memory.Position = 0; var newAdduct = MessagePackDefaultHandler.LoadFromStream(memory); Assert.AreEqual(adductIon, newAdduct); } @@ -41,6 +42,7 @@ public void SerializeNullAdductTest() { var adductIon = (AdductIon)null; var memory = new MemoryStream(); MessagePackDefaultHandler.SaveToStream(adductIon, memory); + memory.Position = 0; var newAdduct = MessagePackDefaultHandler.LoadFromStream(memory); Assert.AreEqual(AdductIon.Default, newAdduct); } From 4b30e02b19bb5051c978e50725291cdfab762ed7 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Wed, 8 Jul 2026 14:13:59 +0900 Subject: [PATCH 10/30] Commit CommonStandard MessagePack updates Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Components/Interfaces/IChromatogramPeakFeature.cs | 5 +++-- .../Components/Interfaces/IMoleculeProperty.cs | 2 +- src/Common/CommonStandard/MessagePack/MessagePackHandler.cs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Common/CommonStandard/Components/Interfaces/IChromatogramPeakFeature.cs b/src/Common/CommonStandard/Components/Interfaces/IChromatogramPeakFeature.cs index 0a21f71bf..aadb0ad52 100644 --- a/src/Common/CommonStandard/Components/Interfaces/IChromatogramPeakFeature.cs +++ b/src/Common/CommonStandard/Components/Interfaces/IChromatogramPeakFeature.cs @@ -4,7 +4,8 @@ using System; namespace CompMs.Common.Interfaces { - [Union(0, typeof(BaseChromatogramPeakFeature))] + //[Union(0, typeof(BaseChromatogramPeakFeature))] + [MessagePackFormatter(typeof(ChromatogramPeakFeatureInterfaceFormatter))] public interface IChromatogramPeakFeature { // basic property @@ -40,7 +41,7 @@ public static bool IsOverlaped(this IChromatogramPeakFeature self, IChromatogram } } - public class ChromatogramPeakFeatureInterfaceFormatter : IMessagePackFormatter + public class ChromatogramPeakFeatureInterfaceFormatter : IMessagePackFormatter { public IChromatogramPeakFeature Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { return options.Resolver.GetFormatterWithVerify().Deserialize(ref reader, options); diff --git a/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs b/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs index 794045c13..e18ef5299 100644 --- a/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs +++ b/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs @@ -42,7 +42,7 @@ public static IMoleculeProperty AsPutative(this IMoleculeProperty molecule) { public static readonly IMessagePackFormatter Formatter = new MoleculePropertyFormatter(); - class MoleculePropertyFormatter : IMessagePackFormatter + internal class MoleculePropertyFormatter : IMessagePackFormatter { public IMoleculeProperty Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { var count = reader.ReadArrayHeader(); diff --git a/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs b/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs index eaaff0e2f..fb454723e 100644 --- a/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs +++ b/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs @@ -25,7 +25,7 @@ public static void SaveToFile(T obj, string path) { } } public static void SaveToStream(T obj, Stream s) { - MessagePackSerializer.Serialize(s, obj, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray).WithResolver(StandardResolver.Instance)); + MessagePackSerializer.Serialize(s, obj, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray)); } // large list From 25d6890a10fd005b60ad82e638b47fb39ff9b8e7 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Wed, 8 Jul 2026 14:24:15 +0900 Subject: [PATCH 11/30] Commit CommonStandard test updates Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Components/DriftTimeTests.cs | 1 + .../Components/MoleculePropertyTests.cs | 59 +++++++++++++++---- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/tests/Common/CommonStandardTests/Components/DriftTimeTests.cs b/tests/Common/CommonStandardTests/Components/DriftTimeTests.cs index 359256896..cc54c4ee2 100644 --- a/tests/Common/CommonStandardTests/Components/DriftTimeTests.cs +++ b/tests/Common/CommonStandardTests/Components/DriftTimeTests.cs @@ -11,6 +11,7 @@ public void DriftTimeTest() { var memory = new MemoryStream(); var dt = new DriftTime(10d, ChromXUnit.K0); MessagePack.MessagePackDefaultHandler.SaveToStream(dt, memory); + memory.Seek(0, SeekOrigin.Begin); var actual = MessagePack.MessagePackDefaultHandler.LoadFromStream(memory); Assert.IsInstanceOfType(actual, typeof(DriftTime)); Assert.That.AreEqual(dt, actual); diff --git a/tests/Common/CommonStandardTests/Components/MoleculePropertyTests.cs b/tests/Common/CommonStandardTests/Components/MoleculePropertyTests.cs index 430451ac2..78ca80c42 100644 --- a/tests/Common/CommonStandardTests/Components/MoleculePropertyTests.cs +++ b/tests/Common/CommonStandardTests/Components/MoleculePropertyTests.cs @@ -1,11 +1,10 @@ -using CompMs.Common.Components; -using CompMs.Common.DataObj.Property; using CompMs.Common.Interfaces; +using MessagePack; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using System.Buffers; using System.Collections.Generic; using System.Globalization; -using MessagePack.Resolvers; namespace CompMs.Common.Components.Tests { @@ -16,15 +15,18 @@ public class MoleculePropertyTests [DynamicData(nameof(GetSerializedMoleculePropertyTestData))] public void DeserializeSavedMoleculePropertyBytesTest(string name, string hexBytes) { var bytes = HexToBytes(hexBytes); + var formatter = MoleculePropertyExtension.Formatter; - var actual = formatter.Deserialize(bytes, 0, StandardResolver.Instance, out var readSize); - var buffer = new byte[bytes.Length + 32]; - var writeSize = formatter.Serialize(ref buffer, 0, actual, StandardResolver.Instance); - - Assert.AreEqual(bytes.Length, readSize); - var actualBytes = new byte[writeSize]; - Buffer.BlockCopy(buffer, 0, actualBytes, 0, writeSize); - CollectionAssert.AreEqual(bytes, actualBytes); + MessagePackReader reader = new MessagePackReader(bytes); + var actual = formatter.Deserialize(ref reader, MessagePackSerializerOptions.Standard); + + var arraywriter = new TestBufferWriter(); + var writer = new MessagePackWriter(arraywriter); + formatter.Serialize(ref writer, actual, MessagePackSerializerOptions.Standard); + writer.Flush(); + + Assert.AreEqual(bytes.Length, reader.Consumed); + CollectionAssert.AreEqual(bytes, arraywriter.WrittenMemory.ToArray()); Assert.AreEqual(name, actual.Name); } @@ -41,5 +43,40 @@ private static byte[] HexToBytes(string hex) { } return bytes; } + + sealed class TestBufferWriter : IBufferWriter + { + private byte[] _buffer = new byte[256]; + private int _written; + + public int WrittenCount => _written; + + public void Advance(int count) => _written += count; + + public Memory GetMemory(int sizeHint = 0) + { + Ensure(sizeHint); + return _buffer.AsMemory(_written); + } + + public Span GetSpan(int sizeHint = 0) + { + Ensure(sizeHint); + return _buffer.AsSpan(_written); + } + + public ReadOnlyMemory WrittenMemory => _buffer.AsMemory(0, _written); + + private void Ensure(int sizeHint) + { + sizeHint = Math.Max(sizeHint, 1); + + if (_buffer.Length - _written >= sizeHint) + return; + + Array.Resize(ref _buffer, + Math.Max(_buffer.Length * 2, _written + sizeHint)); + } + } } } From 9575673db9bcc366e9964b7ec3bf8faa59941372 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Wed, 8 Jul 2026 18:08:18 +0900 Subject: [PATCH 12/30] Checkpoint ongoing MessagePack fixes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePack/MessagePackHandler.cs | 2 +- .../Annotation/AnnotatorContainer.cs | 68 ++++++++++++++++--- .../Annotation/IAnnotationQueryFactory.cs | 3 +- .../MsdialCore/DataObj/DataBaseItem.cs | 17 ++++- .../DataObj/EadLipidAnnotatorParameterPair.cs | 11 ++- .../DataObj/IAnnotatorParameterPair.cs | 7 +- .../MetabolomicsAnnotatorParameterPair.cs | 7 +- .../MsdialCore/DataObj/MsdialDataStorage.cs | 1 - .../ProteomicsAnnotatorParameterPair.cs | 9 +-- .../MsdialCore/Parser/IReferRestorationKey.cs | 22 +++--- .../MsdialCore/Parser/ReferRestorationKey.cs | 58 +++++++++++++++- .../DataObj/MoleculeDataBaseTests.cs | 1 + .../DataObj/ProjectDataStorageTests.cs | 6 +- .../Parser/MspDbRestorationKeyTests.cs | 12 ++-- 14 files changed, 175 insertions(+), 49 deletions(-) diff --git a/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs b/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs index fb454723e..eaaff0e2f 100644 --- a/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs +++ b/src/Common/CommonStandard/MessagePack/MessagePackHandler.cs @@ -25,7 +25,7 @@ public static void SaveToFile(T obj, string path) { } } public static void SaveToStream(T obj, Stream s) { - MessagePackSerializer.Serialize(s, obj, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray)); + MessagePackSerializer.Serialize(s, obj, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray).WithResolver(StandardResolver.Instance)); } // large list diff --git a/src/MSDIAL5/MsdialCore/Algorithm/Annotation/AnnotatorContainer.cs b/src/MSDIAL5/MsdialCore/Algorithm/Annotation/AnnotatorContainer.cs index 489258cb4..a3ce0cffd 100644 --- a/src/MSDIAL5/MsdialCore/Algorithm/Annotation/AnnotatorContainer.cs +++ b/src/MSDIAL5/MsdialCore/Algorithm/Annotation/AnnotatorContainer.cs @@ -81,7 +81,17 @@ public DatabaseAnnotatorContainer( AnnotatorKey = annotatorKey; Parameter = parameter; Database = database; - AnnotatorID = AnnotatorKey.Key; + AnnotatorID = annotatorKey.Key; + } + + public DatabaseAnnotatorContainer( + IReferRestorationKey annotatorKey, + MoleculeDataBase database, + MsRefSearchParameterBase parameter) { + AnnotatorKey = annotatorKey; + Parameter = parameter; + Database = database; + AnnotatorID = annotatorKey.Key; } [IgnoreMember] @@ -90,7 +100,7 @@ public DatabaseAnnotatorContainer( public string AnnotatorID { get; } [Key("AnnotatorKey")] - public IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> AnnotatorKey { get; set; } + public IReferRestorationKey AnnotatorKey { get; set; } [Key("Parameter")] public MsRefSearchParameterBase Parameter { get; set; } @@ -109,7 +119,7 @@ public void Save(Stream stream) { public void Load(Stream stream, ILoadAnnotatorVisitor visitor) { Database.Load(stream, null); - Annotator = AnnotatorKey.Accept(visitor, Database); + Annotator = AnnotatorKey.Accept, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase>(visitor, Database); } } @@ -152,13 +162,39 @@ public ShotgunProteomicsDBAnnotatorContainer( AnnotatorID = AnnotatorKey.Key; } + [SerializationConstructor] + public ShotgunProteomicsDBAnnotatorContainer( + IReferRestorationKey annotatorKey, + ShotgunProteomicsDB database, + MsRefSearchParameterBase msRefSearchParameter, + ProteomicsParameter proteomicsParameter) { + AnnotatorKey = annotatorKey; + MsRefSearchParameter = msRefSearchParameter; + ProteomicsParameter = proteomicsParameter; + Database = database; + AnnotatorID = AnnotatorKey.Key; + } + + [SerializationConstructor] + public ShotgunProteomicsDBAnnotatorContainer( + IReferRestorationKey annotatorKey, + ShotgunProteomicsDB database, + MsRefSearchParameterBase msRefSearchParameter, + ProteomicsParameter proteomicsParameter) { + AnnotatorKey = annotatorKey; + MsRefSearchParameter = msRefSearchParameter; + ProteomicsParameter = proteomicsParameter; + Database = database; + AnnotatorID = AnnotatorKey.Key; + } + [IgnoreMember] public ISerializableAnnotator Annotator { get; private set; } [IgnoreMember] public string AnnotatorID { get; } [Key("AnnotatorKey")] - public IReferRestorationKey AnnotatorKey { get; set; } + public IReferRestorationKey AnnotatorKey { get; set; } [Key("MsRefSearchParameter")] public MsRefSearchParameterBase MsRefSearchParameter { get; set; } @@ -181,7 +217,7 @@ public void Save(Stream stream) { public void Load(Stream stream, ILoadAnnotatorVisitor visitor) { Database.Load(null, null); - Annotator = AnnotatorKey.Accept(visitor, Database); + Annotator = AnnotatorKey.Accept(visitor, Database); } } @@ -221,18 +257,30 @@ public EadLipidDatabaseAnnotatorContainer( AnnotatorID = AnnotatorKey.Key; } + [SerializationConstructor] + public EadLipidDatabaseAnnotatorContainer( + IReferRestorationKey annotatorKey, + EadLipidDatabase database, + MsRefSearchParameterBase parameter) { + AnnotatorKey = annotatorKey; + Parameter = parameter; + Database = database; + AnnotatorID = AnnotatorKey.Key; + } + [IgnoreMember] public ISerializableAnnotator<(IAnnotationQuery, MoleculeMsReference), MoleculeMsReference, MsScanMatchResult, EadLipidDatabase> Annotator { get; private set; } [IgnoreMember] public string AnnotatorID { get; } - [Key(nameof(AnnotatorKey))] - public IReferRestorationKey<(IAnnotationQuery, MoleculeMsReference), MoleculeMsReference, MsScanMatchResult, EadLipidDatabase> AnnotatorKey { get; set; } + [Key("AnnotatorKey")] + //public IReferRestorationKey<(IAnnotationQuery, MoleculeMsReference), MoleculeMsReference, MsScanMatchResult, EadLipidDatabase> AnnotatorKey { get; set; } + public IReferRestorationKey AnnotatorKey { get; set; } - [Key(nameof(Parameter))] + [Key("Parameter")] public MsRefSearchParameterBase Parameter { get; set; } - [Key(nameof(Database))] + [Key("Database")] public EadLipidDatabase Database { get; set; } [IgnoreMember] @@ -247,7 +295,7 @@ public void Save(Stream stream) { public void Load(Stream stream, ILoadAnnotatorVisitor visitor) { Database.Load(stream, null); - Annotator = AnnotatorKey.Accept(visitor, Database); + Annotator = AnnotatorKey.Accept<(IAnnotationQuery, MoleculeMsReference), MoleculeMsReference, MsScanMatchResult, EadLipidDatabase>(visitor, Database); } } } diff --git a/src/MSDIAL5/MsdialCore/Algorithm/Annotation/IAnnotationQueryFactory.cs b/src/MSDIAL5/MsdialCore/Algorithm/Annotation/IAnnotationQueryFactory.cs index a13bdfd35..26e976d9c 100644 --- a/src/MSDIAL5/MsdialCore/Algorithm/Annotation/IAnnotationQueryFactory.cs +++ b/src/MSDIAL5/MsdialCore/Algorithm/Annotation/IAnnotationQueryFactory.cs @@ -1,5 +1,4 @@ -using Accord.Collections; -using CompMs.Common.Components; +using CompMs.Common.Components; using CompMs.Common.DataObj; using CompMs.Common.DataObj.Property; using CompMs.Common.DataObj.Result; diff --git a/src/MSDIAL5/MsdialCore/DataObj/DataBaseItem.cs b/src/MSDIAL5/MsdialCore/DataObj/DataBaseItem.cs index 0e5c45c08..3991f7538 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/DataBaseItem.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/DataBaseItem.cs @@ -17,17 +17,30 @@ public DataBaseItem( List> pairs) { DataBase = dataBase; Pairs = pairs; + PairsForSerialization = pairs; + } + + [SerializationConstructor] + public DataBaseItem( + TDataBase dataBase, + IReadOnlyList pairsForSerialization) { + DataBase = dataBase; + Pairs = pairsForSerialization.Cast>().ToList(); + PairsForSerialization = Pairs; } [IgnoreMember] public string DataBaseID => DataBase.Id; - [Key(nameof(DataBase))] + [Key("DataBase")] public TDataBase DataBase { get; } - [Key(nameof(Pairs))] + [IgnoreMember] public List> Pairs { get; } + [Key("Pairs")] + public IReadOnlyList PairsForSerialization { get; } + private static readonly string DataBasePath = "DataBase"; private static readonly string AnnotatorsPath = "Annotators"; diff --git a/src/MSDIAL5/MsdialCore/DataObj/EadLipidAnnotatorParameterPair.cs b/src/MSDIAL5/MsdialCore/DataObj/EadLipidAnnotatorParameterPair.cs index ab4dbcecd..ab9ad7624 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/EadLipidAnnotatorParameterPair.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/EadLipidAnnotatorParameterPair.cs @@ -14,13 +14,18 @@ public EadLipidAnnotatorParameterPair(IAnnotationQueryFactoryGenerationKey serializableAnnotatorKey) { SerializableAnnotatorKey = serializableAnnotatorKey ?? throw new System.ArgumentNullException(nameof(serializableAnnotatorKey)); } - [Key(nameof(SerializableAnnotatorKey))] - public IAnnotationQueryFactoryGenerationKey SerializableAnnotatorKey { get; } + [SerializationConstructor] + public EadLipidAnnotatorParameterPair(IReferRestorationKey serializableAnnotatorKey) { + SerializableAnnotatorKey = serializableAnnotatorKey ?? throw new System.ArgumentNullException(nameof(serializableAnnotatorKey)); + } + + [Key("SerializableAnnotatorKey")] + //public IAnnotationQueryFactoryGenerationKey SerializableAnnotatorKey { get; } + public IReferRestorationKey SerializableAnnotatorKey { get; } [IgnoreMember] public IAnnotationQueryFactory AnnotationQueryFactory { get; private set; } diff --git a/src/MSDIAL5/MsdialCore/DataObj/IAnnotatorParameterPair.cs b/src/MSDIAL5/MsdialCore/DataObj/IAnnotatorParameterPair.cs index be5e0caf6..cd7735a75 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/IAnnotatorParameterPair.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/IAnnotatorParameterPair.cs @@ -9,7 +9,12 @@ namespace CompMs.MsdialCore.DataObj [Union(0, typeof(MetabolomicsAnnotatorParameterPair))] [Union(1, typeof(ProteomicsAnnotatorParameterPair))] [Union(2, typeof(EadLipidAnnotatorParameterPair))] - public interface IAnnotatorParameterPair where TDataBase : IReferenceDataBase + public interface IAnnotatorParameterPair + { + + } + + public interface IAnnotatorParameterPair : IAnnotatorParameterPair where TDataBase : IReferenceDataBase { string AnnotatorID { get; } IAnnotationQueryFactory AnnotationQueryFactory { get; } diff --git a/src/MSDIAL5/MsdialCore/DataObj/MetabolomicsAnnotatorParameterPair.cs b/src/MSDIAL5/MsdialCore/DataObj/MetabolomicsAnnotatorParameterPair.cs index d95247ca0..c61400961 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/MetabolomicsAnnotatorParameterPair.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/MetabolomicsAnnotatorParameterPair.cs @@ -15,12 +15,13 @@ public MetabolomicsAnnotatorParameterPair(IAnnotationQueryFactoryGenerationKey serializableAnnotatorKey) { + public MetabolomicsAnnotatorParameterPair(IReferRestorationKey serializableAnnotatorKey) { SerializableAnnotatorKey = serializableAnnotatorKey ?? throw new System.ArgumentNullException(nameof(serializableAnnotatorKey)); } - [Key(nameof(SerializableAnnotatorKey))] - public IAnnotationQueryFactoryGenerationKey SerializableAnnotatorKey { get; } + [Key("SerializableAnnotatorKey")] + //public IAnnotationQueryFactoryGenerationKey SerializableAnnotatorKey { get; } + public IReferRestorationKey SerializableAnnotatorKey { get; } [IgnoreMember] public string AnnotatorID => AnnotationQueryFactory?.AnnotatorId ?? SerializableAnnotatorKey.Key; diff --git a/src/MSDIAL5/MsdialCore/DataObj/MsdialDataStorage.cs b/src/MSDIAL5/MsdialCore/DataObj/MsdialDataStorage.cs index 37c6bb657..740575ba0 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/MsdialDataStorage.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/MsdialDataStorage.cs @@ -33,7 +33,6 @@ public interface IMsdialDataStorage where T : ParameterBase { AnnotationQueryFactoryStorage CreateAnnotationQueryFactoryStorage(); } - [MessagePackObject] public abstract class MsdialDataStorageBase { [Key(0)] diff --git a/src/MSDIAL5/MsdialCore/DataObj/ProteomicsAnnotatorParameterPair.cs b/src/MSDIAL5/MsdialCore/DataObj/ProteomicsAnnotatorParameterPair.cs index 0f191dbeb..38a343f51 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/ProteomicsAnnotatorParameterPair.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/ProteomicsAnnotatorParameterPair.cs @@ -17,15 +17,16 @@ public ProteomicsAnnotatorParameterPair(IAnnotationQueryFactoryGenerationKey serializableAnnotatorKey, ProteomicsParameter proteomicsParameter) { + public ProteomicsAnnotatorParameterPair(IReferRestorationKey serializableAnnotatorKey, ProteomicsParameter proteomicsParameter) { SerializableAnnotatorKey = serializableAnnotatorKey ?? throw new System.ArgumentNullException(nameof(serializableAnnotatorKey)); ProteomicsParameter = proteomicsParameter ?? throw new System.ArgumentNullException(nameof(proteomicsParameter)); } - [Key(nameof(SerializableAnnotatorKey))] - public IAnnotationQueryFactoryGenerationKey SerializableAnnotatorKey { get; } + [Key("SerializableAnnotatorKey")] + //public IAnnotationQueryFactoryGenerationKey SerializableAnnotatorKey { get; } + public IReferRestorationKey SerializableAnnotatorKey { get; } - [Key(nameof(ProteomicsParameter))] + [Key("ProteomicsParameter")] public ProteomicsParameter ProteomicsParameter { get; } [IgnoreMember] diff --git a/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs b/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs index 8d755b867..327ccc10f 100644 --- a/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs +++ b/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs @@ -3,28 +3,26 @@ namespace CompMs.MsdialCore.Parser { - [MessagePack.Union(0, typeof(DataBaseRestorationKey))] [MessagePack.Union(1, typeof(MspDbRestorationKey))] [MessagePack.Union(2, typeof(TextDbRestorationKey))] [MessagePack.Union(3, typeof(StandardRestorationKey))] [MessagePack.Union(4, typeof(ShotgunProteomicsRestorationKey))] [MessagePack.Union(5, typeof(EadLipidDatabaseRestorationKey))] + public interface IReferRestorationKey + { + ISerializableAnnotator Accept(ILoadAnnotatorVisitor visitor, TDatabase database); + IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annoatorVIsitor, TDatabase database); + + int Priority { get; } + string Key { get; } + } + public interface IReferRestorationKey : IAnnotationQueryFactoryGenerationKey { ISerializableAnnotator Accept(ILoadAnnotatorVisitor visitor, TDatabase database); } - [MessagePack.Union(0, typeof(DataBaseRestorationKey))] - [MessagePack.Union(1, typeof(MspDbRestorationKey))] - [MessagePack.Union(2, typeof(TextDbRestorationKey))] - [MessagePack.Union(3, typeof(StandardRestorationKey))] - [MessagePack.Union(4, typeof(ShotgunProteomicsRestorationKey))] - [MessagePack.Union(5, typeof(EadLipidDatabaseRestorationKey))] - public interface IAnnotationQueryFactoryGenerationKey { + public interface IAnnotationQueryFactoryGenerationKey : IReferRestorationKey { IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annoatorVIsitor, TDatabase database); - - string Key { get; } - - int Priority { get; } } } diff --git a/src/MSDIAL5/MsdialCore/Parser/ReferRestorationKey.cs b/src/MSDIAL5/MsdialCore/Parser/ReferRestorationKey.cs index bb79181ef..2cb89b3d5 100644 --- a/src/MSDIAL5/MsdialCore/Parser/ReferRestorationKey.cs +++ b/src/MSDIAL5/MsdialCore/Parser/ReferRestorationKey.cs @@ -1,4 +1,5 @@ -using CompMs.Common.Components; +using System; +using CompMs.Common.Components; using CompMs.Common.DataObj.Result; using CompMs.Common.Parameter; using CompMs.Common.Proteomics.DataObj; @@ -6,8 +7,14 @@ using CompMs.MsdialCore.DataObj; using CompMs.MsdialCore.Parameter; +[assembly: MessagePack.MessagePackAssumedFormattable(typeof(IAnnotationQuery))] +[assembly: MessagePack.MessagePackAssumedFormattable(typeof((IAnnotationQuery, MoleculeMsReference)))] + namespace CompMs.MsdialCore.Parser { + [MessagePack.Union(1, typeof(MspDbRestorationKey))] + [MessagePack.Union(2, typeof(TextDbRestorationKey))] + [MessagePack.Union(3, typeof(StandardRestorationKey))] public abstract class DataBaseRestorationKey : IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> { public DataBaseRestorationKey(string key, int priority) { @@ -23,6 +30,22 @@ public DataBaseRestorationKey(string key, int priority) { public abstract ISerializableAnnotator, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> Accept(ILoadAnnotatorVisitor visitor, MoleculeDataBase database); public abstract IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annotatorVisitor, MoleculeDataBase database); + + public ISerializableAnnotator Accept(ILoadAnnotatorVisitor visitor, TDatabase database) + { + if (typeof(TQuery) == typeof(IAnnotationQuery) && typeof(TReference) == typeof(MoleculeMsReference) && typeof(TResult) == typeof(MsScanMatchResult) && typeof(TDatabase) == typeof(MoleculeDataBase)) { + return (ISerializableAnnotator)Accept(visitor, database as MoleculeDataBase); + } + throw new NotSupportedException($"Unsupported type: {typeof(TQuery)}, {typeof(TReference)}, {typeof(TResult)}, {typeof(TDatabase)}"); + } + + public IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annotatorVisitor, TDatabase database) + { + if (typeof(TDatabase) == typeof(MoleculeDataBase)) { + return Accept(factoryVisitor, annotatorVisitor, database as MoleculeDataBase); + } + throw new NotSupportedException($"Unsupported type: {typeof(TDatabase)}"); + } } [MessagePack.MessagePackObject] @@ -80,6 +103,7 @@ public override IAnnotationQueryFactory Accept(IAnnotationQue } } + [MessagePack.Union(4, typeof(ShotgunProteomicsRestorationKey))] public abstract class FastaDbRestorationKey : IReferRestorationKey { public FastaDbRestorationKey(string key, int priority) { Key = key; @@ -94,6 +118,22 @@ public FastaDbRestorationKey(string key, int priority) { public abstract ISerializableAnnotator Accept(ILoadAnnotatorVisitor visitor, ShotgunProteomicsDB database); public abstract IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annotatorVisitor, ShotgunProteomicsDB database); + + public ISerializableAnnotator Accept(ILoadAnnotatorVisitor visitor, TDatabase database) + { + if (typeof(TQuery) == typeof(IPepAnnotationQuery) && typeof(TReference) == typeof(PeptideMsReference) && typeof(TResult) == typeof(MsScanMatchResult) && typeof(TDatabase) == typeof(ShotgunProteomicsDB)) { + return (ISerializableAnnotator)Accept(visitor, database as ShotgunProteomicsDB); + } + throw new NotSupportedException($"Unsupported type: {typeof(TQuery)}, {typeof(TReference)}, {typeof(TResult)}, {typeof(TDatabase)}"); + } + + public IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annotatorVisitor, TDatabase database) + { + if (typeof(TDatabase) == typeof(ShotgunProteomicsDB)) { + return Accept(factoryVisitor, annotatorVisitor, database as ShotgunProteomicsDB); + } + throw new NotSupportedException($"Unsupported type: {typeof(TDatabase)}"); + } } [MessagePack.MessagePackObject] @@ -156,5 +196,21 @@ public EadLipidDatabaseRestorationKey(string key, int priority, MsRefSearchParam public IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annotatorVisitor, EadLipidDatabase database) { return factoryVisitor.Visit(this, annotatorVisitor.Visit(this, database)); } + + public ISerializableAnnotator Accept(ILoadAnnotatorVisitor visitor, TDatabase database) + { + if (typeof(TQuery) == typeof((IAnnotationQuery, MoleculeMsReference)) && typeof(TReference) == typeof(MoleculeMsReference) && typeof(TResult) == typeof(MsScanMatchResult) && typeof(TDatabase) == typeof(EadLipidDatabase)) { + return (ISerializableAnnotator)Accept(visitor, database as EadLipidDatabase); + } + throw new NotSupportedException($"Unsupported type: {typeof(TQuery)}, {typeof(TReference)}, {typeof(TResult)}, {typeof(TDatabase)}"); + } + + public IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annotatorVisitor, TDatabase database) + { + if (typeof(TDatabase) == typeof(EadLipidDatabase)) { + return Accept(factoryVisitor, annotatorVisitor, database as EadLipidDatabase); + } + throw new NotSupportedException($"Unsupported type: {typeof(TDatabase)}"); + } } } diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/MoleculeDataBaseTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/MoleculeDataBaseTests.cs index 890d19b2d..5cbc093e6 100644 --- a/tests/MSDIAL5/MsdialCoreTests/DataObj/MoleculeDataBaseTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/MoleculeDataBaseTests.cs @@ -19,6 +19,7 @@ public void MoleculeDataBaseTest() { using (var stream = new MemoryStream()) { Common.MessagePack.MessagePackDefaultHandler.SaveToStream(db, stream); + stream.Seek(0, SeekOrigin.Begin); actual = Common.MessagePack.MessagePackDefaultHandler.LoadFromStream(stream); } diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/ProjectDataStorageTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/ProjectDataStorageTests.cs index 299868c2e..87070a0f5 100644 --- a/tests/MSDIAL5/MsdialCoreTests/DataObj/ProjectDataStorageTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/ProjectDataStorageTests.cs @@ -16,7 +16,7 @@ public class ProjectDataStorageTests { [TestMethod()] public async Task LoadTest() { - var projectParameter = new ProjectParameter(DateTime.Parse("2021/12/16 23:14:30"), "Folder", "TestProject"); + var projectParameter = new ProjectParameter(DateTime.Parse("2021/12/16 23:14:30+09:00"), "Folder", "TestProject"); var proj = new ProjectDataStorage(projectParameter, new List>()); var storage1 = new MockStorage { Parameter = new ParameterBase { ProjectFileName = "Test1", } }; var storage2 = new MockStorage { Parameter = new ParameterBase { ProjectFileName = "Test2", } }; @@ -53,7 +53,7 @@ public async Task LoadTest() { [TestMethod()] public async Task LoadFaultedTest() { - var projectParameter = new ProjectParameter(DateTime.Parse("2022/03/11 16:23:50"), "Folder", "TestProject"); + var projectParameter = new ProjectParameter(DateTime.Parse("2022/03/11 16:23:50+09:00"), "Folder", "TestProject"); var proj = new ProjectDataStorage(projectParameter, new List>()); var storage1 = new MockStorage { Parameter = new ParameterBase { ProjectFileName = "Test1", } }; var storage2 = new MockStorage { Parameter = new ParameterBase { ProjectFileName = "Test2", } }; @@ -100,7 +100,7 @@ public async Task LoadFaultedTest() { [TestMethod()] public async Task LoadRetryOnNewFolderTest() { - var projectParameter = new ProjectParameter(DateTime.Parse("2022/07/05 16:56:50"), "Folder", "TestProject"); + var projectParameter = new ProjectParameter(DateTime.Parse("2022/07/05 16:56:50+09:00"), "Folder", "TestProject"); var proj = new ProjectDataStorage(projectParameter, new List>()); var storage1 = new MockStorage { Parameter = new ParameterBase { ProjectFileName = "Test1", } }; var storage2 = new MockStorage { Parameter = new ParameterBase { ProjectFileName = "Test2", } }; diff --git a/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs b/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs index 1d2d09214..9cac4e79f 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs @@ -1,6 +1,5 @@ using CompMs.Common.Components; using CompMs.Common.DataObj.Result; -using CompMs.Common.Interfaces; using CompMs.Common.Proteomics.DataObj; using CompMs.MsdialCore.Algorithm.Annotation; using CompMs.MsdialCore.DataObj; @@ -15,13 +14,14 @@ public class MspDbRestorationKeyTests { [TestMethod()] public void MspDbRestorationKeyTest() { - IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> key = new MspDbRestorationKey("MspKey", -1); - IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> expected = key; - IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> actual; + IReferRestorationKey key = new MspDbRestorationKey("MspKey", -1); + IReferRestorationKey expected = key; + IReferRestorationKey actual; using (var stream = new MemoryStream()) { - Common.MessagePack.MessagePackDefaultHandler.SaveToStream, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase>>(key, stream); - actual = Common.MessagePack.MessagePackDefaultHandler.LoadFromStream, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase>>(stream); + Common.MessagePack.MessagePackDefaultHandler.SaveToStream(key, stream); + stream.Seek(0, SeekOrigin.Begin); + actual = Common.MessagePack.MessagePackDefaultHandler.LoadFromStream(stream); } Assert.AreEqual(expected.Key, actual.Key); From 98ad6a407f454acac851883cb3f706bc2972733f Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Wed, 8 Jul 2026 18:43:39 +0900 Subject: [PATCH 13/30] Add deserialization test for LargeListMessagePack --- .../MessagePack/LargeListMessagePackTests.cs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs index b914fea0f..e003f224a 100644 --- a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs +++ b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; using System.IO; +using System.Globalization; namespace CompMs.Common.MessagePack.Tests { @@ -64,6 +65,30 @@ public void SerializeAndDeserializeSmallSampleBytesTest() { Assert.AreEqual(datas.Length, actual.Count); } + [TestMethod()] + public void DeserializeSerializedSmallSampleBytesTest() { + var bytes = HexToBytes("C90000000D63D2000000077092000000009090"); + var actual = LargeListMessagePack.Deserialize(new MemoryStream(bytes)); + + Assert.AreEqual(2, actual.Count); + } + + [TestMethod()] + public void DeserializeSerializedFixedSampleBytesTest() { + var bytes = HexToBytes("C90000000D63D2000000077092000000009090"); + var actual = LargeListMessagePack.Deserialize(new MemoryStream(bytes)); + + Assert.AreEqual(2, actual.Count); + } + + private static byte[] HexToBytes(string hex) { + var bytes = new byte[hex.Length / 2]; + for (var i = 0; i < bytes.Length; i++) { + bytes[i] = byte.Parse(hex.Substring(i * 2, 2), NumberStyles.HexNumber); + } + return bytes; + } + [TestMethod()] [DataRow(100, 100)] [DataRow(1000000, 100)] From 8e249e9681647521f7beda0e486f3eb49e0dc3b3 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Thu, 9 Jul 2026 11:53:26 +0900 Subject: [PATCH 14/30] Add LargeListMessagePack compatibility tests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePack/LargeListMessagePackTests.cs | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs index e003f224a..fe3415c39 100644 --- a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs +++ b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs @@ -65,6 +65,28 @@ public void SerializeAndDeserializeSmallSampleBytesTest() { Assert.AreEqual(datas.Length, actual.Count); } + [TestMethod()] + public void DeserializeSerializedLargeSampleBytesTest() { + var bytes = HexToBytes("C90000002463D20000001DF00E9200000000919A0102030405060708090A919A0B0C0D0E0F1011121314"); + var actual = LargeListMessagePack.Deserialize(new MemoryStream(bytes)); + + Assert.AreEqual(2, actual.Count); + CollectionAssert.AreEqual(new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, actual[0].Xs); + CollectionAssert.AreEqual(new long[] { 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 }, actual[1].Xs); + } + + [TestMethod()] + public void DeserializeSerializedFixedSampleBytesTest() { + var bytes = HexToBytes("C90000002663D20000001FF010920000000092C40401020304C4040506070892C404090A0B0CC4040D0E0F10"); + var actual = LargeListMessagePack.Deserialize(new MemoryStream(bytes)); + + Assert.AreEqual(2, actual.Count); + CollectionAssert.AreEqual(new byte[] { 1, 2, 3, 4 }, actual[0].Xs); + CollectionAssert.AreEqual(new byte[] { 5, 6, 7, 8 }, actual[0].Ys); + CollectionAssert.AreEqual(new byte[] { 9, 10, 11, 12 }, actual[1].Xs); + CollectionAssert.AreEqual(new byte[] { 13, 14, 15, 16 }, actual[1].Ys); + } + [TestMethod()] public void DeserializeSerializedSmallSampleBytesTest() { var bytes = HexToBytes("C90000000D63D2000000077092000000009090"); @@ -74,13 +96,30 @@ public void DeserializeSerializedSmallSampleBytesTest() { } [TestMethod()] - public void DeserializeSerializedFixedSampleBytesTest() { + public void DeserializeSerializedFixedSampleLegacyBytesTest() { var bytes = HexToBytes("C90000000D63D2000000077092000000009090"); var actual = LargeListMessagePack.Deserialize(new MemoryStream(bytes)); Assert.AreEqual(2, actual.Count); } + [TestMethod()] + public void DeserializeAtSerializedFixedSampleBytesTest() { + var bytes = HexToBytes("C90000002663D20000001FF010920000000092C40401020304C4040506070892C404090A0B0CC4040D0E0F10"); + var actual = LargeListMessagePack.DeserializeAt(new MemoryStream(bytes), 1); + + CollectionAssert.AreEqual(new byte[] { 9, 10, 11, 12 }, actual.Xs); + CollectionAssert.AreEqual(new byte[] { 13, 14, 15, 16 }, actual.Ys); + } + + [TestMethod()] + public void DeserializeAtSerializedLargeSampleBytesTest() { + var bytes = HexToBytes("C90000002463D20000001DF00E9200000000919A0102030405060708090A919A0B0C0D0E0F1011121314"); + var actual = LargeListMessagePack.DeserializeAt(new MemoryStream(bytes), 0); + + CollectionAssert.AreEqual(new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, actual.Xs); + } + private static byte[] HexToBytes(string hex) { var bytes = new byte[hex.Length / 2]; for (var i = 0; i < bytes.Length; i++) { From 9e61c2c8bd335aa45416133afc1c50097e61a9a4 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Thu, 9 Jul 2026 12:02:34 +0900 Subject: [PATCH 15/30] Restore LargeListMessagePack compatibility Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePack/LargeListMessagePack.cs | 187 ++++++++++++++++-- .../MessagePack/LargeListMessagePackTests.cs | 10 + 2 files changed, 184 insertions(+), 13 deletions(-) diff --git a/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs b/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs index 4d8b56c48..9e8f0af8c 100644 --- a/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs +++ b/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs @@ -1,36 +1,197 @@ using MessagePack; +using MessagePack.Formatters; using MessagePack.Resolvers; using System; using System.Collections.Generic; using System.IO; +using System.Linq; namespace CompMs.Common.MessagePack { public static class LargeListMessagePack { public const sbyte ExtensionTypeCode = 99; - public const int HeaderSize = 11; - private static MessagePackSerializerOptions Options => MessagePackSerializerOptions.Standard.WithResolver(StandardResolver.Instance).WithCompression(MessagePackCompression.Lz4BlockArray); - - public static void Serialize(Stream stream, IReadOnlyList value, IFormatterResolver resolver = null) { + public static void Serialize(Stream stream, IReadOnlyList value) { if (value is null) { - MessagePackSerializer.Serialize(stream, value, Options.WithResolver(resolver ?? StandardResolver.Instance)); + MessagePackSerializer.Serialize(stream, value, MessagePackSerializerOptions.Standard.WithResolver(StandardResolver.Instance).WithCompression(MessagePackCompression.Lz4BlockArray)); return; } - MessagePackSerializer.Serialize(stream, value, Options.WithResolver(resolver ?? StandardResolver.Instance)); + var options = MessagePackSerializerOptions.Standard.WithResolver(StandardResolver.Instance).WithCompression(MessagePackCompression.Lz4BlockArray); + Memory memory = (value as T[]) ?? value.ToArray(); + var length = memory.Length; + var size = (int)Math.Sqrt(length) + 1; + var iteration = (length + size - 1) / size; + for (int i = 0; i < iteration; i++) { + MessagePackSerializer.Serialize(stream, new SerializingDataContainer { Data = memory.Slice(i * size, Math.Min(size, length - i * size)) }, options); + } + } + + public static List Deserialize(Stream stream) { + using (var memory = new MemoryStream()) { + stream.CopyTo(memory); + var reader = new MessagePackReader(new ReadOnlyMemory(memory.GetBuffer(), 0, (int)memory.Length)); + return DeserializeCore(ref reader, MessagePackSerializerOptions.Standard); + } + } + + public static IEnumerable> DeserializeIncremental(Stream stream) { + using (var memory = new MemoryStream()) { + stream.CopyTo(memory); + var reader = new MessagePackReader(new ReadOnlyMemory(memory.GetBuffer(), 0, (int)memory.Length)); + foreach (var result in DeserializeIncrementalCore(ref reader, MessagePackSerializerOptions.Standard)) { + yield return result; + } + } + } + + public static T DeserializeAt(Stream stream, int index) { + using (var memory = new MemoryStream()) { + stream.CopyTo(memory); + var reader = new MessagePackReader(new ReadOnlyMemory(memory.GetBuffer(), 0, (int)memory.Length)); + return DeserializeAt(ref reader, MessagePackSerializerOptions.Standard, index); + } + } + + public static T DeserializeAt(ref MessagePackReader reader, MessagePackSerializerOptions options, int index) { + while (!reader.End) { + var success = TryDeserializeAtOrSkip(ref reader, options, index, out var result, out int skipArraySize); + if (success) { + return result; + } + index -= skipArraySize; + } + + return default; } - public static List Deserialize(Stream stream, IFormatterResolver resolver = null) { - return MessagePackSerializer.Deserialize>(stream, Options.WithResolver(resolver ?? StandardResolver.Instance)); + private static List DeserializeCore(ref MessagePackReader reader, MessagePackSerializerOptions options) { + var res = new List>(); + while (!reader.End) { + var tmp = DeserializeEach(ref reader, options); + if (tmp != null && tmp.Count > 0) { + res.Add(tmp); + } + } + + return res.SelectMany(r => r).ToList(); } - public static IEnumerable> DeserializeIncremental(Stream stream, IFormatterResolver resolver = null) { - yield return Deserialize(stream, resolver); + private static IEnumerable> DeserializeIncrementalCore(ref MessagePackReader reader, MessagePackSerializerOptions options) { + var results = new List>(); + while (!reader.End) { + results.Add(DeserializeEach(ref reader, options)); + } + + return results; } - public static T DeserializeAt(Stream stream, int index, IFormatterResolver resolver = null) { - var list = Deserialize(stream, resolver); - return index >= 0 && index < list.Count ? list[index] : default; + private static List DeserializeEach(ref MessagePackReader reader, MessagePackSerializerOptions options) { + var resolver = CompositeResolver.Create( + [new DeserializedDataContainer.DeserializedDataContainerFormatter(),], + [StandardResolver.Instance]); + if (!reader.End) { + return MessagePackSerializer.Deserialize>(ref reader, options.WithResolver(resolver).WithCompression(MessagePackCompression.Lz4BlockArray)).Data; + } + + return []; + } + + private static bool TryDeserializeAtOrSkip(ref MessagePackReader reader, MessagePackSerializerOptions options, int index, out T result, out int skipArraySize) { + var resolver = CompositeResolver.Create( + [DeserializedDataContainer.CreateFormatter(index),], + [StandardResolver.Instance]); + var newOptions = options.WithCompression(MessagePackCompression.Lz4BlockArray).WithResolver(resolver); + var deserialized = MessagePackSerializer.Deserialize>(ref reader, newOptions); + result = deserialized.Data is null ? default : deserialized.Data.FirstOrDefault(); + skipArraySize = deserialized.Length; + return result != null; + } + + internal sealed class SerializingDataContainer { + public sbyte VersionCode { get; } = 2; + public Memory Data { get; set; } + + internal class DataContainerFormatter : IMessagePackFormatter?> { + public SerializingDataContainer Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + throw new NotSupportedException(); + } + + public void Serialize(ref MessagePackWriter writer, SerializingDataContainer value, MessagePackSerializerOptions options) { + writer.WriteInt8(value.VersionCode); + writer.WriteArrayHeader(value.Data.Length); + for (int i = 0; i < value.Data.Length; i++) { + MessagePackSerializer.Serialize(ref writer, value.Data.Span[i], options); + } + } + } + } + + private sealed class DeserializedDataContainer { + public List Data { get; set; } + public int Length { get; set; } + + public static IMessagePackFormatter> CreateFormatter(int index) => new SpecificDataFormatter(index); + +#pragma warning disable MsgPack009 + internal class DeserializedDataContainerFormatter : IMessagePackFormatter?> { + public DeserializedDataContainer Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + int length; + if (reader.NextCode == MessagePackCode.Int8) { + _ = reader.ReadSByte(); + length = reader.ReadArrayHeader(); + } + else { + var reader_ = reader.CreatePeekReader(); + length = reader_.ReadArrayHeader(); + reader.ReadRaw(5); + } + var data = new List(length); + for (int i = 0; i < length; i++) { + data.Add(MessagePackSerializer.Deserialize(ref reader, options)); + } + return new DeserializedDataContainer { Length = length, Data = data }; + } + + public void Serialize(ref MessagePackWriter writer, DeserializedDataContainer value, MessagePackSerializerOptions options) { + throw new NotSupportedException(); + } + } + + internal class SpecificDataFormatter : IMessagePackFormatter?> { + private readonly int _index; + + public SpecificDataFormatter(int index) { + _index = index; + } + + public DeserializedDataContainer Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + int length; + if (reader.NextCode == MessagePackCode.Int8) { + _ = reader.ReadSByte(); + length = reader.ReadArrayHeader(); + } + else { + var reader_ = reader.CreatePeekReader(); + length = reader_.ReadArrayHeader(); + reader.ReadRaw(5); + } + if (length <= _index) { + for (int i = 0; i < length; i++) { + reader.Skip(); + } + return new DeserializedDataContainer { Length = length, }; + } + for (int i = 0; i < _index; i++) { + reader.Skip(); + } + return new DeserializedDataContainer { Data = new List { MessagePackSerializer.Deserialize(ref reader, options) }, Length = length }; + } + + public void Serialize(ref MessagePackWriter writer, DeserializedDataContainer value, MessagePackSerializerOptions options) { + throw new NotSupportedException(); + } + } +#pragma warning restore MsgPack009 } } } diff --git a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs index e003f224a..186c6d4c4 100644 --- a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs +++ b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs @@ -73,6 +73,16 @@ public void DeserializeSerializedSmallSampleBytesTest() { Assert.AreEqual(2, actual.Count); } + [TestMethod()] + public void DeserializeSerializedSmallSampleBytesRoundTripTest() { + var bytes = HexToBytes("C90000000D63D2000000077092000000009090"); + var actual = LargeListMessagePack.Deserialize(new MemoryStream(bytes)); + var roundTrip = new MemoryStream(); + LargeListMessagePack.Serialize(roundTrip, actual); + + Assert.AreEqual(actual.Count, LargeListMessagePack.Deserialize(new MemoryStream(roundTrip.ToArray())).Count); + } + [TestMethod()] public void DeserializeSerializedFixedSampleBytesTest() { var bytes = HexToBytes("C90000000D63D2000000077092000000009090"); From 85693d36541272e61ff78296d3dbc197a7cfa03b Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Thu, 9 Jul 2026 14:07:14 +0900 Subject: [PATCH 16/30] Remove duplicate LargeListMessagePack tests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePack/LargeListMessagePackTests.cs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs index 64f73dab4..fe3415c39 100644 --- a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs +++ b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs @@ -120,23 +120,6 @@ public void DeserializeAtSerializedLargeSampleBytesTest() { CollectionAssert.AreEqual(new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, actual.Xs); } - [TestMethod()] - public void DeserializeAtSerializedFixedSampleBytesTest() { - var bytes = HexToBytes("C90000002663D20000001FF010920000000092C40401020304C4040506070892C404090A0B0CC4040D0E0F10"); - var actual = LargeListMessagePack.DeserializeAt(new MemoryStream(bytes), 1); - - CollectionAssert.AreEqual(new byte[] { 9, 10, 11, 12 }, actual.Xs); - CollectionAssert.AreEqual(new byte[] { 13, 14, 15, 16 }, actual.Ys); - } - - [TestMethod()] - public void DeserializeAtSerializedLargeSampleBytesTest() { - var bytes = HexToBytes("C90000002463D20000001DF00E9200000000919A0102030405060708090A919A0B0C0D0E0F1011121314"); - var actual = LargeListMessagePack.DeserializeAt(new MemoryStream(bytes), 0); - - CollectionAssert.AreEqual(new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, actual.Xs); - } - private static byte[] HexToBytes(string hex) { var bytes = new byte[hex.Length / 2]; for (var i = 0; i < bytes.Length; i++) { From 48025dad131accbdd43de3c1c7d35c7fddc1f209 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Thu, 9 Jul 2026 20:33:17 +0900 Subject: [PATCH 17/30] Add MessagePack compatibility fixtures Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePackCompatibilityFixturesTests.cs | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs new file mode 100644 index 000000000..5e519416c --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs @@ -0,0 +1,152 @@ +using CompMs.Common.Components; +using CompMs.Common.Interfaces; +using CompMs.Common.MessagePack; +using CompMs.Common.DataObj.Property; +using CompMs.MsdialCore.MSDec; +using CompMs.MsdialCore.MSDec.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using MessagePack; +using MessagePack.Formatters; +using MessagePack.Resolvers; +using System; +using System.IO; +using NCDK; + +namespace CompMs.MsdialCore.DataObj.Tests +{ + [TestClass] + public class MessagePackCompatibilityFixturesTests + { + [TestMethod] + public void AnnotatedMsDecResultFixtureMatchesCurrentFormat() { + var sample = AnnotatedMSDecResultTestHelper.CreateSample(); + var bytes = SerializeToBytes(sample); + + var roundTrip = Deserialize(bytes); + + Assert.AreEqual(sample.QuantMass, roundTrip.QuantMass); + } + + [TestMethod] + public void QuantifiedChromatogramPeakFixtureMatchesCurrentFormat() { + var sample = QuantifiedChromatogramPeakTestHelper.CreateSample(); + var bytes = SerializeToBytes(sample); + + var roundTrip = Deserialize(bytes); + + Assert.IsInstanceOfType(roundTrip.PeakFeature, typeof(BaseChromatogramPeakFeature)); + Assert.AreEqual(sample.PeakFeature.ChromScanIdLeft, roundTrip.PeakFeature.ChromScanIdLeft); + Assert.AreEqual(sample.PeakFeature.ChromScanIdTop, roundTrip.PeakFeature.ChromScanIdTop); + Assert.AreEqual(sample.PeakFeature.ChromScanIdRight, roundTrip.PeakFeature.ChromScanIdRight); + Assert.AreEqual(sample.PeakFeature.ChromXsLeft.Value, roundTrip.PeakFeature.ChromXsLeft.Value); + Assert.AreEqual(sample.PeakFeature.ChromXsTop.Value, roundTrip.PeakFeature.ChromXsTop.Value); + Assert.AreEqual(sample.PeakFeature.ChromXsRight.Value, roundTrip.PeakFeature.ChromXsRight.Value); + Assert.AreEqual(sample.PeakFeature.PeakHeightLeft, roundTrip.PeakFeature.PeakHeightLeft); + Assert.AreEqual(sample.PeakFeature.PeakHeightTop, roundTrip.PeakFeature.PeakHeightTop); + Assert.AreEqual(sample.PeakFeature.PeakHeightRight, roundTrip.PeakFeature.PeakHeightRight); + Assert.AreEqual(sample.PeakFeature.PeakAreaAboveZero, roundTrip.PeakFeature.PeakAreaAboveZero); + Assert.AreEqual(sample.PeakFeature.PeakAreaAboveBaseline, roundTrip.PeakFeature.PeakAreaAboveBaseline); + Assert.AreEqual(sample.PeakFeature.Mass, roundTrip.PeakFeature.Mass); + Assert.IsInstanceOfType(roundTrip.PeakShape, typeof(ChromatogramPeakShape)); + Assert.AreEqual(sample.PeakShape.GetType(), roundTrip.PeakShape.GetType()); + Assert.AreEqual(sample.MS1RawSpectrumIdTop, roundTrip.MS1RawSpectrumIdTop); + Assert.AreEqual(sample.MS1RawSpectrumIdLeft, roundTrip.MS1RawSpectrumIdLeft); + Assert.AreEqual(sample.MS1RawSpectrumIdRight, roundTrip.MS1RawSpectrumIdRight); + } + + [TestMethod] + public void MoleculePropertyInterfaceFixtureMatchesCurrentFormat() { + IMoleculeProperty sample = new MoleculeProperty( + "Name", + new Formula(new System.Collections.Generic.Dictionary { ["H"] = 2, ["O"] = 1 }), + "Ontology", + "C", + "ABCDEFGHIJKLMNOQRSTUVWXYZ1"); + var bytes = SerializeToBytes(sample, MoleculePropertyExtension.Formatter); + + var roundTrip = Deserialize(bytes, MoleculePropertyExtension.Formatter); + + Assert.AreEqual(sample.Name, roundTrip.Name); + Assert.AreEqual(sample.Formula.ToString(), roundTrip.Formula.ToString()); + Assert.AreEqual(sample.Ontology, roundTrip.Ontology); + Assert.AreEqual(sample.SMILES, roundTrip.SMILES); + Assert.AreEqual(sample.InChIKey, roundTrip.InChIKey); + } + + [TestMethod] + public void ChromatogramPeakFeatureInterfaceFixtureMatchesCurrentFormat() { + IChromatogramPeakFeature sample = new BaseChromatogramPeakFeature { + ChromScanIdLeft = 1, + ChromScanIdTop = 2, + ChromScanIdRight = 3, + ChromXsLeft = new ChromXs(4), + ChromXsTop = new ChromXs(5), + ChromXsRight = new ChromXs(6), + PeakHeightLeft = 7, + PeakHeightTop = 8, + PeakHeightRight = 9, + PeakAreaAboveZero = 10, + PeakAreaAboveBaseline = 11, + Mass = 12, + }; + var bytes = SerializeToBytes(sample, new ChromatogramPeakFeatureInterfaceFormatter()); + + var roundTrip = Deserialize(bytes, new ChromatogramPeakFeatureInterfaceFormatter()); + + Assert.AreEqual(sample.ChromScanIdLeft, roundTrip.ChromScanIdLeft); + Assert.AreEqual(sample.ChromScanIdTop, roundTrip.ChromScanIdTop); + Assert.AreEqual(sample.ChromScanIdRight, roundTrip.ChromScanIdRight); + Assert.AreEqual(sample.ChromXsLeft.Value, roundTrip.ChromXsLeft.Value); + Assert.AreEqual(sample.ChromXsTop.Value, roundTrip.ChromXsTop.Value); + Assert.AreEqual(sample.ChromXsRight.Value, roundTrip.ChromXsRight.Value); + Assert.AreEqual(sample.PeakHeightLeft, roundTrip.PeakHeightLeft); + Assert.AreEqual(sample.PeakHeightTop, roundTrip.PeakHeightTop); + Assert.AreEqual(sample.PeakHeightRight, roundTrip.PeakHeightRight); + Assert.AreEqual(sample.PeakAreaAboveZero, roundTrip.PeakAreaAboveZero); + Assert.AreEqual(sample.PeakAreaAboveBaseline, roundTrip.PeakAreaAboveBaseline); + Assert.AreEqual(sample.Mass, roundTrip.Mass); + } + + [TestMethod] + public void LargeListMessagePackRoundTripsFixtureList() { + var samples = new[] { AnnotatedMSDecResultTestHelper.CreateSample(), AnnotatedMSDecResultTestHelper.CreateSample() }; + using var stream = new MemoryStream(); + LargeListMessagePack.Serialize(stream, samples); + stream.Position = 0; + + var actual = LargeListMessagePack.Deserialize(stream); + + Assert.AreEqual(samples.Length, actual.Count); + Assert.AreEqual(samples[0].QuantMass, actual[0].QuantMass); + Assert.AreEqual(samples[1].QuantMass, actual[1].QuantMass); + } + + private static byte[] SerializeToBytes(T value, IMessagePackFormatter formatter = null) { + var bytes = new byte[256]; + var resolver = formatter is null ? StandardResolver.Instance : new CompositeResolver(formatter); + var offset = resolver.GetFormatterWithVerify().Serialize(ref bytes, 0, value, resolver); + Array.Resize(ref bytes, offset); + return bytes; + } + + private static T Deserialize(byte[] bytes, IMessagePackFormatter formatter = null) { + var resolver = formatter is null ? StandardResolver.Instance : new CompositeResolver(formatter); + return resolver.GetFormatterWithVerify().Deserialize(bytes, 0, resolver, out _); + } + + private sealed class CompositeResolver : IFormatterResolver { + private readonly IMessagePackFormatter formatter; + + public CompositeResolver(IMessagePackFormatter formatter) { + this.formatter = formatter; + } + + public IMessagePackFormatter GetFormatter() { + if (formatter is IMessagePackFormatter typed) { + return typed; + } + return StandardResolver.Instance.GetFormatter(); + } + } + } +} From b07f4286aaa8d1a7f810968cf5bf5faa38fe5a3b Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Fri, 10 Jul 2026 13:52:40 +0900 Subject: [PATCH 18/30] Add legacy MessagePack load coverage Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePack/LargeListMessagePackTests.cs | 13 +++ .../DataObj/DataBaseItemLegacyLoadTests.cs | 82 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 tests/MSDIAL5/MsdialCoreTests/DataObj/DataBaseItemLegacyLoadTests.cs diff --git a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs index 40bcb0d2a..aaad8b97b 100644 --- a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs +++ b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs @@ -78,6 +78,19 @@ public void SaveAndLoadLargeAndLessSampleTest() { Assert.AreEqual(datas.Length, actual.Count); } + [TestMethod()] + public void DeserializeSerializedLargeSampleBytesTest() { + byte[] bytes = { + 0x91, 0x91, 0x9A, + 0x00, 0x01, 0x02, 0x03, 0x04, + 0x05, 0x06, 0x07, 0x08, 0x09 + }; + using var memory = new MemoryStream(bytes); + var actual = LargeListMessagePack.Deserialize(memory); + Assert.AreEqual(1, actual.Count); + CollectionAssert.AreEqual(new long[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, actual[0].Xs); + } + [DataTestMethod()] [DataRow(1, 0)] [DataRow(100, 64)] diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/DataBaseItemLegacyLoadTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/DataBaseItemLegacyLoadTests.cs new file mode 100644 index 000000000..cf3b4c025 --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/DataBaseItemLegacyLoadTests.cs @@ -0,0 +1,82 @@ +using CompMs.Common.Components; +using CompMs.Common.DataObj.Result; +using CompMs.Common.Interfaces; +using CompMs.Common.Proteomics.DataObj; +using CompMs.MsdialCore.Algorithm.Annotation; +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Parser; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; + +namespace CompMs.MsdialCore.DataObj.Tests +{ + [TestClass] + public class DataBaseItemLegacyLoadTests + { + [TestMethod] + public void LoadsLegacyNestedDatabaseArchiveLayout() { + var dataBase = new MockReferenceDataBase("mock-db"); + var item = new DataBaseItem(dataBase, new List>()); + + using var stream = new MemoryStream(); + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) { + var legacyArchiveEntry = archive.CreateEntry("root/mock-db"); + using var legacyStream = legacyArchiveEntry.Open(); + using (var nested = new ZipArchive(legacyStream, ZipArchiveMode.Create, leaveOpen: true)) { + var dataBaseEntry = nested.CreateEntry("DataBase"); + using (var dbStream = dataBaseEntry.Open()) { + dataBase.Save(dbStream, false); + } + } + } + + stream.Position = 0; + using var readArchive = new ZipArchive(stream, ZipArchiveMode.Read); + + item.Load(readArchive, "root", new NoopLoadAnnotatorVisitor(), new NoopAnnotationQueryFactoryGenerationVisitor(), "project"); + + Assert.AreEqual(dataBase.Id, item.DataBase.Id); + } + + private sealed class MockReferenceDataBase : IReferenceDataBase + { + public MockReferenceDataBase(string id) { + Id = id; + } + + public string Id { get; } + + public void Save(Stream stream, bool forceSerialize = false) { + using var writer = new StreamWriter(stream, System.Text.Encoding.UTF8, 1024, leaveOpen: true); + writer.Write(Id); + } + + public void Load(Stream stream, string folderpath) { + using var reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, 1024, leaveOpen: true); + var actual = reader.ReadToEnd(); + Assert.AreEqual(Id, actual); + } + } + + private sealed class NoopLoadAnnotatorVisitor : ILoadAnnotatorVisitor + { + public ISerializableAnnotator, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> Visit(StandardRestorationKey key, MoleculeDataBase database) => throw new NotImplementedException(); + public ISerializableAnnotator, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> Visit(MspDbRestorationKey key, MoleculeDataBase database) => throw new NotImplementedException(); + public ISerializableAnnotator, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> Visit(TextDbRestorationKey key, MoleculeDataBase database) => throw new NotImplementedException(); + public ISerializableAnnotator Visit(ShotgunProteomicsRestorationKey key, ShotgunProteomicsDB database) => throw new NotImplementedException(); + public ISerializableAnnotator<(IAnnotationQuery, MoleculeMsReference), MoleculeMsReference, MsScanMatchResult, EadLipidDatabase> Visit(EadLipidDatabaseRestorationKey key, EadLipidDatabase database) => throw new NotImplementedException(); + } + + private sealed class NoopAnnotationQueryFactoryGenerationVisitor : IAnnotationQueryFactoryGenerationVisitor + { + public IAnnotationQueryFactory Visit(StandardRestorationKey key, IMatchResultFinder finder) => throw new NotImplementedException(); + public IAnnotationQueryFactory Visit(MspDbRestorationKey key, IMatchResultFinder finder) => throw new NotImplementedException(); + public IAnnotationQueryFactory Visit(TextDbRestorationKey key, IMatchResultFinder finder) => throw new NotImplementedException(); + public IAnnotationQueryFactory Visit(ShotgunProteomicsRestorationKey key, IMatchResultFinder finder) => throw new NotImplementedException(); + public IAnnotationQueryFactory Visit(EadLipidDatabaseRestorationKey key, IMatchResultFinder<(IAnnotationQuery, MoleculeMsReference), MsScanMatchResult> finder) => throw new NotImplementedException(); + } + } +} From f20b8932dfe7fdeb4b98369c997ed0d05b47943d Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Fri, 10 Jul 2026 15:07:26 +0900 Subject: [PATCH 19/30] Add MessagePack legacy byte fixtures Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePackCompatibilityFixturesTests.cs | 62 +++++++------------ 1 file changed, 21 insertions(+), 41 deletions(-) diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs index 5e519416c..f13c55379 100644 --- a/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs @@ -55,56 +55,36 @@ public void QuantifiedChromatogramPeakFixtureMatchesCurrentFormat() { } [TestMethod] - public void MoleculePropertyInterfaceFixtureMatchesCurrentFormat() { - IMoleculeProperty sample = new MoleculeProperty( - "Name", - new Formula(new System.Collections.Generic.Dictionary { ["H"] = 2, ["O"] = 1 }), - "Ontology", - "C", - "ABCDEFGHIJKLMNOQRSTUVWXYZ1"); - var bytes = SerializeToBytes(sample, MoleculePropertyExtension.Formatter); + public void MoleculePropertyInterfaceDeserializesLegacyBytes() { + var bytes = new byte[] { 0x95, 0xA4, 0x4E, 0x61, 0x6D, 0x65, 0xDC, 0x00, 0x1B, 0xA3, 0x48, 0x32, 0x4F, 0xCB, 0x40, 0x32, 0x02, 0xB4, 0x5D, 0xFA, 0xFB, 0xED, 0xCB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3, 0x00, 0x82, 0xA1, 0x48, 0x02, 0xA1, 0x4F, 0x01, 0xA8, 0x4F, 0x6E, 0x74, 0x6F, 0x6C, 0x6F, 0x67, 0x79, 0xA1, 0x43, 0xBA, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x31 }; var roundTrip = Deserialize(bytes, MoleculePropertyExtension.Formatter); - Assert.AreEqual(sample.Name, roundTrip.Name); - Assert.AreEqual(sample.Formula.ToString(), roundTrip.Formula.ToString()); - Assert.AreEqual(sample.Ontology, roundTrip.Ontology); - Assert.AreEqual(sample.SMILES, roundTrip.SMILES); - Assert.AreEqual(sample.InChIKey, roundTrip.InChIKey); + Assert.AreEqual("Name", roundTrip.Name); + Assert.AreEqual("H2O", roundTrip.Formula.ToString()); + Assert.AreEqual("Ontology", roundTrip.Ontology); + Assert.AreEqual("C", roundTrip.SMILES); + Assert.AreEqual("ABCDEFGHIJKLMNOQRSTUVWXYZ1", roundTrip.InChIKey); } [TestMethod] - public void ChromatogramPeakFeatureInterfaceFixtureMatchesCurrentFormat() { - IChromatogramPeakFeature sample = new BaseChromatogramPeakFeature { - ChromScanIdLeft = 1, - ChromScanIdTop = 2, - ChromScanIdRight = 3, - ChromXsLeft = new ChromXs(4), - ChromXsTop = new ChromXs(5), - ChromXsRight = new ChromXs(6), - PeakHeightLeft = 7, - PeakHeightTop = 8, - PeakHeightRight = 9, - PeakAreaAboveZero = 10, - PeakAreaAboveBaseline = 11, - Mass = 12, - }; - var bytes = SerializeToBytes(sample, new ChromatogramPeakFeatureInterfaceFormatter()); + public void ChromatogramPeakFeatureInterfaceDeserializesLegacyBytes() { + var bytes = new byte[] { 0x9C, 0x01, 0x02, 0x03, 0x98, 0x92, 0x01, 0x93, 0xCB, 0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x02, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x92, 0x04, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x92, 0x03, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0xCB, 0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x92, 0x01, 0x93, 0xCB, 0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x02, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x92, 0x04, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x92, 0x03, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0xCB, 0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x92, 0x01, 0x93, 0xCB, 0x40, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x02, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x92, 0x04, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x92, 0x03, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0xCB, 0x40, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; var roundTrip = Deserialize(bytes, new ChromatogramPeakFeatureInterfaceFormatter()); - Assert.AreEqual(sample.ChromScanIdLeft, roundTrip.ChromScanIdLeft); - Assert.AreEqual(sample.ChromScanIdTop, roundTrip.ChromScanIdTop); - Assert.AreEqual(sample.ChromScanIdRight, roundTrip.ChromScanIdRight); - Assert.AreEqual(sample.ChromXsLeft.Value, roundTrip.ChromXsLeft.Value); - Assert.AreEqual(sample.ChromXsTop.Value, roundTrip.ChromXsTop.Value); - Assert.AreEqual(sample.ChromXsRight.Value, roundTrip.ChromXsRight.Value); - Assert.AreEqual(sample.PeakHeightLeft, roundTrip.PeakHeightLeft); - Assert.AreEqual(sample.PeakHeightTop, roundTrip.PeakHeightTop); - Assert.AreEqual(sample.PeakHeightRight, roundTrip.PeakHeightRight); - Assert.AreEqual(sample.PeakAreaAboveZero, roundTrip.PeakAreaAboveZero); - Assert.AreEqual(sample.PeakAreaAboveBaseline, roundTrip.PeakAreaAboveBaseline); - Assert.AreEqual(sample.Mass, roundTrip.Mass); + Assert.AreEqual(1, roundTrip.ChromScanIdLeft); + Assert.AreEqual(2, roundTrip.ChromScanIdTop); + Assert.AreEqual(3, roundTrip.ChromScanIdRight); + Assert.AreEqual(4, roundTrip.ChromXsLeft.Value); + Assert.AreEqual(5, roundTrip.ChromXsTop.Value); + Assert.AreEqual(6, roundTrip.ChromXsRight.Value); + Assert.AreEqual(7, roundTrip.PeakHeightLeft); + Assert.AreEqual(8, roundTrip.PeakHeightTop); + Assert.AreEqual(9, roundTrip.PeakHeightRight); + Assert.AreEqual(10, roundTrip.PeakAreaAboveZero); + Assert.AreEqual(11, roundTrip.PeakAreaAboveBaseline); + Assert.AreEqual(12, roundTrip.Mass); } [TestMethod] From f8ef053fdbf4b5142fcfee1680c30614cdb735b8 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Fri, 10 Jul 2026 15:11:45 +0900 Subject: [PATCH 20/30] Add MessagePack compatibility fixtures Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePack/LargeListMessagePackTests.cs | 10 +++++ .../MessagePackCompatibilityFixturesTests.cs | 45 ++++++++----------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs index fe3415c39..39140e84b 100644 --- a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs +++ b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs @@ -95,6 +95,16 @@ public void DeserializeSerializedSmallSampleBytesTest() { Assert.AreEqual(2, actual.Count); } + [TestMethod()] + public void DeserializeSerializedSmallSampleBytesRoundTripTest() { + var bytes = HexToBytes("C90000000D63D2000000077092000000009090"); + var actual = LargeListMessagePack.Deserialize(new MemoryStream(bytes)); + var roundTrip = new MemoryStream(); + LargeListMessagePack.Serialize(roundTrip, actual); + + Assert.AreEqual(actual.Count, LargeListMessagePack.Deserialize(new MemoryStream(roundTrip.ToArray())).Count); + } + [TestMethod()] public void DeserializeSerializedFixedSampleLegacyBytesTest() { var bytes = HexToBytes("C90000000D63D2000000077092000000009090"); diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs index 5e519416c..dde5d6067 100644 --- a/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs @@ -1,16 +1,13 @@ using CompMs.Common.Components; +using CompMs.Common.DataObj.Property; using CompMs.Common.Interfaces; using CompMs.Common.MessagePack; -using CompMs.Common.DataObj.Property; -using CompMs.MsdialCore.MSDec; -using CompMs.MsdialCore.MSDec.Tests; -using Microsoft.VisualStudio.TestTools.UnitTesting; using MessagePack; using MessagePack.Formatters; using MessagePack.Resolvers; -using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Buffers; using System.IO; -using NCDK; namespace CompMs.MsdialCore.DataObj.Tests { @@ -122,31 +119,27 @@ public void LargeListMessagePackRoundTripsFixtureList() { } private static byte[] SerializeToBytes(T value, IMessagePackFormatter formatter = null) { - var bytes = new byte[256]; - var resolver = formatter is null ? StandardResolver.Instance : new CompositeResolver(formatter); - var offset = resolver.GetFormatterWithVerify().Serialize(ref bytes, 0, value, resolver); - Array.Resize(ref bytes, offset); + var option = MessagePackSerializerOptions.Standard; + if (formatter is not null) { + var resolver = CompositeResolver.Create([formatter], [StandardResolver.Instance]); + option = option.WithResolver(resolver); + } + var arraywriter = new ArrayBufferWriter(); + var writer = new MessagePackWriter(arraywriter); + MessagePackSerializer.Serialize(ref writer, value, option); + writer.Flush(); + var bytes = new byte[arraywriter.WrittenCount]; + arraywriter.WrittenMemory.CopyTo(bytes); return bytes; } private static T Deserialize(byte[] bytes, IMessagePackFormatter formatter = null) { - var resolver = formatter is null ? StandardResolver.Instance : new CompositeResolver(formatter); - return resolver.GetFormatterWithVerify().Deserialize(bytes, 0, resolver, out _); - } - - private sealed class CompositeResolver : IFormatterResolver { - private readonly IMessagePackFormatter formatter; - - public CompositeResolver(IMessagePackFormatter formatter) { - this.formatter = formatter; - } - - public IMessagePackFormatter GetFormatter() { - if (formatter is IMessagePackFormatter typed) { - return typed; - } - return StandardResolver.Instance.GetFormatter(); + var option = MessagePackSerializerOptions.Standard; + if (formatter is not null) { + var resolver = CompositeResolver.Create([formatter], [StandardResolver.Instance]); + option = option.WithResolver(resolver); } + return MessagePackSerializer.Deserialize(bytes, option); } } } From 95120df74f1057daa31ec762ab74355230e0aed8 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Fri, 10 Jul 2026 17:32:00 +0900 Subject: [PATCH 21/30] Fix test --- .../MessagePack/LargeListMessagePackTests.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs index 347abebcb..03ab7e72d 100644 --- a/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs +++ b/tests/Common/CommonStandardTests/MessagePack/LargeListMessagePackTests.cs @@ -66,7 +66,7 @@ public void SerializeAndDeserializeSmallSampleBytesTest() { } [TestMethod()] - public void DeserializeSerializedLargeSampleBytesTest() { + public void DeserializeSerializedLargeSampleLegacyBytesTest() { var bytes = HexToBytes("C90000002463D20000001DF00E9200000000919A0102030405060708090A919A0B0C0D0E0F1011121314"); var actual = LargeListMessagePack.Deserialize(new MemoryStream(bytes)); @@ -174,7 +174,11 @@ public void SaveAndLoadLargeAndLessSampleTest() { [TestMethod()] public void DeserializeSerializedLargeSampleBytesTest() { byte[] bytes = { - 0x91, 0x91, 0x9A, + 0xC9, 0x00, 0x00, 0x00, + 0x18, 0x63, 0xD2, 0x00, + 0x00, 0x00, 0x11, 0xF0, + 0x02, 0x91, 0x00, 0x00, + 0x00, 0x00, 0x91, 0x9A, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09 }; From 1b8c24884d4722db36eeb165856eb668a2553fe5 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 13 Jul 2026 14:45:29 +0900 Subject: [PATCH 22/30] Add MessagePack compatibility fixtures for legacy bytes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePackCompatibilityFixturesTests.cs | 13 +++- .../Parser/MspDbRestorationKeyTests.cs | 72 ++++++++++++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs index f13c55379..7d717ac04 100644 --- a/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs @@ -27,6 +27,16 @@ public void AnnotatedMsDecResultFixtureMatchesCurrentFormat() { Assert.AreEqual(sample.QuantMass, roundTrip.QuantMass); } + [TestMethod] + public void AnnotatedMsDecResultDeserializesLegacyBytes() { + var bytes = Convert.FromBase64String("yQAAAHBj0gAAAUVPlMSEAAEADiLwvyMADwgABQ8CAC+Ak5HcACfAwMpKAA8FACQk/8IBAFH/wAHA/0YAE8IHADDCwstaAAACAAEYAAEFAMWAkJWkTWV0YdwAG6AfAA4JAA8CAAHwAMIAgKCgoMtAWNMzMzMzMw=="); + var actual = AnnotatedMSDecResult.Load(new MemoryStream(bytes)); + + Assert.AreEqual(99.3, actual.QuantMass); + Assert.IsNotNull(actual.Molecule); + Assert.AreEqual("Meta", actual.Molecule.Name); + } + [TestMethod] public void QuantifiedChromatogramPeakFixtureMatchesCurrentFormat() { var sample = QuantifiedChromatogramPeakTestHelper.CreateSample(); @@ -69,8 +79,7 @@ public void MoleculePropertyInterfaceDeserializesLegacyBytes() { [TestMethod] public void ChromatogramPeakFeatureInterfaceDeserializesLegacyBytes() { - var bytes = new byte[] { 0x9C, 0x01, 0x02, 0x03, 0x98, 0x92, 0x01, 0x93, 0xCB, 0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x02, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x92, 0x04, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x92, 0x03, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0xCB, 0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x92, 0x01, 0x93, 0xCB, 0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x02, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x92, 0x04, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x92, 0x03, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0xCB, 0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x92, 0x01, 0x93, 0xCB, 0x40, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x02, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x92, 0x04, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x92, 0x03, 0x93, 0xCB, 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0xCB, 0x40, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0x40, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; - + var bytes = Convert.FromBase64String("nAECA5iSAZPLQBAAAAAAAAAAAJICk8u/8AAAAAAAAAEEkgSTy7/wAAAAAAAAAgKSA5PLv/AAAAAAAAADAwDLQBAAAAAAAAAAAJiSAZPLQBQAAAAAAAAAAJICk8u/8AAAAAAAAAEEkgSTy7/wAAAAAAAAAgKSA5PLv/AAAAAAAAADAwDLQBQAAAAAAAAAAJiSAZPLQBgAAAAAAAAAAJICk8u/8AAAAAAAAAEEkgSTy7/wAAAAAAAAAgKSA5PLv/AAAAAAAAADAwDLQBgAAAAAAAAAAMtAHAAAAAAAAMtAIAAAAAAAAMtAIgAAAAAAAMtAJAAAAAAAAMtAJgAAAAAAAMtAKAAAAAAAAA=="); var roundTrip = Deserialize(bytes, new ChromatogramPeakFeatureInterfaceFormatter()); Assert.AreEqual(1, roundTrip.ChromScanIdLeft); diff --git a/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs b/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs index 1d2d09214..ec30ed01a 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs @@ -1,6 +1,5 @@ using CompMs.Common.Components; using CompMs.Common.DataObj.Result; -using CompMs.Common.Interfaces; using CompMs.Common.Proteomics.DataObj; using CompMs.MsdialCore.Algorithm.Annotation; using CompMs.MsdialCore.DataObj; @@ -27,6 +26,70 @@ public void MspDbRestorationKeyTest() { Assert.AreEqual(expected.Key, actual.Key); } + [TestMethod()] + public void MspDbRestorationKeySerializedBytesTest() { + IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> key = new MspDbRestorationKey("MspKey", -1); + IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> expected = key; + IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> actual; + + var bytes = Convert.FromBase64String("kgGCo0tleaZNc3BLZXmoUHJpb3JpdHn/"); + using (var stream = new MemoryStream(bytes)) { + actual = Common.MessagePack.MessagePackDefaultHandler.LoadFromStream, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase>>(stream); + } + + Assert.AreEqual(expected.Key, actual.Key); + } + + [TestMethod()] + public void TextDbRestorationKeyTest() { + IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> key = new TextDbRestorationKey("TextKey", 3); + IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> actual; + + using (var stream = new MemoryStream()) { + Common.MessagePack.MessagePackDefaultHandler.SaveToStream, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase>>(key, stream); + actual = Common.MessagePack.MessagePackDefaultHandler.LoadFromStream, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase>>(stream); + } + + Assert.AreEqual(key.Key, actual.Key); + Assert.AreEqual(key.Priority, actual.Priority); + Assert.IsInstanceOfType(actual, typeof(TextDbRestorationKey)); + } + + [TestMethod()] + public void ShotgunProteomicsRestorationKeyTest() { + var parameter = new CompMs.Common.Parameter.MsRefSearchParameterBase(); + var proteomicsParameter = RoundTrip(new CompMs.MsdialCore.Parameter.ProteomicsParameter()); + IReferRestorationKey key = new ShotgunProteomicsRestorationKey("ShotgunKey", 5, parameter, proteomicsParameter, CompMs.Common.DataObj.Result.SourceType.MspDB); + var actual = RoundTrip(key); + + Assert.AreEqual(key.Key, actual.Key); + Assert.AreEqual(key.Priority, actual.Priority); + Assert.AreEqual(CompMs.Common.DataObj.Result.SourceType.MspDB, ((ShotgunProteomicsRestorationKey)actual).SourceType); + Assert.IsNotNull(((ShotgunProteomicsRestorationKey)actual).MsRefSearchParameter); + Assert.IsNotNull(((ShotgunProteomicsRestorationKey)actual).ProteomicsParameter); + Assert.IsInstanceOfType(actual, typeof(ShotgunProteomicsRestorationKey)); + } + + [TestMethod()] + public void ProteomicsParameterRoundTripsDefaultPayload() { + var actual = RoundTrip(new CompMs.MsdialCore.Parameter.ProteomicsParameter()); + + Assert.IsNotNull(actual.VariableModifications); + Assert.IsNotNull(actual.FixedModifications); + Assert.IsTrue(actual.MaxNumberOfModificationsPerPeptide >= 0); + } + + [TestMethod()] + public void EadLipidDatabaseRestorationKeyTest() { + var parameter = new CompMs.Common.Parameter.MsRefSearchParameterBase(); + IReferRestorationKey<(IAnnotationQuery, MoleculeMsReference), MoleculeMsReference, MsScanMatchResult, EadLipidDatabase> key = new EadLipidDatabaseRestorationKey("EadKey", 7, parameter, CompMs.Common.DataObj.Result.SourceType.MspDB); + var actual = RoundTrip(key); + + Assert.AreEqual(key.Key, actual.Key); + Assert.AreEqual(key.Priority, actual.Priority); + Assert.IsInstanceOfType(actual, typeof(EadLipidDatabaseRestorationKey)); + } + [TestMethod()] public void AcceptTest() { var key = new MspDbRestorationKey("MspKey", -1); @@ -34,6 +97,13 @@ public void AcceptTest() { key.Accept(visitor, null); Assert.IsTrue(visitor.Called); } + + private static T RoundTrip(T value) { + using var stream = new MemoryStream(); + Common.MessagePack.MessagePackDefaultHandler.SaveToStream(value, stream); + stream.Position = 0; + return Common.MessagePack.MessagePackDefaultHandler.LoadFromStream(stream); + } } class MockLoadAnnotator : ILoadAnnotatorVisitor From 6f129d9376de8af34f33b5d68bd866cc28f3801e Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 13 Jul 2026 15:03:34 +0900 Subject: [PATCH 23/30] Expand MessagePack compatibility test coverage Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MessagePackCompatibilityFixturesTests.cs | 2 +- .../Parser/MspDbRestorationKeyTests.cs | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs index 1f2600efa..aa6b0a021 100644 --- a/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/MessagePackCompatibilityFixturesTests.cs @@ -1,11 +1,11 @@ using CompMs.Common.Components; -using CompMs.Common.DataObj.Property; using CompMs.Common.Interfaces; using CompMs.Common.MessagePack; using MessagePack; using MessagePack.Formatters; using MessagePack.Resolvers; using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; using System.Buffers; using System.IO; diff --git a/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs b/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs index 30b59313c..9d535bdd3 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Parser/MspDbRestorationKeyTests.cs @@ -29,13 +29,13 @@ public void MspDbRestorationKeyTest() { [TestMethod()] public void MspDbRestorationKeySerializedBytesTest() { - IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> key = new MspDbRestorationKey("MspKey", -1); - IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> expected = key; - IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> actual; + IReferRestorationKey key = new MspDbRestorationKey("MspKey", -1); + IReferRestorationKey expected = key; + IReferRestorationKey actual; var bytes = Convert.FromBase64String("kgGCo0tleaZNc3BLZXmoUHJpb3JpdHn/"); using (var stream = new MemoryStream(bytes)) { - actual = Common.MessagePack.MessagePackDefaultHandler.LoadFromStream, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase>>(stream); + actual = Common.MessagePack.MessagePackDefaultHandler.LoadFromStream(stream); } Assert.AreEqual(expected.Key, actual.Key); @@ -44,11 +44,12 @@ public void MspDbRestorationKeySerializedBytesTest() { [TestMethod()] public void TextDbRestorationKeyTest() { IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> key = new TextDbRestorationKey("TextKey", 3); - IReferRestorationKey, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase> actual; + IReferRestorationKey actual; using (var stream = new MemoryStream()) { - Common.MessagePack.MessagePackDefaultHandler.SaveToStream, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase>>(key, stream); - actual = Common.MessagePack.MessagePackDefaultHandler.LoadFromStream, MoleculeMsReference, MsScanMatchResult, MoleculeDataBase>>(stream); + Common.MessagePack.MessagePackDefaultHandler.SaveToStream(key, stream); + stream.Position = 0; + actual = Common.MessagePack.MessagePackDefaultHandler.LoadFromStream(stream); } Assert.AreEqual(key.Key, actual.Key); @@ -61,7 +62,7 @@ public void ShotgunProteomicsRestorationKeyTest() { var parameter = new CompMs.Common.Parameter.MsRefSearchParameterBase(); var proteomicsParameter = RoundTrip(new CompMs.MsdialCore.Parameter.ProteomicsParameter()); IReferRestorationKey key = new ShotgunProteomicsRestorationKey("ShotgunKey", 5, parameter, proteomicsParameter, CompMs.Common.DataObj.Result.SourceType.MspDB); - var actual = RoundTrip(key); + var actual = RoundTrip(key); Assert.AreEqual(key.Key, actual.Key); Assert.AreEqual(key.Priority, actual.Priority); @@ -84,7 +85,7 @@ public void ProteomicsParameterRoundTripsDefaultPayload() { public void EadLipidDatabaseRestorationKeyTest() { var parameter = new CompMs.Common.Parameter.MsRefSearchParameterBase(); IReferRestorationKey<(IAnnotationQuery, MoleculeMsReference), MoleculeMsReference, MsScanMatchResult, EadLipidDatabase> key = new EadLipidDatabaseRestorationKey("EadKey", 7, parameter, CompMs.Common.DataObj.Result.SourceType.MspDB); - var actual = RoundTrip(key); + var actual = RoundTrip(key); Assert.AreEqual(key.Key, actual.Key); Assert.AreEqual(key.Priority, actual.Priority); From 1d6718bb6ecb7a0eb1a74c1fa1e5ecafc1ecc36c Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 13 Jul 2026 15:38:53 +0900 Subject: [PATCH 24/30] Delete the commented-out legacy declaration --- .../MsdialCore/Algorithm/Annotation/AnnotatorContainer.cs | 2 -- .../MsdialCore/DataObj/EadLipidAnnotatorParameterPair.cs | 1 - .../MsdialCore/DataObj/MetabolomicsAnnotatorParameterPair.cs | 1 - .../MsdialCore/DataObj/ProteomicsAnnotatorParameterPair.cs | 1 - 4 files changed, 5 deletions(-) diff --git a/src/MSDIAL5/MsdialCore/Algorithm/Annotation/AnnotatorContainer.cs b/src/MSDIAL5/MsdialCore/Algorithm/Annotation/AnnotatorContainer.cs index a3ce0cffd..1280ff7f7 100644 --- a/src/MSDIAL5/MsdialCore/Algorithm/Annotation/AnnotatorContainer.cs +++ b/src/MSDIAL5/MsdialCore/Algorithm/Annotation/AnnotatorContainer.cs @@ -162,7 +162,6 @@ public ShotgunProteomicsDBAnnotatorContainer( AnnotatorID = AnnotatorKey.Key; } - [SerializationConstructor] public ShotgunProteomicsDBAnnotatorContainer( IReferRestorationKey annotatorKey, ShotgunProteomicsDB database, @@ -274,7 +273,6 @@ public EadLipidDatabaseAnnotatorContainer( public string AnnotatorID { get; } [Key("AnnotatorKey")] - //public IReferRestorationKey<(IAnnotationQuery, MoleculeMsReference), MoleculeMsReference, MsScanMatchResult, EadLipidDatabase> AnnotatorKey { get; set; } public IReferRestorationKey AnnotatorKey { get; set; } [Key("Parameter")] diff --git a/src/MSDIAL5/MsdialCore/DataObj/EadLipidAnnotatorParameterPair.cs b/src/MSDIAL5/MsdialCore/DataObj/EadLipidAnnotatorParameterPair.cs index ab9ad7624..9277eff11 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/EadLipidAnnotatorParameterPair.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/EadLipidAnnotatorParameterPair.cs @@ -24,7 +24,6 @@ public EadLipidAnnotatorParameterPair(IReferRestorationKey serializableAnnotator } [Key("SerializableAnnotatorKey")] - //public IAnnotationQueryFactoryGenerationKey SerializableAnnotatorKey { get; } public IReferRestorationKey SerializableAnnotatorKey { get; } [IgnoreMember] diff --git a/src/MSDIAL5/MsdialCore/DataObj/MetabolomicsAnnotatorParameterPair.cs b/src/MSDIAL5/MsdialCore/DataObj/MetabolomicsAnnotatorParameterPair.cs index c61400961..8d6cef041 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/MetabolomicsAnnotatorParameterPair.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/MetabolomicsAnnotatorParameterPair.cs @@ -20,7 +20,6 @@ public MetabolomicsAnnotatorParameterPair(IReferRestorationKey serializableAnnot } [Key("SerializableAnnotatorKey")] - //public IAnnotationQueryFactoryGenerationKey SerializableAnnotatorKey { get; } public IReferRestorationKey SerializableAnnotatorKey { get; } [IgnoreMember] diff --git a/src/MSDIAL5/MsdialCore/DataObj/ProteomicsAnnotatorParameterPair.cs b/src/MSDIAL5/MsdialCore/DataObj/ProteomicsAnnotatorParameterPair.cs index 38a343f51..c9f6f3cf5 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/ProteomicsAnnotatorParameterPair.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/ProteomicsAnnotatorParameterPair.cs @@ -23,7 +23,6 @@ public ProteomicsAnnotatorParameterPair(IReferRestorationKey serializableAnnotat } [Key("SerializableAnnotatorKey")] - //public IAnnotationQueryFactoryGenerationKey SerializableAnnotatorKey { get; } public IReferRestorationKey SerializableAnnotatorKey { get; } [Key("ProteomicsParameter")] From 5cb02939086beac1e7d6108666b76c92c85fd259 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 13 Jul 2026 18:34:40 +0900 Subject: [PATCH 25/30] Update LargeListMessagePack --- .../MessagePack/LargeListMessagePack.cs | 111 +++++++++--------- 1 file changed, 56 insertions(+), 55 deletions(-) diff --git a/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs b/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs index 9e8f0af8c..2e28c8760 100644 --- a/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs +++ b/src/Common/CommonStandard/MessagePack/LargeListMessagePack.cs @@ -27,101 +27,102 @@ public static void Serialize(Stream stream, IReadOnlyList value) { } public static List Deserialize(Stream stream) { - using (var memory = new MemoryStream()) { - stream.CopyTo(memory); - var reader = new MessagePackReader(new ReadOnlyMemory(memory.GetBuffer(), 0, (int)memory.Length)); - return DeserializeCore(ref reader, MessagePackSerializerOptions.Standard); - } + return DeserializeCore(stream, MessagePackSerializerOptions.Standard); } public static IEnumerable> DeserializeIncremental(Stream stream) { - using (var memory = new MemoryStream()) { - stream.CopyTo(memory); - var reader = new MessagePackReader(new ReadOnlyMemory(memory.GetBuffer(), 0, (int)memory.Length)); - foreach (var result in DeserializeIncrementalCore(ref reader, MessagePackSerializerOptions.Standard)) { - yield return result; - } + foreach (var result in DeserializeIncrementalCore(stream, MessagePackSerializerOptions.Standard)) { + yield return result; } } public static T DeserializeAt(Stream stream, int index) { - using (var memory = new MemoryStream()) { - stream.CopyTo(memory); - var reader = new MessagePackReader(new ReadOnlyMemory(memory.GetBuffer(), 0, (int)memory.Length)); - return DeserializeAt(ref reader, MessagePackSerializerOptions.Standard, index); - } + return DeserializeAt(stream, MessagePackSerializerOptions.Standard, index); } - public static T DeserializeAt(ref MessagePackReader reader, MessagePackSerializerOptions options, int index) { - while (!reader.End) { - var success = TryDeserializeAtOrSkip(ref reader, options, index, out var result, out int skipArraySize); + public static T DeserializeAt(Stream stream, MessagePackSerializerOptions options, int index) { + while (index >= 0) { + var success = TryDeserializeAtOrSkip(stream, options, index, out var result, out int skipArraySize); if (success) { return result; } + else if (skipArraySize == 0) { + break; + } index -= skipArraySize; } return default; } - private static List DeserializeCore(ref MessagePackReader reader, MessagePackSerializerOptions options) { - var res = new List>(); - while (!reader.End) { - var tmp = DeserializeEach(ref reader, options); - if (tmp != null && tmp.Count > 0) { - res.Add(tmp); + private static List DeserializeCore(Stream stream, MessagePackSerializerOptions options) { + var res = new List(); + while (true) { + var tmp = DeserializeEach(stream, options); + if (tmp is null) { + break; } + res.AddRange(tmp); } - - return res.SelectMany(r => r).ToList(); + return res; } - private static IEnumerable> DeserializeIncrementalCore(ref MessagePackReader reader, MessagePackSerializerOptions options) { - var results = new List>(); - while (!reader.End) { - results.Add(DeserializeEach(ref reader, options)); + private static IEnumerable> DeserializeIncrementalCore(Stream stream, MessagePackSerializerOptions options) { + while (true) { + var data = DeserializeEach(stream, options); + if (data is null) { + yield break; + } + yield return data; } - - return results; } - private static List DeserializeEach(ref MessagePackReader reader, MessagePackSerializerOptions options) { + private static List? DeserializeEach(Stream stream, MessagePackSerializerOptions options) { var resolver = CompositeResolver.Create( [new DeserializedDataContainer.DeserializedDataContainerFormatter(),], [StandardResolver.Instance]); - if (!reader.End) { - return MessagePackSerializer.Deserialize>(ref reader, options.WithResolver(resolver).WithCompression(MessagePackCompression.Lz4BlockArray)).Data; + try { + return MessagePackSerializer.Deserialize>(stream, options.WithResolver(resolver).WithCompression(MessagePackCompression.Lz4BlockArray))?.Data; + } + catch (MessagePackSerializationException) { + return default; } - - return []; } - private static bool TryDeserializeAtOrSkip(ref MessagePackReader reader, MessagePackSerializerOptions options, int index, out T result, out int skipArraySize) { + private static bool TryDeserializeAtOrSkip(Stream stream, MessagePackSerializerOptions options, int index, out T? result, out int skipArraySize) { var resolver = CompositeResolver.Create( [DeserializedDataContainer.CreateFormatter(index),], [StandardResolver.Instance]); var newOptions = options.WithCompression(MessagePackCompression.Lz4BlockArray).WithResolver(resolver); - var deserialized = MessagePackSerializer.Deserialize>(ref reader, newOptions); - result = deserialized.Data is null ? default : deserialized.Data.FirstOrDefault(); - skipArraySize = deserialized.Length; - return result != null; + try { + var deserialized = MessagePackSerializer.Deserialize>(stream, newOptions); + result = deserialized.Data is null ? default : deserialized.Data.FirstOrDefault(); + skipArraySize = deserialized.Length; + return result != null; + } + catch (MessagePackSerializationException) { + result = default; + skipArraySize = 0; + return false; + } } + [MessagePackFormatter(typeof(DataContainerFormatter<>))] internal sealed class SerializingDataContainer { public sbyte VersionCode { get; } = 2; public Memory Data { get; set; } + } - internal class DataContainerFormatter : IMessagePackFormatter?> { - public SerializingDataContainer Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { - throw new NotSupportedException(); - } + internal class DataContainerFormatter : IMessagePackFormatter?> { + public SerializingDataContainer Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + throw new NotSupportedException(); + } - public void Serialize(ref MessagePackWriter writer, SerializingDataContainer value, MessagePackSerializerOptions options) { - writer.WriteInt8(value.VersionCode); - writer.WriteArrayHeader(value.Data.Length); - for (int i = 0; i < value.Data.Length; i++) { - MessagePackSerializer.Serialize(ref writer, value.Data.Span[i], options); - } + public void Serialize(ref MessagePackWriter writer, SerializingDataContainer value, MessagePackSerializerOptions options) { + writer.WriteInt8(value.VersionCode); + writer.WriteArrayHeader(value.Data.Length); + for (int i = 0; i < value.Data.Length; i++) { + MessagePackSerializer.Serialize(ref writer, value.Data.Span[i], options); } } } @@ -132,7 +133,7 @@ private sealed class DeserializedDataContainer { public static IMessagePackFormatter> CreateFormatter(int index) => new SpecificDataFormatter(index); -#pragma warning disable MsgPack009 +#pragma warning disable MsgPack009, MsgPack010 internal class DeserializedDataContainerFormatter : IMessagePackFormatter?> { public DeserializedDataContainer Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { int length; @@ -191,7 +192,7 @@ public void Serialize(ref MessagePackWriter writer, DeserializedDataContainer throw new NotSupportedException(); } } -#pragma warning restore MsgPack009 +#pragma warning restore MsgPack009, MsgPack010 } } } From 57927d19a15563456792f1941c98aa2719846916 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 13 Jul 2026 18:40:42 +0900 Subject: [PATCH 26/30] Fix IMoleculeProperty serializer --- .../Components/Interfaces/IMoleculeProperty.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs b/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs index e18ef5299..355415b64 100644 --- a/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs +++ b/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs @@ -58,6 +58,11 @@ public IMoleculeProperty Deserialize(ref MessagePackReader reader, MessagePackSe } public void Serialize(ref MessagePackWriter writer, IMoleculeProperty value, MessagePackSerializerOptions options) { + if (value is null) + { + writer.WriteNil(); + return; + } writer.WriteArrayHeader(5); writer.Write(value?.Name); options.Resolver.GetFormatterWithVerify().Serialize(ref writer, value?.Formula, options); From 7e047933d5ba2431fe32454d4f804c88da842f54 Mon Sep 17 00:00:00 2001 From: Yuki Matsuzawa Date: Mon, 13 Jul 2026 18:40:52 +0900 Subject: [PATCH 27/30] Fix AdductIon deserializer --- src/Common/CommonStandard/DataObj/Property/AdductIon.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/CommonStandard/DataObj/Property/AdductIon.cs b/src/Common/CommonStandard/DataObj/Property/AdductIon.cs index 4fecf4ca3..99e853ad1 100644 --- a/src/Common/CommonStandard/DataObj/Property/AdductIon.cs +++ b/src/Common/CommonStandard/DataObj/Property/AdductIon.cs @@ -2,7 +2,6 @@ using CompMs.Common.Parser; using MessagePack; using MessagePack.Formatters; -using MessagePack.Resolvers; using System; using System.Collections.Concurrent; @@ -175,6 +174,9 @@ public AdductIon Deserialize(ref MessagePackReader reader, MessagePackSerializer } var count = reader.ReadArrayHeader(); if (count < 3) { + for (int i = 0; i < count; i++) { + reader.Skip(); + } return Default; } reader.Skip(); From 3fc52f710cbb2b53a9772a97a2f328512e83dff6 Mon Sep 17 00:00:00 2001 From: YukiMatsuzawa <122433968+YukiMatsuzawa@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:06:35 +0900 Subject: [PATCH 28/30] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../Components/Interfaces/IMoleculeProperty.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs b/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs index 355415b64..b921cfa78 100644 --- a/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs +++ b/src/Common/CommonStandard/Components/Interfaces/IMoleculeProperty.cs @@ -44,7 +44,10 @@ public static IMoleculeProperty AsPutative(this IMoleculeProperty molecule) { internal class MoleculePropertyFormatter : IMessagePackFormatter { - public IMoleculeProperty Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + public IMoleculeProperty? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { + if (reader.TryReadNil()) { + return null; + } var count = reader.ReadArrayHeader(); if (count != 5) { throw new MessagePackSerializationException($"Unexpected array length for {nameof(IMoleculeProperty)}: {count}."); From e17708c29f0a123c151929e1cb81339432b9cc79 Mon Sep 17 00:00:00 2001 From: YukiMatsuzawa <122433968+YukiMatsuzawa@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:07:43 +0900 Subject: [PATCH 29/30] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs b/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs index 327ccc10f..7aac54253 100644 --- a/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs +++ b/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs @@ -11,7 +11,7 @@ namespace CompMs.MsdialCore.Parser public interface IReferRestorationKey { ISerializableAnnotator Accept(ILoadAnnotatorVisitor visitor, TDatabase database); - IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annoatorVIsitor, TDatabase database); + IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annotatorVisitor, TDatabase database); int Priority { get; } string Key { get; } From 05f220518117c8c8dd330babec15be1aaad877a6 Mon Sep 17 00:00:00 2001 From: YukiMatsuzawa <122433968+YukiMatsuzawa@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:08:02 +0900 Subject: [PATCH 30/30] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs b/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs index 7aac54253..37fad8f53 100644 --- a/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs +++ b/src/MSDIAL5/MsdialCore/Parser/IReferRestorationKey.cs @@ -23,6 +23,6 @@ public interface IReferRestorationKey : IReferRestorationKey { - IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annoatorVIsitor, TDatabase database); + IAnnotationQueryFactory Accept(IAnnotationQueryFactoryGenerationVisitor factoryVisitor, ILoadAnnotatorVisitor annotatorVisitor, TDatabase database); } }