diff --git a/CHANGELOG.md b/CHANGELOG.md index 465d8b3..add1120 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## Unreleased +XLSX: decode dates with custom `` format codes (e.g. `dd/mm/yyyy`) using the workbook's own format code instead of leaking through as raw serial numbers. Removed the `raw` decoding mode in favour of always returning typed values. + +Built-in date cells (numFmtId 14-17, 22) and ODS date cells now render as `yyyy-MM-dd` by default instead of an ISO timestamp like `2008-07-21T00:00:00.000`. Added a `dateFormat` parameter to `SpreadsheetDecoder.decodeBytes` and `SpreadsheetDecoder.decodeBuffer` so callers can choose any pattern (e.g. `dd/MM/yyyy`, `yyyy.MM.dd`); custom-numFmt date cells honour the caller's `dateFormat` when explicitly set, otherwise fall back to the workbook's own format code. + ## 2.3.0 Update dependencies diff --git a/README.md b/README.md index 1ab36b7..370238e 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,22 @@ This implementation doesn't support following features: - hidden rows (visible in resulting tables) - hidden columns (visible in resulting tables) -For XLSX format, this implementation only supports native Excel format for date, time and boolean type conversion. -In other words, custom format for date, time, boolean aren't supported and then file exported from LibreOffice as well. +For XLSX format, this implementation supports the native Excel formats for date, time and boolean type conversion, plus custom `` format codes declared in the workbook (e.g. `dd/mm/yyyy`). + +Important: Excel often stores date cells as numeric serial values and only formats them for display. The decoder applies the workbook's format code to render the date as a string. If you need a fixed text representation independent of the workbook formatting, the source cell must be stored as text in the spreadsheet. + +### Customising the date output format + +By default, date cells (XLSX built-in numFmtIds 14-17, 22 and ODS `date` cells) are rendered as `yyyy-MM-dd` (e.g. `2008-07-21`). You can override this from the call site by passing `dateFormat`: + + var decoder = SpreadsheetDecoder.decodeBytes( + bytes, + dateFormat: 'dd/MM/yyyy', // → 21/07/2008 + ); + +Supported tokens (case-insensitive): `yyyy`, `yy`, `mm`/`m` (month — context-sensitive vs minutes), `dd`/`d`/`ddd`/`dddd`, `hh`/`h`, `ss`/`s`, `AM/PM`. Any other characters in the pattern (`-`, `/`, `.`, spaces, etc.) are emitted as literal separators. + +For cells that use a custom `` format code declared in the workbook, the workbook's own format code is used unless you explicitly pass a non-default `dateFormat`, in which case your format wins. ## License diff --git a/example/spreadsheet_decoder.dart b/example/spreadsheet_decoder.dart index 02999ae..0c61ecf 100644 --- a/example/spreadsheet_decoder.dart +++ b/example/spreadsheet_decoder.dart @@ -1,11 +1,11 @@ import 'dart:io'; -import 'package:path/path.dart'; import 'package:spreadsheet_decoder/spreadsheet_decoder.dart'; void main(List args) { - var file = 'test/files/test.xlsx'; + var file = 'test/files/new_sample_file_bytco.xlsx'; var bytes = File(file).readAsBytesSync(); - var decoder = SpreadsheetDecoder.decodeBytes(bytes, update: true); + var decoder = SpreadsheetDecoder.decodeBytes(bytes, + update: false, dateFormat: 'dd/MM/yyyy'); for (var table in decoder.tables.keys) { print(table); print(decoder.tables[table]!.maxCols); @@ -15,31 +15,31 @@ void main(List args) { } } - var sheet = decoder.tables.keys.first; - decoder - ..updateCell(sheet, 0, 0, "L'oiseau <\"coucou\">") - ..updateCell(sheet, 1, 0, 'B') - ..updateCell(sheet, 2, 0, 'C') - ..updateCell(sheet, 1, 1, 42.3) - ..insertRow(sheet, 1) - ..insertRow(sheet, 13) - ..updateCell(sheet, 0, 13, 'A14') - ..updateCell(sheet, 0, 12, 'A13') - ..insertColumn(sheet, 0) - ..removeRow(sheet, 1) - ..removeColumn(sheet, 2); + // var sheet = decoder.tables.keys.first; + // decoder + // ..updateCell(sheet, 0, 0, "L'oiseau <\"coucou\">") + // ..updateCell(sheet, 1, 0, 'B') + // ..updateCell(sheet, 2, 0, 'C') + // ..updateCell(sheet, 1, 1, 42.3) + // ..insertRow(sheet, 1) + // ..insertRow(sheet, 13) + // ..updateCell(sheet, 0, 13, 'A14') + // ..updateCell(sheet, 0, 12, 'A13') + // ..insertColumn(sheet, 0) + // ..removeRow(sheet, 1) + // ..removeColumn(sheet, 2); - File(join('test/out/${basename(file)}')) - ..createSync(recursive: true) - ..writeAsBytesSync(decoder.encode()); + // File(join('test/out/${basename(file)}')) + // ..createSync(recursive: true) + // ..writeAsBytesSync(decoder.encode()); - print('************************************************************'); - for (var table in decoder.tables.keys) { - print(table); - print(decoder.tables[table]!.maxCols); - print(decoder.tables[table]!.maxRows); - for (var row in decoder.tables[table]!.rows) { - print('$row'); - } - } + // print('************************************************************'); + // for (var table in decoder.tables.keys) { + // print(table); + // print(decoder.tables[table]!.maxCols); + // print(decoder.tables[table]!.maxRows); + // for (var row in decoder.tables[table]!.rows) { + // print('$row'); + // } + // } } diff --git a/lib/spreadsheet_decoder.dart b/lib/spreadsheet_decoder.dart index 3c0b08e..b202527 100644 --- a/lib/spreadsheet_decoder.dart +++ b/lib/spreadsheet_decoder.dart @@ -1,7 +1,6 @@ library spreadsheet_decoder; import 'dart:convert'; -import 'dart:typed_data'; import 'package:archive/archive.dart'; import 'package:xml/xml.dart'; diff --git a/lib/src/ods.dart b/lib/src/ods.dart index 261b736..4817de9 100644 --- a/lib/src/ods.dart +++ b/lib/src/ods.dart @@ -19,9 +19,12 @@ class OdsDecoder extends SpreadsheetDecoder { String get extension => '.ods'; final Map> _styleNames = {}; - OdsDecoder(Archive archive, {bool update = false}) { + OdsDecoder(Archive archive, + {bool update = false, + String dateFormat = SpreadsheetDecoder.defaultDateFormat}) { _archive = archive; _update = update; + _dateFormat = dateFormat; _tables = {}; _parseContent(); } @@ -96,8 +99,11 @@ class OdsDecoder extends SpreadsheetDecoder { void _parseContent() { var file = _archive.findFile(contentXML); - file?.decompress(); - var content = XmlDocument.parse(utf8.decode(file?.content)); + if (file == null) { + throw FormatException('Missing required file: $contentXML'); + } + file.decompress(); + var content = XmlDocument.parse(utf8.decode(file.content)); if (_update == true) { _archiveFiles = {}; _sheets = {}; @@ -185,6 +191,7 @@ class OdsDecoder extends SpreadsheetDecoder { dynamic _readCell(XmlElement node) { dynamic value; var type = node.getAttribute('office:value-type'); + switch (type) { case 'float': case 'percentage': @@ -196,8 +203,9 @@ class OdsDecoder extends SpreadsheetDecoder { node.getAttribute('office:boolean-value')!.toLowerCase() == 'true'; break; case 'date': - value = DateTime.parse(node.getAttribute('office:date-value')!) - .toIso8601String(); + value = _formatDateTimeWithCode( + DateTime.parse(node.getAttribute('office:date-value')!), + _dateFormat); break; case 'time': value = node.getAttribute('office:time-value'); diff --git a/lib/src/spreadsheet.dart b/lib/src/spreadsheet.dart index 0316ad1..7cc28dd 100644 --- a/lib/src/spreadsheet.dart +++ b/lib/src/spreadsheet.dart @@ -13,7 +13,8 @@ String _normalizeNewLine(String text) { return text.replaceAll('\r\n', '\n'); } -SpreadsheetDecoder _newSpreadsheetDecoder(Archive archive, bool update) { +SpreadsheetDecoder _newSpreadsheetDecoder( + Archive archive, bool update, String dateFormat) { // Lookup at file format String? format; @@ -34,9 +35,9 @@ SpreadsheetDecoder _newSpreadsheetDecoder(Archive archive, bool update) { switch (format) { case _spreasheetOds: - return OdsDecoder(archive, update: update); + return OdsDecoder(archive, update: update, dateFormat: dateFormat); case _spreasheetXlsx: - return XlsxDecoder(archive, update: update); + return XlsxDecoder(archive, update: update, dateFormat: dateFormat); default: throw UnsupportedError('Spreadsheet format unsupported'); } @@ -44,7 +45,16 @@ SpreadsheetDecoder _newSpreadsheetDecoder(Archive archive, bool update) { /// Decode a spreadsheet file. abstract class SpreadsheetDecoder { + /// Default output format used for date cells when none is provided. + /// Format tokens follow Excel-style codes (case-insensitive): + /// `yyyy`/`yy`, `mm`/`m` (month — context-sensitive), + /// `dd`/`d`/`ddd`/`dddd`, `hh`/`h`, `ss`/`s`, `AM/PM`. + /// Common intl-style patterns like `dd/MM/yyyy` also work because the + /// pattern is matched case-insensitively. + static const String defaultDateFormat = 'yyyy-MM-dd'; + late bool _update; + late String _dateFormat; late Archive _archive; late Map _sheets; late Map _xmlFiles; @@ -63,16 +73,25 @@ abstract class SpreadsheetDecoder { SpreadsheetDecoder(); + /// Decode an XLSX/ODS spreadsheet from raw [data] bytes. + /// + /// Pass [dateFormat] to control how date cells are rendered as strings + /// (e.g. `'dd/MM/yyyy'`, `'yyyy-MM-dd'`). Defaults to + /// [defaultDateFormat] (`yyyy-MM-dd`). factory SpreadsheetDecoder.decodeBytes(List data, - {bool update = false, bool verify = false}) { + {bool update = false, + bool verify = false, + String dateFormat = defaultDateFormat}) { var archive = ZipDecoder().decodeBytes(data, verify: verify); - return _newSpreadsheetDecoder(archive, update); + return _newSpreadsheetDecoder(archive, update, dateFormat); } - factory SpreadsheetDecoder.decodeBuffer(InputStreamBase input, - {bool update = false, bool verify = false}) { - var archive = ZipDecoder().decodeBuffer(input, verify: verify); - return _newSpreadsheetDecoder(archive, update); + factory SpreadsheetDecoder.decodeBuffer(InputStream input, + {bool update = false, + bool verify = false, + String dateFormat = defaultDateFormat}) { + var archive = ZipDecoder().decodeStream(input, verify: verify); + return _newSpreadsheetDecoder(archive, update, dateFormat); } /// Dump XML content (for debug purpose) @@ -169,7 +188,7 @@ abstract class SpreadsheetDecoder { var content = utf8.encode(xml); _archiveFiles[xmlFile] = ArchiveFile(xmlFile, content.length, content); } - return ZipEncoder().encode(_cloneArchive(_archive)) as List; + return ZipEncoder().encode(_cloneArchive(_archive)); } /// Encode data url @@ -188,10 +207,10 @@ abstract class SpreadsheetDecoder { if (_archiveFiles.containsKey(file.name)) { copy = _archiveFiles[file.name]!; } else { - var content = file.content as Uint8List; - var compress = file.compress; + var content = file.content; + var compression = file.compression; copy = ArchiveFile(file.name, content.length, content) - ..compress = compress; + ..compression = compression; } clone.addFile(copy); } diff --git a/lib/src/xlsx.dart b/lib/src/xlsx.dart index 22421bd..55a8749 100644 --- a/lib/src/xlsx.dart +++ b/lib/src/xlsx.dart @@ -75,6 +75,263 @@ String _twoDigits(int n) { return '0$n'; } +String _fourDigits(int n) { + var v = n.abs().toString().padLeft(4, '0'); + return n < 0 ? '-$v' : v; +} + +const List _monthNames = [ + 'January', 'February', 'March', 'April', 'May', 'June', + 'July', 'August', 'September', 'October', 'November', 'December' // +]; + +const List _dayNames = [ + 'Monday', 'Tuesday', 'Wednesday', 'Thursday', + 'Friday', 'Saturday', 'Sunday' // +]; + +/// Strip literal/escaped portions of an Excel number format code so token +/// scanning only sees real format tokens. Removes "..." literals, +/// [..] bracket sections (colors, conditions, locale modifiers like +/// `[$-409]`) and `\X` escape pairs. +String _stripFormatLiterals(String code) { + var buf = StringBuffer(); + var i = 0; + while (i < code.length) { + var ch = code[i]; + if (ch == '"') { + i++; + while (i < code.length && code[i] != '"') { + i++; + } + if (i < code.length) i++; + continue; + } + if (ch == r'\' && i + 1 < code.length) { + i += 2; + continue; + } + if (ch == '[') { + while (i < code.length && code[i] != ']') { + i++; + } + if (i < code.length) i++; + continue; + } + buf.write(ch); + i++; + } + return buf.toString(); +} + +/// Returns true if the given Excel format code represents a date or +/// date+time value (contains year or day tokens). +bool _isDateTimeFormatCode(String code) { + var stripped = _stripFormatLiterals(code).toLowerCase(); + var section = stripped.split(';').first; + return section.contains('y') || section.contains('d'); +} + +/// Returns true if [fmtId] is a built-in (implicit) number format that Excel +/// renders as a date, meaning the cell value holds a date serial. +/// +/// Besides the well-known western formats (14-17, 22) this covers the +/// locale-specific ranges the spec reserves for CJK (27-36, 50-58) and Thai +/// (71-81) date formats. Those ids carry no `` entry in the workbook, +/// so they have to be recognised by id alone. +bool _isBuiltInDateFormat(int fmtId) { + if (((fmtId >= 14) && (fmtId <= 17)) || (fmtId == 22)) return true; + // CJK: yyyy年m月, m月d日, yyyy年m月d日, era-based ge.m.d, ... + if ((fmtId >= 27) && (fmtId <= 31)) return true; + if (fmtId == 36) return true; + if ((fmtId >= 50) && (fmtId <= 54)) return true; + if ((fmtId == 57) || (fmtId == 58)) return true; + // Thai: ว/ด/ปปปป, ว-ดดด-ปป, d/m/bb, ... + if ((fmtId >= 71) && (fmtId <= 74)) return true; + if ((fmtId == 77) || (fmtId == 81)) return true; + return false; +} + +/// Returns true if [fmtId] is a built-in number format that Excel renders as a +/// time of day only (no date part). Companion to [_isBuiltInDateFormat]. +bool _isBuiltInTimeFormat(int fmtId) { + if ((fmtId >= 18) && (fmtId <= 21)) return true; + // CJK: h時mm分, 上午/下午h時mm分ss秒, ... + if ((fmtId >= 32) && (fmtId <= 35)) return true; + if ((fmtId >= 45) && (fmtId <= 47)) return true; + if ((fmtId == 55) || (fmtId == 56)) return true; + // Thai: ช:นน, นน:ทท.0, ... + if ((fmtId == 75) || (fmtId == 76)) return true; + if ((fmtId >= 78) && (fmtId <= 80)) return true; + return false; +} + +/// Returns true if the format code represents a time-only value +/// (hours/minutes/seconds without a date part). +bool _isTimeOnlyFormatCode(String code) { + var stripped = _stripFormatLiterals(code).toLowerCase(); + var section = stripped.split(';').first; + if (section.contains('y') || section.contains('d')) return false; + return section.contains('h') || section.contains('s'); +} + +/// Format a [DateTime] using a (subset of) Excel number format tokens. +/// Supports y/yy/yyyy, m/mm/mmm/mmmm/mmmmm (month or minute by context), +/// d/dd/ddd/dddd, h/hh, s/ss, AM/PM and A/P. Literal characters and +/// quoted text pass through unchanged. +String _formatDateTimeWithCode(DateTime date, String code) { + var section = code.split(';').first; + var lower = section.toLowerCase(); + var twelveHour = lower.contains('am/pm') || lower.contains('a/p'); + var hour12 = ((date.hour + 11) % 12) + 1; + + var buf = StringBuffer(); + var i = 0; + var lastWasHour = false; + while (i < section.length) { + var ch = section[i]; + var lc = ch.toLowerCase(); + + if (ch == '"') { + i++; + while (i < section.length && section[i] != '"') { + buf.write(section[i]); + i++; + } + if (i < section.length) i++; + continue; + } + if (ch == r'\' && i + 1 < section.length) { + buf.write(section[i + 1]); + i += 2; + continue; + } + if (ch == '[') { + var end = section.indexOf(']', i + 1); + if (end == -1) { + i++; + continue; + } + var inner = section.substring(i + 1, end).toLowerCase(); + if (inner == 'h' || inner == 'hh') { + buf.write(date.hour.toString()); + } else if (inner == 'm' || inner == 'mm') { + buf.write(date.minute.toString()); + } else if (inner == 's' || inner == 'ss') { + buf.write(date.second.toString()); + } + i = end + 1; + continue; + } + + // AM/PM marker (check before single-char fallthrough) + if (i + 5 <= section.length && + section.substring(i, i + 5).toLowerCase() == 'am/pm') { + var src = section.substring(i, i + 5); + var meridiem = date.hour < 12 ? 'AM' : 'PM'; + if (src == src.toLowerCase()) meridiem = meridiem.toLowerCase(); + buf.write(meridiem); + i += 5; + continue; + } + if (i + 3 <= section.length && + section.substring(i, i + 3).toLowerCase() == 'a/p') { + var src = section.substring(i, i + 3); + var meridiem = date.hour < 12 ? 'A' : 'P'; + if (src == src.toLowerCase()) meridiem = meridiem.toLowerCase(); + buf.write(meridiem); + i += 3; + continue; + } + + if ('ymdhs'.contains(lc)) { + var run = lc; + var j = i + 1; + while (j < section.length && section[j].toLowerCase() == lc) { + run += lc; + j++; + } + switch (lc) { + case 'y': + if (run.length >= 4) { + buf.write(_fourDigits(date.year)); + } else { + buf.write(_twoDigits(date.year % 100)); + } + lastWasHour = false; + break; + case 'd': + if (run.length == 1) { + buf.write(date.day.toString()); + } else if (run.length == 2) { + buf.write(_twoDigits(date.day)); + } else if (run.length == 3) { + buf.write(_dayNames[(date.weekday - 1) % 7].substring(0, 3)); + } else { + buf.write(_dayNames[(date.weekday - 1) % 7]); + } + lastWasHour = false; + break; + case 'h': + var h = twelveHour ? hour12 : date.hour; + if (run.length == 1) { + buf.write(h.toString()); + } else { + buf.write(_twoDigits(h)); + } + lastWasHour = true; + break; + case 's': + if (run.length == 1) { + buf.write(date.second.toString()); + } else { + buf.write(_twoDigits(date.second)); + } + lastWasHour = false; + break; + case 'm': + var isMinute = lastWasHour; + if (!isMinute) { + var k = j; + while (k < section.length && section[k] == ' ') { + k++; + } + if (k < section.length && section[k].toLowerCase() == 's') { + isMinute = true; + } + } + if (isMinute) { + if (run.length == 1) { + buf.write(date.minute.toString()); + } else { + buf.write(_twoDigits(date.minute)); + } + } else { + if (run.length == 1) { + buf.write(date.month.toString()); + } else if (run.length == 2) { + buf.write(_twoDigits(date.month)); + } else if (run.length == 3) { + buf.write(_monthNames[date.month - 1].substring(0, 3)); + } else if (run.length == 5) { + buf.write(_monthNames[date.month - 1].substring(0, 1)); + } else { + buf.write(_monthNames[date.month - 1]); + } + } + lastWasHour = false; + break; + } + i = j; + continue; + } + + buf.write(ch); + i++; + } + return buf.toString(); +} + /// Returns the coordinates from a cell name. /// "A1" returns [1, 1] and the "B3" return [2, 3]. List cellCoordsFromCellId(String cellId) { @@ -97,13 +354,19 @@ class XlsxDecoder extends SpreadsheetDecoder { final List _sharedStrings = []; final List _numFormats = []; + // Custom number format codes from (numFmtId >= 164 typically, + // but spec allows any id when overridden). Maps numFmtId -> formatCode. + final Map _customNumFormats = {}; String? _stylesTarget; String? _sharedStringsTarget; final Map _worksheetTargets = {}; - XlsxDecoder(Archive archive, {bool update = false}) { + XlsxDecoder(Archive archive, + {bool update = false, + String dateFormat = SpreadsheetDecoder.defaultDateFormat}) { _archive = archive; _update = update; + _dateFormat = dateFormat; if (_update == true) { _archiveFiles = {}; _sheets = {}; @@ -259,6 +522,19 @@ class XlsxDecoder extends SpreadsheetDecoder { if (styles != null) { styles.decompress(); var document = XmlDocument.parse(utf8.decode(styles.content)); + + // Parse custom numFmts (formatCode strings keyed by numFmtId). + var numFmtsElems = document.findAllElements('numFmts'); + if (numFmtsElems.isNotEmpty) { + numFmtsElems.first.findElements('numFmt').forEach((node) { + var idAttr = node.getAttribute('numFmtId'); + var codeAttr = node.getAttribute('formatCode'); + if (idAttr != null && codeAttr != null) { + _customNumFormats[int.parse(idAttr)] = codeAttr; + } + }); + } + document .findAllElements('cellXfs') .first @@ -305,8 +581,11 @@ class XlsxDecoder extends SpreadsheetDecoder { void _parseContent() { var workbook = _archive.findFile('xl/workbook.xml'); - workbook?.decompress(); - var document = XmlDocument.parse(utf8.decode(workbook?.content)); + if (workbook == null) { + throw FormatException('Missing required file: xl/workbook.xml'); + } + workbook.decompress(); + var document = XmlDocument.parse(utf8.decode(workbook.content)); document.findAllElements('sheet').forEach((node) { _parseTable(node); }); @@ -321,9 +600,12 @@ class XlsxDecoder extends SpreadsheetDecoder { final namePath = target.startsWith('/') ? target.substring(1) : 'xl/$target'; var file = _archive.findFile(namePath); - file?.decompress(); + if (file == null) { + throw FormatException('Missing required file: $namePath'); + } + file.decompress(); - var content = XmlDocument.parse(utf8.decode(file?.content)); + var content = XmlDocument.parse(utf8.decode(file.content)); var worksheet = content.findElements('worksheet').first; var sheet = worksheet.findElements('sheetData').first; @@ -414,21 +696,41 @@ class XlsxDecoder extends SpreadsheetDecoder { if (s != null) { var fmtId = _numFormats[int.parse(s)]; // date - if (((fmtId >= 14) && (fmtId <= 17)) || (fmtId == 22)) { + if (_isBuiltInDateFormat(fmtId)) { var delta = num.parse(_parseValue(content)) * 24 * 3600 * 1000; - var date = DateTime(1899, 12, 30); - value = date - .add(Duration(milliseconds: delta.toInt())) - .toIso8601String(); + var date = DateTime(1899, 12, 30) + .add(Duration(milliseconds: delta.toInt())); + value = _formatDateTimeWithCode(date, _dateFormat); // time - } else if (((fmtId >= 18) && (fmtId <= 21)) || - ((fmtId >= 45) && (fmtId <= 47))) { + } else if (_isBuiltInTimeFormat(fmtId)) { var delta = num.parse(_parseValue(content)) * 24 * 3600 * 1000; var date = DateTime(0); date = date.add(Duration(milliseconds: delta.toInt())); value = '${_twoDigits(date.hour)}:${_twoDigits(date.minute)}:${_twoDigits(date.second)}'; // number + } else if (_customNumFormats.containsKey(fmtId)) { + // Custom number format declared in . If the caller + // supplied a non-default [dateFormat], honour it for date + // cells; otherwise fall back to the workbook's own format + // code so the visible output matches Excel. + var workbookCode = _customNumFormats[fmtId]!; + var serial = num.parse(_parseValue(content)); + if (_isDateTimeFormatCode(workbookCode)) { + var delta = serial * 24 * 3600 * 1000; + var date = DateTime(1899, 12, 30) + .add(Duration(milliseconds: delta.toInt())); + var code = _dateFormat == SpreadsheetDecoder.defaultDateFormat + ? workbookCode + : _dateFormat; + value = _formatDateTimeWithCode(date, code); + } else if (_isTimeOnlyFormatCode(workbookCode)) { + var delta = serial * 24 * 3600 * 1000; + var date = DateTime(0).add(Duration(milliseconds: delta.toInt())); + value = _formatDateTimeWithCode(date, workbookCode); + } else { + value = serial; + } } else { value = num.parse(_parseValue(content)); } diff --git a/pubspec.yaml b/pubspec.yaml index 0da5762..9ec9245 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,16 +1,16 @@ name: spreadsheet_decoder description: A lightweight library for parsing and updating spreadsheet documents -version: 2.3.0 +version: 2.3.3 homepage: https://github.com/sestegra/spreadsheet environment: sdk: "^3.0.0" dependencies: - archive: ^3.6.1 - xml: ^6.5.0 + archive: ^4.0.9 + xml: ^7.0.1 dev_dependencies: - path: ^1.9.0 - lints: ^4.0.0 - test: ^1.25.8 + path: ^1.9.1 + lints: ^6.1.0 + test: ^1.31.1 diff --git a/test/.DS_Store b/test/.DS_Store new file mode 100644 index 0000000..5b6463d Binary files /dev/null and b/test/.DS_Store differ diff --git a/test/common.dart b/test/common.dart index e189120..b8aa90f 100644 --- a/test/common.dart +++ b/test/common.dart @@ -92,9 +92,9 @@ var expectedFormat = >{ 'Hello World', 'One line\nTwo lines\nThree lines', 0.124, - '2006-02-01T00:00:00.000', - '2006-02-01T13:37:00.000', - '2006-02-01T13:37:42.000', + '2006-02-01', + '2006-02-01', + '2006-02-01', '13:37:00', '13:37:42', ] @@ -327,7 +327,7 @@ void testXlsx() { expect(decoder.tables.length, expectedNoPhonetics.keys.length); decoder.tables.forEach((name, table) { expect(table.rows, expectedNoPhonetics[name]); - }); + }); }); }); } diff --git a/test/common_html.dart b/test/common_html.dart index 53d9974..0766b5c 100644 --- a/test/common_html.dart +++ b/test/common_html.dart @@ -33,7 +33,9 @@ String readBase64(String filename) { } SpreadsheetDecoder decode(String filename, {bool update = false}) { - return SpreadsheetDecoder.decodeBytes(Base64Decoder().convert(readBase64(filename)), update: update); + return SpreadsheetDecoder.decodeBytes( + Base64Decoder().convert(readBase64(filename)), + update: update); } void save(String file, List data) {} diff --git a/test/common_io.dart b/test/common_io.dart index 1a6a299..78b0368 100644 --- a/test/common_io.dart +++ b/test/common_io.dart @@ -11,8 +11,12 @@ String readBase64(String filename) { return base64Encode(_readBytes(filename)); } -SpreadsheetDecoder decode(String filename, {bool update = false}) { - return SpreadsheetDecoder.decodeBytes(_readBytes(filename), update: update, verify: true); +SpreadsheetDecoder decode(String filename, + {bool update = false, String? dateFormat}) { + return SpreadsheetDecoder.decodeBytes(_readBytes(filename), + update: update, + verify: true, + dateFormat: dateFormat ?? SpreadsheetDecoder.defaultDateFormat); } void save(String file, List data) { diff --git a/test/custom_date_format_test.dart b/test/custom_date_format_test.dart new file mode 100644 index 0000000..6555611 --- /dev/null +++ b/test/custom_date_format_test.dart @@ -0,0 +1,107 @@ +@TestOn('vm') +library spreadsheet_custom_date_format_test; + +import 'package:test/test.dart'; + +import 'common_io.dart'; + +/// Tests that custom date format codes (numFmtId outside the +/// built-in 14-22/45-47 range) are recognised and decoded as formatted +/// date strings instead of leaking through as raw numbers. +void main() { + group('Custom date number format (xlsx):', () { + test('sample_date_format.xlsx — dd/mm/yyyy custom numFmt', () { + var decoder = decode('sample_date_format.xlsx'); + var sheet = decoder.tables.values.first; + + // Cells L2, P2, R2 use the custom numFmtId=59 with + // formatCode="dd/mm/yyyy" and raw serial values 39650, 45782, 46144. + // Column L = index 11, P = 15, R = 17 (0-based). + final row2 = sheet.rows[1]; + + final ddmmyyyy = RegExp(r'^\d{2}/\d{2}/\d{4}$'); + + expect(row2[11], isA(), + reason: 'L2 must decode as a date string, not a number'); + expect(row2[15], isA(), + reason: 'P2 must decode as a date string, not a number'); + expect(row2[17], isA(), + reason: 'R2 must decode as a date string, not a number'); + + expect(row2[11] as String, matches(ddmmyyyy), + reason: 'L2 should match the dd/mm/yyyy format code'); + expect(row2[15] as String, matches(ddmmyyyy)); + expect(row2[17] as String, matches(ddmmyyyy)); + + // The exact day-of-month follows the existing decoder's serial-to-date + // math (1899-12-30 baseline, the same convention used for the + // built-in numFmtId 14-22 path). These are the deterministic outputs + // for the three serials in this fixture. + expect(row2[11], equals('21/07/2008')); + expect(row2[15], equals('05/05/2025')); + expect(row2[17], equals('02/05/2026')); + + // Sanity: none of the date cells should leak through as numbers. + expect(row2[11], isNot(isA())); + expect(row2[15], isNot(isA())); + expect(row2[17], isNot(isA())); + }); + }); + + group('Built-in date numFmt + dateFormat option (xlsx):', () { + test('default dateFormat renders built-in date as yyyy-MM-dd', () { + // new_sample_file_bytco.xlsx contains a column whose cell uses the + // built-in numFmtId=14 (m/d/yyyy). The decoded value must be a + // date-only string in the package default format, not an ISO + // timestamp like 2008-07-21T00:00:00.000. + var decoder = decode('new_sample_file_bytco.xlsx'); + var sheet = decoder.tables.values.first; + + final iso = RegExp(r'^\d{4}-\d{2}-\d{2}$'); + var foundDateCell = false; + for (var row in sheet.rows) { + for (var cell in row) { + if (cell is String && iso.hasMatch(cell)) { + foundDateCell = true; + // No leftover ISO time component. + expect(cell, isNot(contains('T'))); + expect(cell, isNot(contains(':'))); + } + } + } + expect(foundDateCell, isTrue, + reason: 'Expected at least one yyyy-MM-dd date cell in fixture'); + }); + + test('caller-supplied dateFormat dd/MM/yyyy is honoured', () { + var decoder = + decode('new_sample_file_bytco.xlsx', dateFormat: 'dd/MM/yyyy'); + var sheet = decoder.tables.values.first; + + final ddmmyyyy = RegExp(r'^\d{2}/\d{2}/\d{4}$'); + var foundDateCell = false; + for (var row in sheet.rows) { + for (var cell in row) { + if (cell is String && ddmmyyyy.hasMatch(cell)) { + foundDateCell = true; + } + } + } + expect(foundDateCell, isTrue, + reason: 'Expected at least one dd/MM/yyyy date cell'); + }); + + test('caller-supplied dateFormat overrides custom workbook code', () { + // sample_date_format.xlsx normally renders custom-numFmt dates as + // dd/mm/yyyy because that's the workbook's format code. When the + // caller passes a non-default dateFormat the custom branch must + // honour the caller's preference instead. + var decoder = decode('sample_date_format.xlsx', dateFormat: 'yyyy.MM.dd'); + var sheet = decoder.tables.values.first; + final row2 = sheet.rows[1]; + expect(row2[11], equals('2008.07.21')); + expect(row2[15], equals('2025.05.05')); + expect(row2[17], equals('2026.05.02')); + }); + }); +} diff --git a/test/files/new_sample_file_bytco.xlsx b/test/files/new_sample_file_bytco.xlsx new file mode 100644 index 0000000..fc39cd6 Binary files /dev/null and b/test/files/new_sample_file_bytco.xlsx differ diff --git a/test/files/sample_date_format.xlsx b/test/files/sample_date_format.xlsx new file mode 100644 index 0000000..d7a7228 Binary files /dev/null and b/test/files/sample_date_format.xlsx differ diff --git a/test/files/subject-Data Structures and Algorithms-of-FY - MCA-teaching-plan-sample-1784613441518.xlsx b/test/files/subject-Data Structures and Algorithms-of-FY - MCA-teaching-plan-sample-1784613441518.xlsx new file mode 100644 index 0000000..4ccbe94 Binary files /dev/null and b/test/files/subject-Data Structures and Algorithms-of-FY - MCA-teaching-plan-sample-1784613441518.xlsx differ diff --git a/test/files/test_date_excel.xlsx b/test/files/test_date_excel.xlsx new file mode 100644 index 0000000..fdb7be8 Binary files /dev/null and b/test/files/test_date_excel.xlsx differ diff --git a/test/teaching_plan_sample_test.dart b/test/teaching_plan_sample_test.dart new file mode 100644 index 0000000..bcd3ff8 --- /dev/null +++ b/test/teaching_plan_sample_test.dart @@ -0,0 +1,229 @@ +@TestOn('vm') +library spreadsheet_teaching_plan_sample_test; + +import 'package:spreadsheet_decoder/spreadsheet_decoder.dart'; +import 'package:test/test.dart'; + +import 'common_io.dart'; + +const filename = + 'subject-Data Structures and Algorithms-of-FY - MCA-teaching-plan-sample-1784613441518.xlsx'; +const sheetName = 'Teaching Plan Sample'; + +const headers = [ + 'Topic', + 'SubTopic', + 'CourseOutcome', + 'LearningOutcome', + 'Hours', + 'Minutes', + 'PlanningDate', + 'ExecutionDate', +]; + +const topicCol = 0; +const subTopicCol = 1; +const courseOutcomeCol = 2; +const learningOutcomeCol = 3; +const hoursCol = 4; +const minutesCol = 5; +const planningDateCol = 6; +const executionDateCol = 7; + +/// The PlanningDate column mixes two storage forms in the source workbook: +/// +/// * a real date serial styled with `numFmtId="58"` (a built-in +/// locale-specific date format) — these are the rows that used to decode +/// as a bare number such as `46212`; +/// * a shared string holding pre-formatted `d/M/yyyy` text — passed through +/// verbatim by the decoder. +/// +/// Keyed by row index in [SpreadsheetTable.rows] (row 0 is the header). +const serialBackedPlanningDates = { + 1: '2026-07-09', // serial 46212 + 2: '2026-09-09', // serial 46274 + 3: '2026-11-09', // serial 46335 + 13: '2026-05-10', // serial 46152 + 14: '2026-07-10', // serial 46213 + 15: '2026-09-10', // serial 46275 + 16: '2026-12-10', // serial 46366 + 25: '2026-02-11', // serial 46064 + 26: '2026-04-11', // serial 46123 + 35: '2026-02-12', // serial 46065 + 36: '2026-04-12', // serial 46124 + 37: '2026-07-12', // serial 46215 + 38: '2026-09-12', // serial 46277 + 39: '2026-11-12', // serial 46338 +}; + +const textBackedPlanningDates = { + 4: '16/9/2026', + 5: '18/9/2026', + 6: '18/9/2026', + 7: '21/9/2026', + 8: '21/9/2026', + 9: '21/9/2026', + 10: '23/9/2026', + 11: '28/9/2026', + 12: '30/9/2026', + 17: '14/10/2026', + 18: '16/10/2026', + 19: '19/10/2026', + 20: '21/10/2026', + 21: '23/10/2026', + 22: '26/10/2026', + 23: '28/10/2026', + 24: '30/10/2026', + 27: '13/11/2026', + 28: '16/11/2026', + 29: '18/11/2026', + 30: '20/11/2026', + 31: '23/11/2026', + 32: '25/11/2026', + 33: '27/11/2026', + 34: '30/11/2026', + 40: '14/12/2026', + 41: '16/12/2026', +}; + +String _cell(dynamic value, {int width = 34}) { + if (value == null) return 'null'; + var text = '${value.runtimeType}:${value.toString().replaceAll('\n', r'\n')}'; + return text.length > width ? '${text.substring(0, width - 1)}…' : text; +} + +/// Dumps the decoded grid so the parsed output can be eyeballed in the test +/// log. Run with `dart test -r expanded` to see it. +void logTable(SpreadsheetTable table) { + print('sheet "$sheetName": ${table.maxRows} rows x ${table.maxCols} cols'); + for (var i = 0; i < table.rows.length; i++) { + var cells = table.rows[i].map(_cell).join(' | '); + print('[${i.toString().padLeft(2)}] $cells'); + } +} + +void main() { + group('Teaching plan sample excel file:', () { + test('decode xlsx without update', () { + var decoder = decode(filename, update: false); + expect(decoder, isNotNull); + expect(decoder.tables.isNotEmpty, isTrue); + expect(decoder.tables.containsKey(sheetName), isTrue); + + var table = decoder.tables[sheetName]!; + expect(table.maxRows, greaterThan(0)); + expect(table.maxCols, greaterThan(0)); + expect(table.rows.length, equals(table.maxRows)); + }); + + test('decode xlsx with update', () { + var decoder = decode(filename, update: true); + expect(decoder, isNotNull); + expect(decoder.tables.containsKey(sheetName), isTrue); + + var table = decoder.tables[sheetName]!; + expect(table.maxRows, greaterThan(0)); + expect(table.maxCols, greaterThan(0)); + + // Test reading rows and specific cells + for (var row in table.rows) { + expect(row, isA()); + } + }); + + test('logs parsed data', () { + var table = decode(filename).tables[sheetName]!; + logTable(table); + + // Guard the dump above against silently logging an empty grid. + expect(table.rows, isNotEmpty); + expect(table.rows.every((row) => row.length == table.maxCols), isTrue); + }); + + test('grid shape and header row', () { + var table = decode(filename).tables[sheetName]!; + + expect(table.maxRows, equals(42)); + expect(table.maxCols, equals(8)); + expect(table.rows.length, equals(42)); + expect(table.rows.first, equals(headers)); + }); + + test('column value types', () { + var table = decode(filename).tables[sheetName]!; + var body = table.rows.skip(1); + + for (var row in body) { + // Topic is blank on the row covered by a vertical merge. + expect(row[topicCol], anyOf(isNull, isA())); + expect(row[subTopicCol], isA()); + expect(row[courseOutcomeCol], matches(RegExp(r'^CO\d$'))); + expect(row[learningOutcomeCol], equals('UO1')); + expect(row[hoursCol], isA()); + expect(row[minutesCol], isA()); + // Never a raw serial: every planning date renders as text. + expect(row[planningDateCol], isA()); + // ExecutionDate is an empty styled cell (``). + expect(row[executionDateCol], isNull); + } + + expect(body.map((row) => row[topicCol]).where((t) => t == null).length, + equals(1)); + }); + + test('PlanningDate serials decode as dates, not numbers', () { + var table = decode(filename).tables[sheetName]!; + + serialBackedPlanningDates.forEach((index, expected) { + expect(table.rows[index][planningDateCol], equals(expected), + reason: 'row $index (numFmtId 58 date serial)'); + }); + }); + + test('PlanningDate text cells pass through unchanged', () { + var table = decode(filename).tables[sheetName]!; + + textBackedPlanningDates.forEach((index, expected) { + expect(table.rows[index][planningDateCol], equals(expected), + reason: 'row $index (shared string)'); + }); + }); + + test('every PlanningDate row is accounted for', () { + var table = decode(filename).tables[sheetName]!; + var covered = { + ...serialBackedPlanningDates.keys, + ...textBackedPlanningDates.keys, + }; + + expect(covered.length, equals(table.rows.length - 1)); + expect(covered, equals({for (var i = 1; i < table.rows.length; i++) i})); + }); + + test('dateFormat option is applied to PlanningDate serials', () { + var table = decode(filename, dateFormat: 'dd/MM/yyyy').tables[sheetName]!; + + expect(table.rows[1][planningDateCol], equals('09/07/2026')); + expect(table.rows[13][planningDateCol], equals('10/05/2026')); + expect(table.rows[39][planningDateCol], equals('12/11/2026')); + + // Text cells are not reformatted — they were never date serials. + expect(table.rows[4][planningDateCol], equals('16/9/2026')); + }); + + test('update round-trip preserves decoded PlanningDate values', () { + var original = decode(filename, update: true); + var reopened = SpreadsheetDecoder.decodeBytes(original.encode()); + + var before = original.tables[sheetName]!; + var after = reopened.tables[sheetName]!; + + expect(after.maxRows, equals(before.maxRows)); + for (var i = 0; i < before.rows.length; i++) { + expect(after.rows[i][planningDateCol], + equals(before.rows[i][planningDateCol]), + reason: 'row $i'); + } + }); + }); +} diff --git a/tool/archive.dart b/tool/archive.dart index 0b5eac0..a81b83c 100644 --- a/tool/archive.dart +++ b/tool/archive.dart @@ -1,5 +1,4 @@ import 'dart:io'; -import 'dart:typed_data'; import 'package:archive/archive.dart'; import 'package:path/path.dart'; @@ -7,9 +6,9 @@ Archive cloneArchive(Archive archive) { var clone = Archive(); for (var file in archive.files) { if (file.isFile) { - var content = file.content as Uint8List; + var content = file.content; var copy = ArchiveFile(file.name, content.length, content) - ..compress = file.compress; + ..compression = file.compression; clone.addFile(copy); } } @@ -37,7 +36,7 @@ void main(List args) { print(input); var archive = ZipDecoder().decodeBytes(File(input).readAsBytesSync(), verify: true); - var zip = ZipEncoder().encode(cloneArchive(archive)) as List; + var zip = ZipEncoder().encode(cloneArchive(archive)); try { archive = ZipDecoder().decodeBytes(zip, verify: true); var file = File('test/out/example/${basename(input)}')