paramset 302 - kilosort3 - `SpikeSortingError: Spike sorting in docker failed`

[MilagrosMarin]

{'table_name': '_s_i_clustering',
 'key_hash': '98ba0c248f24d9be05ee7eafca91b34c',
 'status': 'error',
 'key': {'organoid_id': 'O09',
  'experiment_start_time': datetime.datetime(2023, 5, 3, 17, 33),
  'insertion_number': 0,
  'start_time': datetime.datetime(2023, 5, 11, 4, 22),
  'end_time': datetime.datetime(2023, 5, 11, 4, 27),
  'paramset_idx': 302},
 'error_message': 'SpikeSortingError: Spike sorting in docker failed with the following error:\nTraceback (most recent call last):\n  File "/mnt/efs/works/org/utah/proj/organoids/outbox/O09-12_raw/202305110422_202305110427/O09/kilosort3_302/kilosort3/in_container_sorter_script.py", line 23, in <module>\n    sorting = run_sorter_local(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/runsorter.py", line 292, in run_sorter_local\n    SorterClass.setup_recording(recording, folder, verbose=verbose)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/basesorter.py", line 242, in setup_recording\n    cls._setup_recording(recording, sorter_output_folder, sorter_params, verbose)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/external/kilosortbase.py", line 155, in _setup_recording\n    write_binary_recording(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/recording_tools.py", line 137, in write_binary_recording\n    executor = ChunkRecordingExecutor(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 433, in __init__\n    self.chunk_size = ensure_chunk_size(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 315, in ensure_chunk_size\n    chunk_size = chunk_duration_to_chunk_size(chunk_duration, recording)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 265, in chunk_duration_to_chunk_size\n    raise ValueError("chunk_duration must be str or float")\nValueError: chunk_duration must be str or float\n',
 'error_stack': 'Traceback (most recent call last):\n  File "/opt/conda/lib/python3.10/site-packages/datajoint/autopopulate.py", line 397, in _populate1\n    make(dict(key), **(make_kwargs or {}))\n  File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 278, in make\n    _run_sorter()\n  File "/opt/conda/lib/python3.10/site-packages/element_interface/utils.py", line 235, in wrapped\n    results = func(*args, **kwargs)\n  File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 264, in _run_sorter\n    si_sorting: si.sorters.BaseSorter = si.sorters.run_sorter(\n  File "/opt/conda/lib/python3.10/site-packages/spikeinterface/sorters/runsorter.py", line 196, in run_sorter\n    return run_sorter_container(\n  File "/opt/conda/lib/python3.10/site-packages/spikeinterface/sorters/runsorter.py", line 673, in run_sorter_container\n    raise SpikeSortingError(f"Spike sorting in {mode} failed with the following error:\\n{run_sorter_output}")\nspikeinterface.sorters.utils.misc.SpikeSortingError: Spike sorting in docker failed with the following error:\nTraceback (most recent call last):\n  File "/mnt/efs/works/org/utah/proj/organoids/outbox/O09-12_raw/202305110422_202305110427/O09/kilosort3_302/kilosort3/in_container_sorter_script.py", line 23, in <module>\n    sorting = run_sorter_local(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/runsorter.py", line 292, in run_sorter_local\n    SorterClass.setup_recording(recording, folder, verbose=verbose)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/basesorter.py", line 242, in setup_recording\n    cls._setup_recording(recording, sorter_output_folder, sorter_params, verbose)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/external/kilosortbase.py", line 155, in _setup_recording\n    write_binary_recording(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/recording_tools.py", line 137, in write_binary_recording\n    executor = ChunkRecordingExecutor(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 433, in __init__\n    self.chunk_size = ensure_chunk_size(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 315, in ensure_chunk_size\n    chunk_size = chunk_duration_to_chunk_size(chunk_duration, recording)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 265, in chunk_duration_to_chunk_size\n    raise ValueError("chunk_duration must be str or float")\nValueError: chunk_duration must be str or float\n\n',
 'timestamp': datetime.datetime(2025, 7, 29, 12, 0, 50)}

{'paramset_idx': 302,
 'clustering_method': 'kilosort3',
 'paramset_desc': 'kilosort3 for organoids,balanced sensitivity, medium whitening range',
 'param_set_hash': UUID('b51514ca-d2ba-a155-7d64-56b489bc2312'),
 'params': {'SI_PREPROCESSING_METHOD': 'organoid_preprocessing',
  'SI_SORTING_PARAMS': {'detect_threshold': 4.5,
   'projection_threshold': [7, 7],
   'preclust_threshold': 6,
   'whiteningRange': 12,
   'car': False,
   'minFR': 0.03,
   'minfr_goodchannels': 0.05,
   'nblocks': 5,
   'sig': 20,
   'freq_min': 300,
   'sigmaMask': 30,
   'lam': 20.0,
   'nPCs': 3,
   'ntbuff': 128,
   'nfilt_factor': 3,
   'do_correction': True,
   'NT': None,
   'AUCsplit': 0.8,
   'wave_length': 61,
   'keep_good_only': False,
   'skip_kilosort_preprocessing': False,
   'scaleproc': None,
   'save_rez_to_mat': False,
   'delete_tmp_files': ('matlab_files',),
   'delete_recording_dat': False,
   'pool_engine': 'process',
   'n_jobs': 1,
   'chunk_duration': 5,
   'progress_bar': True,
   'mp_context': None,
   'max_threads_per_worker': 1},
  'SI_POSTPROCESSING_PARAMS': {'extensions': {'random_spikes': {},
    'waveforms': {},
    'templates': {},
    'noise_levels': {},
    'correlograms': {},
    'isi_histograms': {},
    'principal_components': {'n_components': 5, 'mode': 'by_channel_local'},
    'spike_amplitudes': {},
    'spike_locations': {},
    'template_metrics': {'include_multi_channel_metrics': True},
    'template_similarity': {},
    'unit_locations': {},
    'quality_metrics': {}},
   'job_kwargs': {'n_jobs': -1, 'chunk_duration': '1s'},
   'export_to_phy': True,
   'export_report': True}}}

[MilagrosMarin]

Removed "chunk_duration=5" and "n_jobs=1" from "SI_SORTING_PARAMS" and rerun job.

[MilagrosMarin]

After re-running, this error occurred:
Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/datajoint/autopopulate.py", line 397, in _populate1 make(dict(key), **(make_kwargs or {})) File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 191, in make si_recording = si_recording.channel_slice( AttributeError: 'ConcatenateSegmentRecording' object has no attribute 'channel_slice'

This is fixed in dj-sciops/utah_organoids_element-array-ephys#63

[MilagrosMarin]

Error:
Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/datajoint/autopopulate.py", line 397, in _populate1 make(dict(key), **(make_kwargs or {})) File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 212, in make si_recording = si_preproc_func(si_recording) File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_preprocessing.py", line 42, in organoid_preprocessing recording = si.preprocessing.bandpass_filter( File "/opt/conda/lib/python3.10/site-packages/spikeinterface/core/core_tools.py", line 35, in source_class_or_dict_of_sources_classes return source_class(*args, **kwargs) File "/opt/conda/lib/python3.10/site-packages/spikeinterface/preprocessing/filter.py", line 233, in __init__ FilterRecording.__init__( File "/opt/conda/lib/python3.10/site-packages/spikeinterface/preprocessing/filter.py", line 105, in __init__ dtype = fix_dtype(recording, dtype) File "/opt/conda/lib/python3.10/site-packages/spikeinterface/preprocessing/filter.py", line 425, in fix_dtype raise ValueError( ValueError: Unsigned types are not supported, since they don't interact well with various preprocessing steps. You can use spikeinterface.preprocessing.unsigned_to_signed to convert the recording to a signed type.For more information, please see https://spikeinterface.readthedocs.io/en/stable/how_to/unsigned_to_signed.html

Fixed here: dj-sciops/utah_organoids_element-array-ephys#64

[MilagrosMarin]

Error related to the usage of drift correction.
Solution: The parameter for enabling drift correction should be disabled because it is not necessary for this type of recording. This will be modified in the params for paramset_idx=302 and the task will be re-run.