paramset 301 - kilosort3 - `SpikeSortingError: Spike sorting in docker failed`

[MilagrosMarin]

{'table_name': '_s_i_clustering',
 'key_hash': '461c5607f3776ae3b208328bd623b9a4',
 'status': 'error',
 'key': {'organoid_id': 'O09',
  'experiment_start_time': datetime.datetime(2023, 5, 3, 17, 33),
  'insertion_number': 0,
  'start_time': datetime.datetime(2023, 5, 11, 4, 22),
  'end_time': datetime.datetime(2023, 5, 11, 4, 27),
  'paramset_idx': 301},
 'error_message': 'SpikeSortingError: Spike sorting in docker failed with the following error:\nTraceback (most recent call last):\n  File "/mnt/efs/works/org/utah/proj/organoids/outbox/O09-12_raw/202305110422_202305110427/O09/kilosort3_301/kilosort3/in_container_sorter_script.py", line 23, in <module>\n    sorting = run_sorter_local(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/runsorter.py", line 292, in run_sorter_local\n    SorterClass.setup_recording(recording, folder, verbose=verbose)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/basesorter.py", line 242, in setup_recording\n    cls._setup_recording(recording, sorter_output_folder, sorter_params, verbose)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/external/kilosortbase.py", line 155, in _setup_recording\n    write_binary_recording(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/recording_tools.py", line 137, in write_binary_recording\n    executor = ChunkRecordingExecutor(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 433, in __init__\n    self.chunk_size = ensure_chunk_size(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 315, in ensure_chunk_size\n    chunk_size = chunk_duration_to_chunk_size(chunk_duration, recording)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 265, in chunk_duration_to_chunk_size\n    raise ValueError("chunk_duration must be str or float")\nValueError: chunk_duration must be str or float\n',
 'error_stack': 'Traceback (most recent call last):\n  File "/opt/conda/lib/python3.10/site-packages/datajoint/autopopulate.py", line 397, in _populate1\n    make(dict(key), **(make_kwargs or {}))\n  File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 278, in make\n    _run_sorter()\n  File "/opt/conda/lib/python3.10/site-packages/element_interface/utils.py", line 235, in wrapped\n    results = func(*args, **kwargs)\n  File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 264, in _run_sorter\n    si_sorting: si.sorters.BaseSorter = si.sorters.run_sorter(\n  File "/opt/conda/lib/python3.10/site-packages/spikeinterface/sorters/runsorter.py", line 196, in run_sorter\n    return run_sorter_container(\n  File "/opt/conda/lib/python3.10/site-packages/spikeinterface/sorters/runsorter.py", line 673, in run_sorter_container\n    raise SpikeSortingError(f"Spike sorting in {mode} failed with the following error:\\n{run_sorter_output}")\nspikeinterface.sorters.utils.misc.SpikeSortingError: Spike sorting in docker failed with the following error:\nTraceback (most recent call last):\n  File "/mnt/efs/works/org/utah/proj/organoids/outbox/O09-12_raw/202305110422_202305110427/O09/kilosort3_301/kilosort3/in_container_sorter_script.py", line 23, in <module>\n    sorting = run_sorter_local(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/runsorter.py", line 292, in run_sorter_local\n    SorterClass.setup_recording(recording, folder, verbose=verbose)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/basesorter.py", line 242, in setup_recording\n    cls._setup_recording(recording, sorter_output_folder, sorter_params, verbose)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/external/kilosortbase.py", line 155, in _setup_recording\n    write_binary_recording(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/recording_tools.py", line 137, in write_binary_recording\n    executor = ChunkRecordingExecutor(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 433, in __init__\n    self.chunk_size = ensure_chunk_size(\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 315, in ensure_chunk_size\n    chunk_size = chunk_duration_to_chunk_size(chunk_duration, recording)\n  File "/root/.local/lib/python3.9/site-packages/spikeinterface/core/job_tools.py", line 265, in chunk_duration_to_chunk_size\n    raise ValueError("chunk_duration must be str or float")\nValueError: chunk_duration must be str or float\n\n',
 'timestamp': datetime.datetime(2025, 7, 29, 12, 0, 3)}

{'paramset_idx': 301,
 'clustering_method': 'kilosort3',
 'paramset_desc': 'kilosort3 optimized for 100-um probe, sparse units, minimal channel spread',
 'param_set_hash': UUID('90ed1a20-ca95-1c59-2608-67248716dca4'),
 'params': {'SI_PREPROCESSING_METHOD': 'organoid_preprocessing',
  'SI_SORTING_PARAMS': {'detect_threshold': 5,
   'projection_threshold': [8, 4],
   'preclust_threshold': 6,
   'whiteningRange': 6,
   'car': False,
   'minFR': 0.01,
   'minfr_goodchannels': 0.02,
   'nblocks': 5,
   'sig': 20,
   'freq_min': 300,
   'sigmaMask': 30,
   'lam': 20.0,
   'nPCs': 3,
   'ntbuff': 128,
   'nfilt_factor': 2,
   'do_correction': True,
   'NT': None,
   'AUCsplit': 0.8,
   'wave_length': 61,
   'keep_good_only': False,
   'skip_kilosort_preprocessing': False,
   'scaleproc': None,
   'save_rez_to_mat': False,
   'delete_tmp_files': ('matlab_files',),
   'delete_recording_dat': False,
   'pool_engine': 'process',
   'n_jobs': 1,
   'chunk_duration': 5,
   'progress_bar': True,
   'mp_context': None,
   'max_threads_per_worker': 1},
  'SI_POSTPROCESSING_PARAMS': {'extensions': {'random_spikes': {},
    'waveforms': {},
    'templates': {},
    'noise_levels': {},
    'correlograms': {},
    'isi_histograms': {},
    'principal_components': {'n_components': 5, 'mode': 'by_channel_local'},
    'spike_amplitudes': {},
    'spike_locations': {},
    'template_metrics': {'include_multi_channel_metrics': True},
    'template_similarity': {},
    'unit_locations': {},
    'quality_metrics': {}},
   'job_kwargs': {'n_jobs': -1, 'chunk_duration': '1s'},
   'export_to_phy': True,
   'export_report': True}}}

[MilagrosMarin]

Removed "chunk_duration=5" and "n_jobs=1" from "SI_SORTING_PARAMS" and rerun job.

[MilagrosMarin]

After re-running, this error occurred:
Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/datajoint/autopopulate.py", line 397, in _populate1 make(dict(key), **(make_kwargs or {})) File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 191, in make si_recording = si_recording.channel_slice( AttributeError: 'ConcatenateSegmentRecording' object has no attribute 'channel_slice'

This is fixed in dj-sciops/utah_organoids_element-array-ephys#63

[MilagrosMarin]

Next error:
Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/datajoint/autopopulate.py", line 397, in _populate1 make(dict(key), **(make_kwargs or {})) File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 212, in make si_recording = si_preproc_func(si_recording) File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_preprocessing.py", line 42, in organoid_preprocessing recording = si.preprocessing.bandpass_filter( File "/opt/conda/lib/python3.10/site-packages/spikeinterface/core/core_tools.py", line 35, in source_class_or_dict_of_sources_classes return source_class(*args, **kwargs) File "/opt/conda/lib/python3.10/site-packages/spikeinterface/preprocessing/filter.py", line 233, in __init__ FilterRecording.__init__( File "/opt/conda/lib/python3.10/site-packages/spikeinterface/preprocessing/filter.py", line 105, in __init__ dtype = fix_dtype(recording, dtype) File "/opt/conda/lib/python3.10/site-packages/spikeinterface/preprocessing/filter.py", line 425, in fix_dtype raise ValueError( ValueError: Unsigned types are not supported, since they don't interact well with various preprocessing steps. You can use spikeinterface.preprocessing.unsigned_to_signed to convert the recording to a signed type.For more information, please see https://spikeinterface.readthedocs.io/en/stable/how_to/unsigned_to_signed.html

Fixed here: dj-sciops/utah_organoids_element-array-ephys#64

[MilagrosMarin]

New error:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.10/site-packages/datajoint/autopopulate.py", line 397, in _populate1
    make(dict(key), **(make_kwargs or {}))
  File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 278, in make
    _run_sorter()
  File "/opt/conda/lib/python3.10/site-packages/element_interface/utils.py", line 235, in wrapped
    results = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/element_array_ephys/spike_sorting/si_spike_sorting.py", line 264, in _run_sorter
    si_sorting: si.sorters.BaseSorter = si.sorters.run_sorter(
  File "/opt/conda/lib/python3.10/site-packages/spikeinterface/sorters/runsorter.py", line 204, in run_sorter
    return run_sorter_container(
  File "/opt/conda/lib/python3.10/site-packages/spikeinterface/sorters/runsorter.py", line 681, in run_sorter_container
    raise SpikeSortingError(f"Spike sorting in {mode} failed with the following error:\n{run_sorter_output}")
spikeinterface.sorters.utils.misc.SpikeSortingError: Spike sorting in docker failed with the following error:
write_binary_recording (no parallelization): 100%|██████████| 300/300 [00:21<00:00, 14.24it/s]
RUNNING SHELL SCRIPT: /mnt/efs/works/org/utah/proj/organoids/outbox/O09-12_raw/202305110422_202305110427/O09/kilosort3_301/kilosort3/spike_sorting/sorter_output/run_kilosort3.sh
Warning: X does not support locale C.UTF-8

Time   0s. Computing whitening matrix.. 

Getting channel whitening matrix... 

Channel-whitening matrix computed. 

Time   3s. Loading raw data and applying filters... 

Time  11s. Finished preprocessing 92 batches. 

Drift correction ENABLED

vertical pitch size is 100 

horizontal pitch size is 5 

     0



   622



0.08 sec, 1 batches, 5 spikes 

2.28 sec, 92 batches, 30859 spikes 

time 19.51, Shifted up/down 92 batches. 

0.03 sec, 1 batches, 0 spikes 

----------------------------------------Error using indexing

An unexpected error occurred trying to launch a kernel. The CUDA error was:

invalid configuration argument
Error running kilosort3
Traceback (most recent call last):
  File "/mnt/efs/works/org/utah/proj/organoids/outbox/O09-12_raw/202305110422_202305110427/O09/kilosort3_301/kilosort3/in_container_sorter_script.py", line 23, in <module>
    sorting = run_sorter_local(
  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/runsorter.py", line 302, in run_sorter_local
    SorterClass.run_from_folder(folder, raise_error, verbose)
  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/basesorter.py", line 310, in run_from_folder
    raise SpikeSortingError(
spikeinterface.sorters.utils.misc.SpikeSortingError: Spike sorting error trace:
Traceback (most recent call last):
  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/basesorter.py", line 270, in run_from_folder
    SorterClass._run_from_folder(sorter_output_folder, sorter_params, verbose)
  File "/root/.local/lib/python3.9/site-packages/spikeinterface/sorters/external/kilosortbase.py", line 217, in _run_from_folder
    raise Exception(f"{cls.sorter_name} returned a non-zero exit code")
Exception: kilosort3 returned a non-zero exit code

Spike sorting failed. You can inspect the runtime trace in /mnt/efs/works/org/utah/proj/organoids/outbox/O09-12_raw/202305110422_202305110427/O09/kilosort3_301/kilosort3/spike_sorting/spikeinterface_log.json.

Solution: the parameter for enabling drift correction should be disabled because it is not necessary for this type of recording. This will be modified in the params for paramset_idx=301 and the task will be re-run.