spikeinterface
spikeinterface copied to clipboard
SpikeInterface
OSError: too many open files
Hi,
I tried to extract waveform from one sorted session, and it keeps crashing:
> 20231201_az_WDAN07 imec0.ap waveforms not extracted, extracting now. [210/738]
extract waveforms shared_memory multi buffer: 100%|██████████████████████████████| 3645/3645 [00:00<00:00, 8958.39it/s]
extract waveforms shared_memory multi buffer: 100%|█████████████████████████████| 3645/3645 [00:00<00:00, 10124.65it/s]
extract waveforms shared_memory multi buffer: 100%|█████████████████████████████| 3645/3645 [00:00<00:00, 10273.34it/s]
extract waveforms shared_memory multi buffer: 100%|██████████████████████████████| 3645/3645 [00:00<00:00, 8616.66it/s]
extract waveforms shared_memory multi buffer: 100%|█████████████████████████████| 3645/3645 [00:00<00:00, 10356.94it/s]
extract waveforms shared_memory multi buffer: 100%|█████████████████████████████| 3645/3645 [00:00<00:00, 10295.60it/s]
---------------------------------------------------------------------------
AssertionError Traceback (most recent call last)
File /opt/neuropixels/pixels/pixels/behaviours/base.py:916, in Behaviour.sort_spikes(self, CatGT_app, old)
915 try:
--> 916 waveforms = si.WaveformExtractor.load_from_folder(
917 folder=cache,
918 sorting=ks3_output,
919 )
920 print(f"> {self.name} {stream_id} waveforms extracted, now it is loaded.\n")
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py:141, in WaveformExtractor.load_from_folder(cls, folder, with_recording, sorting)
140 folder = Path(folder)
--> 141 assert folder.is_dir(), f"This waveform folder does not exists {folder}"
143 if not with_recording:
144 # load
AssertionError: This waveform folder does not exists /home/amz/interim/npx/20231201_az_WDAN07/cache_0
During handling of the above exception, another exception occurred:
OSError Traceback (most recent call last)
File ~/Documents/git/analysis/analysis/preprocessing/preprocessing.py:31, in <module>
17 """
18 # step 1: do CatGT
19 az_utils.run_catgt(
(...)
27
28 """
29 # step 2: spikesorting
30 #exp.sort_spikes()
---> 31 exp.sort_spikes(CatGT_app=catgt_dir)
32 stop = timeit.default_timer()
33 print(f"> Spike sorting of {exp} took {(stop - start)/60} mins.")
File /opt/neuropixels/pixels/pixels/experiment.py:126, in Experiment.sort_spikes(self, CatGT_app)
123 for i, session in enumerate(self.sessions)
124 print(">>>>> Sorting spikes for session {} ({} / {})"
125 .format(session.name, i + 1, len(self.sessions)))
--> 126 session.sort_spikes(CatGT_app=CatGT_app)
File /opt/neuropixels/pixels/pixels/behaviours/base.py:925, in Behaviour.sort_spikes(self, CatGT_app, old)
922 print(f"> {self.name} {stream_id} waveforms not extracted, extracting now.\n")
923 #if ks3_output.count_total_num_spikes()
924 # extract waveforms
--> 925 waveforms = si.extract_waveforms(
926 recording=concat_rec, #recording=test, # for testing
927 sorting=ks3_output,
928 folder=cache,
929 load_if_exists=True, # load extracted if available
930 #load_if_exists=False, # re-calculate everytime
931 max_spikes_per_unit=500, # None will extract all waveforms
932 ms_before=2.0, # time before trough
933 ms_after=3.0, # time after trough
934 overwrite=False,
935 #overwrite=True,
936 **job_kwargs,
937 )
939 """
940 # TODO: remove redundant units by keeping minimum shift, highest_amplitude, or
941 # max_spikes
(...)
946 )
947 """
948 # export to phy, with pc feature calculated.
949 # copy recording.dat to output so that individual waveforms can be
950 # seen in waveformview.
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py:1643, in extract_waveforms(recording, sorting, folder, mode, precompute_template, ms_before, ms_after, max_spikes_per_unit, overwrite, return
_scaled, dtype, sparse, sparsity, sparsity_temp_folder, num_spikes_for_sparsity, unit_batch_size, allow_unfiltered, use_relative_path, seed, load_if_exists, **kwargs)
1627 we = WaveformExtractor.create(
1628 recording,
1629 sorting,
(...)
1634 sparsity=sparsity,
1635 )
1636 we.set_params(
1637 ms_before=ms_before,
1638 ms_after=ms_after,
(...)
1641 return_scaled=return_scaled,
1642 )
-> 1643 we.run_extract_waveforms(seed=seed, **job_kwargs)
1645 if precompute_template is not None:
1646 we.precompute_templates(modes=precompute_template)
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py:1402, in WaveformExtractor.run_extract_waveforms(self, seed, **job_kwargs)
1399 else:
1400 sparsity_mask = self.sparsity.mask
-> 1402 wfs_arrays = extract_waveforms_to_buffers(
1403 self.recording,
1404 spikes,
1405 unit_ids,
1406 nbefore,
1407 nafter,
1408 mode=mode,
1409 return_scaled=return_scaled,
1410 folder=wf_folder,
1411 dtype=p["dtype"],
1412 sparsity_mask=sparsity_mask,
1413 copy=copy,
1414 **job_kwargs,
1415 )
1416 if self.folder is None:
1417 self._memory_objects["wfs_arrays"] = wfs_arrays
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_tools.py:92, in extract_waveforms_to_buffers(recording, spikes, unit_ids, nbefore, nafter, mode, return_scaled, folder, dtype, sparsity_mask, copy, **job_
kwargs)
89 dtype = "float32"
90 dtype = np.dtype(dtype)
---> 92 waveforms_by_units, arrays_info = allocate_waveforms_buffers(
93 recording, spikes, unit_ids, nbefore, nafter, mode=mode, folder=folder, dtype=dtype, sparsity_mask=sparsity_mask
94 )
96 distribute_waveforms_to_buffers(
97 recording,
98 spikes,
(...)
106 **job_kwargs,
107 )
109 if mode == "memmap":
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_tools.py:188, in allocate_waveforms_buffers(recording, spikes, unit_ids, nbefore, nafter, mode, folder, dtype, sparsity_mask)
186 if mode == "memmap":
187 filename = str(folder / f"waveforms_{unit_id}.npy")
--> 188 arr = np.lib.format.open_memmap(filename, mode="w+", dtype=dtype, shape=shape)
189 waveforms_by_units[unit_id] = arr
190 arrays_info[unit_id] = filename
File ~/.conda/envs/pixels/lib/python3.10/site-packages/numpy/lib/format.py:885, in open_memmap(filename, mode, dtype, shape, fortran_order, version)
882 if mode == 'w+':
883 mode = 'r+'
--> 885 marray = numpy.memmap(filename, dtype=dtype, shape=shape, order=order,
886 mode=mode, offset=offset)
888 return marray
File ~/.conda/envs/pixels/lib/python3.10/site-packages/numpy/core/memmap.py:267, in memmap.__new__(subtype, filename, dtype, mode, offset, shape, order)
265 bytes -= start
266 array_offset = offset - start
--> 267 mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
269 self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm,
270 offset=array_offset, order=order)
271 self._mmap = mm
OSError: [Errno 24] Too many open files
Traceback (most recent call last):
File "/opt/neuropixels/pixels/pixels/behaviours/base.py", line 916, in sort_spikes
waveforms = si.WaveformExtractor.load_from_folder(
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py", line 141, in load_from_folder
assert folder.is_dir(), f"This waveform folder does not exists {folder}"
AssertionError: This waveform folder does not exists /home/amz/interim/npx/20231201_az_WDAN07/cache_0
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/shellapp.py", line 435, in _run_cmd_line_code
self._exec_file(fname, shell_futures=True)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/shellapp.py", line 360, in _exec_file
self.shell.safe_execfile(full_filename,
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 2738, in safe_execfile
py3compat.execfile(
File "/home/amz/.local/lib/python3.10/site-packages/IPython/utils/py3compat.py", line 55, in execfile
exec(compiler(f.read(), fname, "exec"), glob, loc)
File "/home/amz/Documents/git/analysis/analysis/preprocessing/preprocessing.py", line 31, in <module>
exp.sort_spikes(CatGT_app=catgt_dir)
File "/opt/neuropixels/pixels/pixels/experiment.py", line 126, in sort_spikes
session.sort_spikes(CatGT_app=CatGT_app)
File "/opt/neuropixels/pixels/pixels/behaviours/base.py", line 925, in sort_spikes
waveforms = si.extract_waveforms(
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py", line 1643, in extract_waveforms
we.run_extract_waveforms(seed=seed, **job_kwargs)
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py", line 1402, in run_extract_waveforms
wfs_arrays = extract_waveforms_to_buffers(
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_tools.py", line 92, in extract_waveforms_to_buffers
waveforms_by_units, arrays_info = allocate_waveforms_buffers(
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_tools.py", line 188, in allocate_waveforms_buffers
arr = np.lib.format.open_memmap(filename, mode="w+", dtype=dtype, shape=shape)
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/numpy/lib/format.py", line 885, in open_memmap
marray = numpy.memmap(filename, dtype=dtype, shape=shape, order=order,
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/numpy/core/memmap.py", line 267, in __new__
mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
OSError: [Errno 24] Too many open files
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/amz/.conda/envs/pixels/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/amz/.local/lib/python3.10/site-packages/IPython/__init__.py", line 124, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/amz/.local/lib/python3.10/site-packages/traitlets/config/application.py", line 975, in launch_instance
app.initialize(argv)
File "/home/amz/.local/lib/python3.10/site-packages/traitlets/config/application.py", line 110, in inner
return method(app, *args, **kwargs)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/terminal/ipapp.py", line 285, in initialize
self.init_code()
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/shellapp.py", line 310, in init_code
self._run_cmd_line_code()
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/shellapp.py", line 437, in _run_cmd_line_code
self.shell.showtraceback(tb_offset=4)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 2007, in showtraceback
self.debugger(force=True)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 1107, in debugger
self.InteractiveTB.debugger(force=True)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 924, in debugger
self.pdb = self.debugger_cls()
File "/home/amz/.local/lib/python3.10/site-packages/IPython/terminal/debugger.py", line 29, in __init__
self.pt_init(pt_session_options)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/terminal/debugger.py", line 90, in pt_init
self.pt_loop = asyncio.new_event_loop()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/events.py", line 782, in new_event_loop
return get_event_loop_policy().new_event_loop()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/events.py", line 673, in new_event_loop
return self._loop_factory()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/unix_events.py", line 64, in __init__
super().__init__(selector)
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/selector_events.py", line 56, in __init__
self._make_self_pipe()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/selector_events.py", line 103, in _make_self_pipe
self._ssock, self._csock = socket.socketpair()
File "/home/amz/.conda/envs/pixels/lib/python3.10/socket.py", line 607, in socketpair
a, b = _socket.socketpair(family, type, proto)
OSError: [Errno 24] Too many open files
If you suspect this is an IPython 8.4.0 bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]
You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.
Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True
Exception ignored in: <function BaseEventLoop.__del__ at 0x7f127c371750>
Traceback (most recent call last):
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/base_events.py", line 690, in __del__
self.close()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/unix_events.py", line 68, in close
super().close()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/selector_events.py", line 87, in close
self._close_self_pipe()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/selector_events.py", line 94, in _close_self_pipe
self._remove_reader(self._ssock.fileno())
AttributeError: '_UnixSelectorEventLoop' object has no attribute '_ssock'
Could you please advise?
Thanks very much!
@a-zmz, this is a bug that has been seen before. You can try the temporary solution given in: #919
@samuelgarcia with the refactor coming do you still have plans to fix this or just wait until the refactor is done?
Hi @a-zmz and Zach.
Yes at the moment ulimit -n 8192 is the way to go on linux.
And yes, the futur SortingResult will have a new etension for extracting wevform that will be mono buffer and so will avoid this error.
Since the waveformextractor is now old api, moving forward the best solution for this issue is to upgrade to the SortingAnalzyer which has the monobuffer.