Merge branch 'master' into add_buffer_api_like_behavior_for_blackrock

Author: h-mayorquin

neo/rawio/axonrawio.py

@@ -46,6 +46,7 @@
 
 """
 
+import os
 import struct
 import datetime
 from io import open, BufferedReader
@@ -160,9 +161,17 @@ def _parse_header(self):
         self._t_starts = {}
         self._buffer_descriptions = {0: {}}
         self._stream_buffer_slice = {stream_id: None}
+        # Offsets and segment sizes come from header fields that can be corrupt
+        # (truncated file, header surgery). Do the arithmetic in Python ints so it
+        # cannot silently overflow as numpy int32 would, and validate the implied
+        # data extent against the file on disk so a bad header raises a clear error
+        # rather than returning garbage or negative signal sizes.
+        head_offset = int(head_offset)
+        file_size = os.path.getsize(self.filename)
+
         pos = 0
         for seg_index in range(nb_segment):
-            length = episode_array[seg_index]["len"]
+            length = int(episode_array[seg_index]["len"])
 
             if version < 2.0:
                 fSynchTimeUnit = info["fSynchTimeUnit"]
@@ -172,6 +181,11 @@ def _parse_header(self):
             if (fSynchTimeUnit != 0) and (mode == 1):
                 length /= fSynchTimeUnit
 
+            if length < 0:
+                raise NeoReadWriteError(
+                    f"Negative segment size ({length}) parsed from {self.filename}; the file header is corrupt."
+                )
+
             self._buffer_descriptions[0][seg_index] = {}
             self._buffer_descriptions[0][seg_index][buffer_id] = {
                 "type": "raw",
@@ -190,6 +204,13 @@ def _parse_header(self):
                 t_start = t_start * fSynchTimeUnit * 1e-6
             self._t_starts[seg_index] = t_start
 
+        implied_data_end = head_offset + pos * sig_dtype.itemsize
+        if implied_data_end > file_size:
+            raise NeoReadWriteError(
+                f"ABF header implies {pos} samples ending at byte {implied_data_end}, which exceeds the "
+                f"file size of {file_size} bytes for {self.filename}; the file header is corrupt or the file is truncated."
+            )
+
         # Create channel header. By default assume channels 0..nbchannel-1 were sampled in order,
         # which is always the case for version >= 2.0. For version < 2.0 the channel ids come from
         # nADCSamplingSeq (the ADC sampling sequence) and are also used to index the per-channel
@@ -490,8 +511,13 @@ def parse_axon_soup(filename):
 
     Returns
     -------
-    dict or None
-        Header dictionary with file metadata, or None if file signature is invalid
+    dict
+        Header dictionary with file metadata.
+
+    Raises
+    ------
+    NeoReadWriteError
+        If the file does not start with a valid ABF signature (b"ABF " or b"ABF2").
     """
     with open(filename, "rb") as fid:
         f = StructFile(fid)
@@ -502,7 +528,14 @@ def parse_axon_soup(filename):
         elif signature == b"ABF2":
             return _parse_abf_v2(f, headerDescriptionV2)
         else:
-            return None
+            # The first 4 bytes are the ABF magic; anything else means the file is not an ABF
+            # file, is corrupt, or is an unsupported variant. Raise here rather than returning
+            # None so the caller gets a clear error instead of a downstream NoneType access.
+            raise NeoReadWriteError(
+                f"Could not parse {filename} as an ABF file: expected the header to start with "
+                f"signature b'ABF ' or b'ABF2', but found {signature}. The file is not an ABF "
+                f"file, is corrupt, or is an unsupported variant."
+            )
 
 
 def _parse_abf_v1(f, header_description):

neo/resources/neuropixels_probe_features.json

@@ -1519,6 +1519,55 @@
             "tip_length_um": "209",
             "total_electrodes": "960"
         },
+        "NP1400": {
+            "adc_bit_depth": "10",
+            "adc_range_vpp": "1.2",
+            "ap_band_list_hz": "300,10000",
+            "ap_gain_list": "50,125,250,500,1000,1500,2000,3000",
+            "ap_sample_frequency_hz": "30000",
+            "banks_per_shank": "2.50",
+            "blue_emitter0_dist_from_elec0_um": "74",
+            "blue_emitter_count": "14",
+            "channel_mapping_type": "simple bank",
+            "channels_per_bank": "384",
+            "cols_per_shank": "2",
+            "databus_decoder_type": "1.0",
+            "datasheet": "N/A",
+            "description": "Opto-II pre-Alpha",
+            "electrode_layout_type": "linear",
+            "electrode_pitch_horz_um": "48",
+            "electrode_pitch_vert_um": "20",
+            "electrode_select_api_type": "selectElectrode",
+            "electrode_size_horz_direction_um": "12",
+            "electrode_size_vert_direction_um": "12",
+            "electrodes_per_shank": "960",
+            "emitter_pitch_um": "100",
+            "even_row_horz_offset_left_edge_to_leftmost_electrode_center_um": "11",
+            "has_adc_calibration_file": "Y",
+            "has_ap_bandpass": "Y",
+            "has_on_chip_gnd_ref": "N",
+            "has_tip_ref": "Y",
+            "imro_table_format_type": "imro_np1000",
+            "is_commercial": "N",
+            "lf_band_list_hz": "0.5,500",
+            "lf_gain_list": "50,125,250,500,1000,1500,2000,3000",
+            "lf_sample_frequency_hz": "2500",
+            "mux_table_format_type": "mux_np1000",
+            "num_readout_channels": "384",
+            "num_shanks": "1",
+            "odd_row_horz_offset_left_edge_to_leftmost_electrode_center_um": "11",
+            "on_shank_ref_chan": "191",
+            "part_number": "NP1400",
+            "red_emitter0_dist_from_elec0_um": "68",
+            "red_emitter_count": "14",
+            "rows_per_shank": "480",
+            "shank_pitch_um": "0",
+            "shank_thickness_um": "35",
+            "shank_tip_to_base_um": "10000",
+            "shank_width_um": "70",
+            "tip_length_um": "209",
+            "total_electrodes": "960"
+        },
         "NP2000": {
             "adc_bit_depth": "14",
             "adc_range_vpp": "1",
@@ -2009,6 +2058,104 @@
             "tip_length_um": "206",
             "total_electrodes": "5120"
         },
+        "NP2022": {
+            "adc_bit_depth": "12",
+            "adc_range_vpp": "1.24",
+            "ap_band_list_hz": "0.5,10000",
+            "ap_gain_list": "100",
+            "ap_sample_frequency_hz": "30000",
+            "banks_per_shank": "3.33",
+            "blue_emitter0_dist_from_elec0_um": "0",
+            "blue_emitter_count": "0",
+            "channel_mapping_type": "2.0SS scrambled",
+            "channels_per_bank": "384",
+            "cols_per_shank": "2",
+            "databus_decoder_type": "2.0 Ph2C",
+            "datasheet": "NP2022 NEUROPIXELS 2.0 QB SAPIENS DATASHEET",
+            "description": "Neuropixels 2.0 multi shank probe quad base NHP with cap (Ph 2C), cabling, sharpened, sterile packaging",
+            "electrode_layout_type": "staggered",
+            "electrode_pitch_horz_um": "20",
+            "electrode_pitch_vert_um": "15",
+            "electrode_select_api_type": "selectElectrode",
+            "electrode_size_horz_direction_um": "12",
+            "electrode_size_vert_direction_um": "12",
+            "electrodes_per_shank": "1280",
+            "emitter_pitch_um": "0",
+            "even_row_horz_offset_left_edge_to_leftmost_electrode_center_um": "27",
+            "has_adc_calibration_file": "N",
+            "has_ap_bandpass": "N",
+            "has_on_chip_gnd_ref": "Y",
+            "has_tip_ref": "Y",
+            "imro_table_format_type": "imro_np2020",
+            "is_commercial": "Y",
+            "lf_band_list_hz": "0",
+            "lf_gain_list": "1",
+            "lf_sample_frequency_hz": "0",
+            "mux_table_format_type": "mux_np2020",
+            "num_readout_channels": "1536",
+            "num_shanks": "4",
+            "odd_row_horz_offset_left_edge_to_leftmost_electrode_center_um": "11",
+            "on_shank_ref_chan": "-1",
+            "part_number": "NP2022",
+            "red_emitter0_dist_from_elec0_um": "0",
+            "red_emitter_count": "0",
+            "rows_per_shank": "640",
+            "shank_pitch_um": "2141.45",
+            "shank_thickness_um": "122",
+            "shank_tip_to_base_um": "10000",
+            "shank_width_um": "70",
+            "tip_length_um": "206",
+            "total_electrodes": "5120"
+        },
+        "NP2300": {
+            "adc_bit_depth": "12",
+            "adc_range_vpp": "1.24",
+            "ap_band_list_hz": "0.5,10000",
+            "ap_gain_list": "100",
+            "ap_sample_frequency_hz": "30000",
+            "banks_per_shank": "3.33",
+            "blue_emitter0_dist_from_elec0_um": "56",
+            "blue_emitter_count": "44",
+            "channel_mapping_type": "2.0SS scrambled",
+            "channels_per_bank": "384",
+            "cols_per_shank": "2",
+            "databus_decoder_type": "2.0 Ph2B",
+            "datasheet": "",
+            "description": "Opto-II Alpha",
+            "electrode_layout_type": "linear",
+            "electrode_pitch_horz_um": "48",
+            "electrode_pitch_vert_um": "15",
+            "electrode_select_api_type": "selectElectrodeMask",
+            "electrode_size_horz_direction_um": "12",
+            "electrode_size_vert_direction_um": "12",
+            "electrodes_per_shank": "1280",
+            "emitter_pitch_um": "100",
+            "even_row_horz_offset_left_edge_to_leftmost_electrode_center_um": "11",
+            "has_adc_calibration_file": "N",
+            "has_ap_bandpass": "N",
+            "has_on_chip_gnd_ref": "Y",
+            "has_tip_ref": "Y",
+            "imro_table_format_type": "imro_np2003",
+            "is_commercial": "N",
+            "lf_band_list_hz": "0",
+            "lf_gain_list": "1",
+            "lf_sample_frequency_hz": "0",
+            "mux_table_format_type": "mux_np2000",
+            "num_readout_channels": "384",
+            "num_shanks": "1",
+            "odd_row_horz_offset_left_edge_to_leftmost_electrode_center_um": "11",
+            "on_shank_ref_chan": "-1",
+            "part_number": "NP2300",
+            "red_emitter0_dist_from_elec0_um": "56",
+            "red_emitter_count": "44",
+            "rows_per_shank": "640",
+            "shank_pitch_um": "0",
+            "shank_thickness_um": "30",
+            "shank_tip_to_base_um": "10000",
+            "shank_width_um": "70",
+            "tip_length_um": "206",
+            "total_electrodes": "1280"
+        },
         "NP3000": {
             "adc_bit_depth": "10",
             "adc_range_vpp": "1.2",
@@ -2156,6 +2303,55 @@
             "tip_length_um": "203",
             "total_electrodes": "1280"
         },
+        "NP3012": {
+            "adc_bit_depth": "12",
+            "adc_range_vpp": "1.34",
+            "ap_band_list_hz": "0.5,10000",
+            "ap_gain_list": "100",
+            "ap_sample_frequency_hz": "30000",
+            "banks_per_shank": "1.40",
+            "blue_emitter0_dist_from_elec0_um": "0",
+            "blue_emitter_count": "0",
+            "channel_mapping_type": "simple bank",
+            "channels_per_bank": "912",
+            "cols_per_shank": "2",
+            "databus_decoder_type": "channel reordering",
+            "datasheet": "N/A",
+            "description": "Neuropixels NXT alpha version B single shank metal cap",
+            "electrode_layout_type": "linear",
+            "electrode_pitch_horz_um": "32",
+            "electrode_pitch_vert_um": "15",
+            "electrode_select_api_type": "selectElectrode",
+            "electrode_size_horz_direction_um": "12",
+            "electrode_size_vert_direction_um": "12",
+            "electrodes_per_shank": "1280",
+            "emitter_pitch_um": "0",
+            "even_row_horz_offset_left_edge_to_leftmost_electrode_center_um": "27",
+            "has_adc_calibration_file": "N",
+            "has_ap_bandpass": "N",
+            "has_on_chip_gnd_ref": "Y",
+            "has_tip_ref": "Y",
+            "imro_table_format_type": "imro_np3010",
+            "is_commercial": "N",
+            "lf_band_list_hz": "0",
+            "lf_gain_list": "1",
+            "lf_sample_frequency_hz": "0",
+            "mux_table_format_type": "mux_np3010",
+            "num_readout_channels": "1536",
+            "num_shanks": "1",
+            "odd_row_horz_offset_left_edge_to_leftmost_electrode_center_um": "27",
+            "on_shank_ref_chan": "-1",
+            "part_number": "NP3012",
+            "red_emitter0_dist_from_elec0_um": "0",
+            "red_emitter_count": "0",
+            "rows_per_shank": "640",
+            "shank_pitch_um": "0",
+            "shank_thickness_um": "30",
+            "shank_tip_to_base_um": "10000",
+            "shank_width_um": "70",
+            "tip_length_um": "203",
+            "total_electrodes": "1280"
+        },
         "NP3020": {
             "adc_bit_depth": "12",
             "adc_range_vpp": "1.34",
@@ -2317,7 +2513,7 @@
             "cols_per_shank": "2",
             "databus_decoder_type": "channel reordering",
             "datasheet": "N/A",
-            "description": "Neuropixels 3.0 alpha version B multishank silicon cap",
+            "description": "Neuropixels NXT alpha version B multishank silicon cap",
             "electrode_layout_type": "linear",
             "electrode_pitch_horz_um": "32",
             "electrode_pitch_vert_um": "15",
@@ -2366,7 +2562,7 @@
             "cols_per_shank": "2",
             "databus_decoder_type": "channel reordering",
             "datasheet": "N/A",
-            "description": "Neuropixels 3.0 alpha version B multishank metal cap",
+            "description": "Neuropixels NXT alpha version B multishank metal cap",
             "electrode_layout_type": "linear",
             "electrode_pitch_horz_um": "32",
             "electrode_pitch_vert_um": "15",
@@ -2695,7 +2891,7 @@
         "imro_np3020_val_def": "type:pn|{3020,3022} num_channels:num_readout_channels channel:[0,num_readout_channels-1] shank:[0,num_shanks-1] bank:[0,banks_per_shank-1] ref_id:(0,ext)(1,gnd)(2,tip0)(3,tip1)(4,tip2)(5,tip3) electrode:[0,electrodes_per_shank-1]"
     },
     "z_meta": {
-        "table_version": "1.7"
+        "table_version": "1.8"
     },
     "z_mux_tables": {
         "mux_np1000": "(32,12)(0 1 24 25 48 49 72 73 96 97 120 121 144 145 168 169 192 193 216 217 240 241 264 265 288 289 312 313 336 337 360 361)(2 3 26 27 50 51 74 75 98 99 122 123 146 147 170 171 194 195 218 219 242 243 266 267 290 291 314 315 338 339 362 363)(4 5 28 29 52 53 76 77 100 101 124 125 148 149 172 173 196 197 220 221 244 245 268 269 292 293 316 317 340 341 364 365)(6 7 30 31 54 55 78 79 102 103 126 127 150 151 174 175 198 199 222 223 246 247 270 271 294 295 318 319 342 343 366 367)(8 9 32 33 56 57 80 81 104 105 128 129 152 153 176 177 200 201 224 225 248 249 272 273 296 297 320 321 344 345 368 369)(10 11 34 35 58 59 82 83 106 107 130 131 154 155 178 179 202 203 226 227 250 251 274 275 298 299 322 323 346 347 370 371)(12 13 36 37 60 61 84 85 108 109 132 133 156 157 180 181 204 205 228 229 252 253 276 277 300 301 324 325 348 349 372 373)(14 15 38 39 62 63 86 87 110 111 134 135 158 159 182 183 206 207 230 231 254 255 278 279 302 303 326 327 350 351 374 375)(16 17 40 41 64 65 88 89 112 113 136 137 160 161 184 185 208 209 232 233 256 257 280 281 304 305 328 329 352 353 376 377)(18 19 42 43 66 67 90 91 114 115 138 139 162 163 186 187 210 211 234 235 258 259 282 283 306 307 330 331 354 355 378 379)(20 21 44 45 68 69 92 93 116 117 140 141 164 165 188 189 212 213 236 237 260 261 284 285 308 309 332 333 356 357 380 381)(22 23 46 47 70 71 94 95 118 119 142 143 166 167 190 191 214 215 238 239 262 263 286 287 310 311 334 335 358 359 382 383)",

neo/test/rawiotest/test_axonrawio.py

@@ -2,6 +2,7 @@
 import unittest
 
 from neo.rawio.axonrawio import AxonRawIO, parse_axon_soup
+from neo.core import NeoReadWriteError
 
 from neo.test.rawiotest.common_rawio_test import BaseTestRawIO
 
@@ -29,6 +30,43 @@ def test_read_raw_protocol(self):
 
         reader.read_raw_protocol()
 
+    def test_integer_overflow_size_raises(self):
+        # An ABF header that claims more samples than the file can hold must raise a
+        # clear error instead of silently returning an overflowed signal size.
+        path = self.get_local_path("axon/intracellular_data/files_with_errors/integer_overflow_size.abf")
+        expected_error = (
+            "ABF header implies 3221225472 samples ending at byte 6442457600, which exceeds the "
+            f"file size of 8704 bytes for {path}; the file header is corrupt or the file is truncated."
+        )
+        reader = AxonRawIO(filename=path)
+        with self.assertRaises(NeoReadWriteError) as cm:
+            reader.parse_header()
+        self.assertEqual(str(cm.exception), expected_error)
+
+    def test_negative_segment_size_raises(self):
+        # An ABF header with a negative segment size must raise a clear error
+        # instead of silently returning a negative signal size.
+        path = self.get_local_path("axon/intracellular_data/files_with_errors/negative_synch_length.abf")
+        expected_error = f"Negative segment size (-1041598657) parsed from {path}; the file header is corrupt."
+        reader = AxonRawIO(filename=path)
+        with self.assertRaises(NeoReadWriteError) as cm:
+            reader.parse_header()
+        self.assertEqual(str(cm.exception), expected_error)
+
+    def test_unparseable_file_raises(self):
+        # A file whose header does not start with a valid ABF signature must raise a clear error
+        # rather than a cryptic NoneType error deep in parsing. The fixture has a zeroed signature.
+        path = self.get_local_path("axon/intracellular_data/files_with_errors/unparseable_header.abf")
+        expected_msg = (
+            f"Could not parse {path} as an ABF file: expected the header to start with signature "
+            f"b'ABF ' or b'ABF2', but found b'\\x00\\x00\\x00\\x00'. The file is not an ABF file, "
+            f"is corrupt, or is an unsupported variant."
+        )
+        reader = AxonRawIO(filename=path)
+        with self.assertRaises(NeoReadWriteError) as cm:
+            reader.parse_header()
+        self.assertEqual(str(cm.exception), expected_msg)
+
     def test_v1_reads_real_acquisition_date(self):
         # ABF1 stores the calendar date in lFileStartDate (a YYYYMMDD-packed integer). Older neo
         # ignored that field and hardcoded 1900-01-01, so the recording date was always wrong for