Resolve basic keys in Cython and lower per-call overhead (part 2/4 of #744)

CMakeLists.txt

@@ -17,6 +17,9 @@ project(python-blosc2)
 # Update to the latest minimum C-Blosc2 version, and to the actual bundled one
 set(BLOSC2_MIN_VERSION 3.3.0)
 set(BLOSC2_BUNDLED_VERSION v3.3.5)
+# TEMPORARY: build against the companion c-blosc2 PR branch until it is merged and released
+set(BLOSC2_BUNDLED_REPOSITORY https://github.com/FabianIsensee/c-blosc2)
+set(BLOSC2_BUNDLED_VERSION 4832086768ac91996e70bc5831792e4c4c27959c)
 #set(BLOSC2_BUNDLED_VERSION a54e259aeb1c0e62d6bbea28934369d4831e4046)
 
 if(WIN32 AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")
@@ -219,7 +222,7 @@ else()
     set(BLOSC_INSTALL ON)
     include(FetchContent)
     FetchContent_Declare(blosc2
-        GIT_REPOSITORY https://github.com/Blosc/c-blosc2
+        GIT_REPOSITORY ${BLOSC2_BUNDLED_REPOSITORY}
         GIT_TAG ${BLOSC2_BUNDLED_VERSION}
         # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../c-blosc2
     )

RELEASE_NOTES.md

@@ -127,6 +127,40 @@ XXX version-specific blurb XXX
   plotting requires an explicit bounded row window. Client caches are separate
   from a shared cat2lite server cache; repository allowances are per root.
 
+### Faster NDArray indexing, writes and serialization
+
+- Basic keys (integers, unit-step slices, `...`, `None`) in
+  `NDArray.__getitem__` and `__setitem__` are resolved in Cython: `a[3, 5, 7]`
+  and row reads take ~30% less time, keys with NumPy integers up to 5x less.
+- Creating small arrays costs a third less (`zeros((100, 100))` 31 -> 20 µs);
+  `arange`, `linspace`, `eye`, `reshape` and lazy functions such as `clip` no
+  longer call `inspect.stack()` (`clip()` 84 -> 12 µs).
+- `SChunk.decompress_chunk()` no longer reads the chunk from a frame twice.
+
+### Bug fixes
+
+- Fixed crashes of lazy expressions over in-memory frame-backed arrays
+  (`contiguous=True`, or `ndarray_from_cframe()` without a copy) and of the fast
+  `matmul` path with on-disk or frame-backed operands.
+- `NDArray.__getitem__` accepts the keys NumPy accepts that raised or gave a
+  wrong shape: huge slice bounds, bool steps, 0-d integer arrays, scalar bools
+  (placed and broadcast as in NumPy) and `None` next to array indices.
+- `SChunk.update_special()` on an NDArray's super-chunk wrote chunks with the
+  wrong block size, so later reads of the array failed.
+- `update_data()`/`insert_data()` with `copy=False` leaked their chunk when the
+  write failed.
+- True positions of 1-D bool masks no longer run past the array length.
+- `blosc2.nthreads` and the `CParams`/`DParams` defaults follow the
+  `BLOSC_NTHREADS` environment variable, which C-Blosc2 always applied; importing
+  blosc2 no longer overrides `NUMEXPR_NUM_THREADS`.
+
+### Compatibility notes
+
+- Requires the next C-Blosc2 release, for its rule that a failed chunk write
+  without a copy leaves the chunk with the caller.
+- Keys with `None` return arrays that own their data (formerly views of a
+  temporary).
+
 ## Changes from 4.14.0 to 4.14.1
 
 Python-Blosc2 4.14.1 is a security and feature release introducing safe

doc/reference/ndarray.rst

@@ -14,7 +14,7 @@ It is a direct alias for ``array.vlmeta`` and uses the same persistent storage.
 .. autoclass:: NDArray
     :members:
     :inherited-members:
-    :exclude-members: get_slice, set_slice, get_slice_numpy, get_oindex_numpy, set_oindex_numpy
+    :exclude-members: get_slice, set_slice, get_slice_numpy, get_basic_numpy, get_basic_start_stop, get_oindex_numpy, set_oindex_numpy
     :member-order: groupwise
 
     :Special Methods:

src/blosc2/__init__.py

@@ -503,6 +503,8 @@ def _raise(exc):
 cpu_info = get_cpu_info()
 nthreads = ncores = cpu_info.get("count", 1)
 """Number of threads to be used in compression/decompression.
+
+It starts from the ``BLOSC_NTHREADS`` environment variable if that is set.
 """
 # Protection against too many threads
 nthreads = min(nthreads, 64)
@@ -515,14 +517,22 @@ def _raise(exc):
     # Experiments say that, when using a large number of threads, it is better to not use them all
     if nthreads > 16:
         nthreads -= nthreads // 8
-    # Only call set_num_threads if within NUMEXPR_MAX_THREADS limit to avoid warning
+    # C-Blosc2 uses BLOSC_NTHREADS, when set to a valid number, for every compression and
+    # decompression, whatever the nthreads in their parameters; start from it (as C-Blosc2 reads
+    # it), so that nthreads and the CParams/DParams defaults say how many threads are used
+    from .blosc2_ext import context_nthreads
+
+    nthreads = context_nthreads(nthreads)
+    # Only call set_num_threads if within NUMEXPR_MAX_THREADS limit to avoid warning, and not
+    # over the thread count that the user gave numexpr with NUMEXPR_NUM_THREADS
     numexpr_max_env = os.environ.get("NUMEXPR_MAX_THREADS")
     numexpr_max: int | None = None
     if numexpr_max_env is not None:
         with contextlib.suppress(ValueError):
             numexpr_max = int(numexpr_max_env)
-    if numexpr_max is None or nthreads <= numexpr_max:
-        numexpr.set_num_threads(nthreads)
+    if not os.environ.get("NUMEXPR_NUM_THREADS") and (numexpr_max is None or nthreads <= numexpr_max):
+        # BLOSC_NTHREADS can go beyond numexpr's own maximum, which it refuses with an error
+        numexpr.set_num_threads(min(nthreads, numexpr.MAX_THREADS))
 
 # This import must be before ndarray and schunk
 from .storage import (  # noqa: I001