diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bb52ca3e6..e7c2a6e9cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support for map comprehensions - Added USB CDC port drivers for ESP32, RP2, and STM32 platforms - Added a Linux `gpio` driver for the generic_unix port (in `avm_unix`) using sysfs +- Added support for non-byte-aligned bitstrings, including with the legacy construction + opcodes emitted by OTP 26-27 with `no_bs_create_bin`, and added `erlang:bitstring_to_list/1` ### Changed - Updated network type db() to dbm() to reflect the actual representation of the type @@ -64,12 +66,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed `ahttp_client` crash on non-numeric or negative `Content-Length` values - Fixed `ahttp_client` crash on headers with empty or all-whitespace values - Fixed a bug in `supervisor` handling of failing child +- Fixed a one-byte out-of-bounds read and write in bit-granular bitstring copies, which could + touch the byte past an exactly-sized destination or source buffer +- Fixed matching an `all`-sized segment with a non-power-of-two unit, which the JIT tested with an + `and` mask and rejected as unsupported +- Fixed a `binary` segment accepting a non-byte-aligned bitstring source instead of raising `badarg` - Fixed two bugs related to closing fds in `atomvm:subprocess/4` - Fixed `erlang:localtime/1` memory leak, use-after-free, and TZ restore bugs on newlib/picolibc - Fixed ESP32 I2C driver resource leaks, half-closed state, and close-during-transmission errors - Fixed several underallocation issues that could trigger data corruption on `binary:replace`, `zlib:compress` and bsd socket recv code. - Fixed a bug where `catch` would raise on regular atom results - Fixed ESP32 socket driver holding the global socket-list lock across blocking TCP connects, leaking the port on connect failure, losing concurrent `accept` waiters, leaking `netbuf` on receive error paths, and a recycled-`netconn` race between socket close and the event handler +- Fixed a negative dynamic segment size being scaled by the segment unit before validation, which + could let a match succeed with a wrapped-around size, and could move the match offset before the + start of the binary and crash under JIT +- Fixed the JIT untagging small integers with a logical instead of an arithmetic shift, turning a + negative integer into a large positive one ## [0.7.0-alpha.1] - 2026-04-06 diff --git a/doc/src/apidocs/libatomvm/functions.rst b/doc/src/apidocs/libatomvm/functions.rst index 3ae61e9fd4..1a6570fcc3 100644 --- a/doc/src/apidocs/libatomvm/functions.rst +++ b/doc/src/apidocs/libatomvm/functions.rst @@ -241,7 +241,6 @@ Functions .. doxygenfunction:: term_binary_heap_size .. doxygenfunction:: term_binary_size_is_heap_binary .. doxygenfunction:: term_boxed_size -.. doxygenfunction:: term_bs_insert_binary .. doxygenfunction:: term_compare .. doxygenfunction:: term_create_empty_binary .. doxygenfunction:: term_create_uninitialized_binary diff --git a/libs/estdlib/src/erlang.erl b/libs/estdlib/src/erlang.erl index 760063cec6..76e411b7e6 100644 --- a/libs/estdlib/src/erlang.erl +++ b/libs/estdlib/src/erlang.erl @@ -71,6 +71,7 @@ binary_to_integer/1, binary_to_integer/2, binary_to_list/1, + bitstring_to_list/1, atom_to_binary/1, atom_to_binary/2, atom_to_list/1, @@ -151,6 +152,7 @@ hd/1, is_atom/1, is_binary/1, + is_bitstring/1, is_boolean/1, is_float/1, is_function/1, @@ -892,6 +894,21 @@ binary_to_integer(_Binary, _Base) -> binary_to_list(_Binary) -> erlang:nif_error(undefined). +%%----------------------------------------------------------------------------- +%% @param Bitstring Bitstring to convert to list +%% @returns a list of bytes, with a final element holding the trailing bits if +%% `Bitstring' is not a whole number of bytes +%% @doc Convert a bitstring to a list of bytes. +%% +%% If the number of bits in `Bitstring' is not divisible by `8', the +%% last element of the list is a bitstring containing the trailing +%% `1..7' bits, e.g. `bitstring_to_list(<<1:1>>)' returns `[<<1:1>>]'. +%% @end +%%----------------------------------------------------------------------------- +-spec bitstring_to_list(Bitstring :: bitstring()) -> [byte() | bitstring()]. +bitstring_to_list(_Bitstring) -> + erlang:nif_error(undefined). + %%----------------------------------------------------------------------------- %% @param Atom Atom to convert %% @returns a binary with the atom's name @@ -1863,6 +1880,19 @@ is_atom(_Term) -> is_binary(_Term) -> erlang:nif_error(undefined). +%%----------------------------------------------------------------------------- +%% @param Term the term to test +%% @returns `true' if `Term' is a bitstring; `false', otherwise. +%% @doc Return `true' if `Term' is a bitstring (including a binary); +%% `false', otherwise. +%% +%% This function may be used in a guard expression. +%% @end +%%----------------------------------------------------------------------------- +-spec is_bitstring(Term :: term()) -> boolean(). +is_bitstring(_Term) -> + erlang:nif_error(undefined). + %%----------------------------------------------------------------------------- %% @param Term the term to test %% @returns `true' if `Term' is a boolean (`true' or `false'); `false', otherwise. diff --git a/libs/jit/src/jit.erl b/libs/jit/src/jit.erl index efd301a97e..3226a52d0c 100644 --- a/libs/jit/src/jit.erl +++ b/libs/jit/src/jit.erl @@ -802,9 +802,18 @@ first_pass(<>, MMod, MSt0, State0) -> {MSt1, Arg1, Rest2} = decode_compact_term(Rest1, MMod, MSt0, State0), ?TRACE("OP_IS_BINARY ~p, ~p\n", [Label, Arg1]), MSt2 = verify_is_binary(Arg1, Label, MMod, MSt1), - MSt3 = MMod:free_native_registers(MSt2, [Arg1]), - ?ASSERT_ALL_NATIVE_FREE(MSt3), - first_pass(Rest2, MMod, MSt3, State0); + {MSt3, Reg} = MMod:move_to_native_register(MSt2, Arg1), + {MSt4, Reg} = MMod:and_(MSt3, {free, Reg}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt5, TagReg} = MMod:get_array_element(MSt4, Reg, 0), + {MSt6, TagReg} = MMod:and_(MSt5, {free, TagReg}, ?TERM_BOXED_TAG_MASK), + %% is_binary/1 is false for a non-byte-aligned bitstring + MSt7 = MMod:if_block(MSt6, {{free, TagReg}, '==', ?TERM_BOXED_SUB_BINARY}, fun(BSt0) -> + {BSt1, OffsetReg} = MMod:get_array_element(BSt0, Reg, 2), + cond_jump_to_label({{free, OffsetReg}, '&', 16#7, '!=', 0}, Label, MMod, BSt1) + end), + MSt8 = MMod:free_native_registers(MSt7, [Reg]), + ?ASSERT_ALL_NATIVE_FREE(MSt8), + first_pass(Rest2, MMod, MSt8, State0); % 55 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -1307,7 +1316,13 @@ first_pass(<>, MMod, MSt0, State0) -> is_integer(SizeReg) -> {MSt4, SizeReg * Unit}; true -> - MSt5 = MMod:mul(MSt4, SizeReg, Unit), + %% A negative size fails the match (nomatch), as on BEAM. It has + %% to be rejected before it is scaled by the unit: the scaled + %% size is added to the match state offset, and the capacity + %% check would then compare against a negative offset and let + %% the match proceed before the start of the bitstring. + MSt4a = cond_jump_to_label({SizeReg, '<', 0}, Fail, MMod, MSt4), + MSt5 = MMod:mul(MSt4a, SizeReg, Unit), {MSt5, SizeReg} end, {MSt7, BSBinaryReg} = MMod:get_array_element(MSt6, MatchStateRegPtr, 1), @@ -1345,7 +1360,13 @@ first_pass(<>, MMod, MSt0, State0) -> is_integer(SizeReg) -> {MSt4, SizeReg * Unit}; true -> - MSt5 = MMod:mul(MSt4, SizeReg, Unit), + %% A negative size fails the match (nomatch), as on BEAM. It has + %% to be rejected before it is scaled by the unit: the scaled + %% size is added to the match state offset, and the capacity + %% check would then compare against a negative offset and let + %% the match proceed before the start of the bitstring. + MSt4a = cond_jump_to_label({SizeReg, '<', 0}, Fail, MMod, MSt4), + MSt5 = MMod:mul(MSt4a, SizeReg, Unit), {MSt5, SizeReg} end, {MSt7, BSBinaryReg} = MMod:get_array_element(MSt6, MatchStateRegPtr, 1), @@ -1380,10 +1401,6 @@ first_pass(<>, MMod, MSt0, State0) -> {MSt5, BSOffsetReg0} = MMod:get_array_element(MSt4, MatchStateRegPtr, 2), MSt6 = if - Unit =/= 8 -> - MMod:call_primitive_last(MSt5, ?PRIM_RAISE_ERROR, [ - ctx, jit_state, offset, ?UNSUPPORTED_ATOM - ]); FlagsValue =/= 0 -> MMod:call_primitive_last(MSt5, ?PRIM_RAISE_ERROR, [ ctx, jit_state, offset, ?UNSUPPORTED_ATOM @@ -1391,47 +1408,53 @@ first_pass(<>, MMod, MSt0, State0) -> true -> MSt5 end, - MSt7 = MMod:if_block(MSt6, {BSOffsetReg0, '&', 16#7, '!=', 0}, fun(BlockSt) -> - MMod:call_primitive_last(BlockSt, ?PRIM_RAISE_ERROR, [ctx, jit_state, offset, ?BADARG_ATOM]) - end), - {MSt8, BSOffsetReg1} = MMod:shift_right(MSt7, {free, BSOffsetReg0}, 3), - {MSt9, BSBinaryReg0} = MMod:and_(MSt8, {free, BSBinaryReg0}, ?TERM_PRIMARY_CLEAR_MASK), - {MSt10, SizeReg} = MMod:get_array_element(MSt9, {free, BSBinaryReg0}, 1), - {MSt13, SizeValue} = + % Remaining bits = bit_size(binary) - offset; sizes are bit-granular and + % the offset may be unaligned (the slice primitive copies in that case). + {MSt7, BinPtrReg} = MMod:and_(MSt6, {free, BSBinaryReg0}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt8, RemainingReg} = term_bit_size({ptr, BinPtrReg}, MMod, MSt7), + MSt9 = MMod:free_native_registers(MSt8, [BinPtrReg]), + MSt10 = MMod:sub(MSt9, RemainingReg, BSOffsetReg0), + {MSt14, SizeBits} = if Size =:= ?ALL_ATOM -> - MSt11 = MMod:sub(MSt10, SizeReg, BSOffsetReg1), - {MSt11, SizeReg}; + % all takes the whole remainder, which must be a multiple of + % the segment unit + MSt11 = bs_jump_unless_multiple_of_unit(RemainingReg, Unit, Fail, MMod, MSt10), + {MSt11, RemainingReg}; is_integer(Size) -> - % SizeReg is binary size % Size is a tagged integer: (N bsl 4) bor 0xF - % SizeBytes is the raw byte count - SizeBytes = Size bsr 4, - MSt11 = MMod:sub(MSt10, SizeReg, SizeBytes), - MSt12 = cond_jump_to_label({{free, SizeReg}, '<', BSOffsetReg1}, Fail, MMod, MSt11), - {MSt12, SizeBytes}; + SizeBitsLit = (Size bsr 4) * Unit, + MSt11 = cond_jump_to_label( + {{free, RemainingReg}, '<', SizeBitsLit}, Fail, MMod, MSt10 + ), + {MSt11, SizeBitsLit}; true -> {MSt11, SizeValReg} = MMod:move_to_native_register(MSt10, Size), MSt12 = MMod:if_else_block( MSt11, {SizeValReg, '==', ?ALL_ATOM}, fun(BSt0) -> - BSt1 = MMod:sub(BSt0, SizeReg, BSOffsetReg1), - MMod:free_native_registers(BSt1, [SizeValReg]) + BSt1 = bs_jump_unless_multiple_of_unit( + RemainingReg, Unit, Fail, MMod, BSt0 + ), + MMod:move_to_native_register(BSt1, RemainingReg, SizeValReg) end, fun(BSt0) -> - {BSt1, SizeValReg} = term_to_int(SizeValReg, 0, MMod, BSt0), - BSt2 = MMod:sub(BSt1, SizeReg, SizeValReg), - BSt3 = cond_jump_to_label({SizeReg, '<', BSOffsetReg1}, Fail, MMod, BSt2), - BSt4 = MMod:move_to_native_register(BSt3, SizeValReg, SizeReg), - MMod:free_native_registers(BSt4, [SizeValReg]) + %% A size that isn't a small integer (a bignum) cannot + %% fit the remaining capacity: fail the match, as the + %% interpreter does, rather than raise badarg. + {BSt1, SizeValReg} = term_to_int(SizeValReg, Fail, MMod, BSt0), + %% A negative size fails the match (nomatch), as on BEAM. + BSt2 = cond_jump_to_label({SizeValReg, '<', 0}, Fail, MMod, BSt1), + BSt3 = MMod:mul(BSt2, SizeValReg, Unit), + cond_jump_to_label({RemainingReg, '<', SizeValReg}, Fail, MMod, BSt3) end ), - {MSt12, SizeReg} + MSt13 = MMod:free_native_registers(MSt12, [RemainingReg]), + {MSt13, SizeValReg} end, - {MSt14, NewOffsetReg} = MMod:copy_to_native_register(MSt13, BSOffsetReg1), - MSt15 = MMod:add(MSt14, NewOffsetReg, SizeValue), - MSt16 = MMod:shift_left(MSt15, NewOffsetReg, 3), + {MSt15, NewOffsetReg} = MMod:copy_to_native_register(MSt14, BSOffsetReg0), + MSt16 = MMod:add(MSt15, NewOffsetReg, SizeBits), % Write new offset MSt17 = MMod:move_to_array_element(MSt16, NewOffsetReg, MatchStateRegPtr, 2), MSt18 = MMod:free_native_registers(MSt17, [NewOffsetReg]), @@ -1439,23 +1462,30 @@ first_pass(<>, MMod, MSt0, State0) -> MSt20 = MMod:free_native_registers(MSt19, [TrimResultReg]), {MSt21, BSBinaryReg1} = MMod:get_array_element(MSt20, {free, MatchStateRegPtr}, 1), MSt22 = MMod:or_(MSt21, BSBinaryReg1, ?TERM_PRIMARY_BOXED), - {MSt23, HeapSizeReg} = MMod:call_primitive(MSt22, ?PRIM_TERM_SUB_BINARY_HEAP_SIZE, [ - BSBinaryReg1, SizeValue + {MSt23, TmpPtrReg} = MMod:copy_to_native_register(MSt22, BSBinaryReg1), + {MSt24, TmpPtrReg} = MMod:and_(MSt23, {free, TmpPtrReg}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt25, HeapSizeReg} = MMod:call_primitive(MSt24, ?PRIM_BITSTRING_SLICE_HEAP_SIZE, [ + {free, TmpPtrReg}, BSOffsetReg0, SizeBits ]), - {MSt24, BSBinaryReg2} = memory_ensure_free_with_extra_root( - BSBinaryReg1, Live, {free, HeapSizeReg}, MMod, MSt23 + {MSt26, BSBinaryReg2} = memory_ensure_free_with_extra_root( + BSBinaryReg1, Live, {free, HeapSizeReg}, MMod, MSt25 ), - {MSt25, ResultTerm} = MMod:call_primitive(MSt24, ?PRIM_TERM_MAYBE_CREATE_SUB_BINARY, [ - ctx, {free, BSBinaryReg2}, {free, BSOffsetReg1}, {free, SizeValue} + SizeBitsArg = + if + is_integer(SizeBits) -> SizeBits; + true -> {free, SizeBits} + end, + {MSt27, ResultTerm} = MMod:call_primitive(MSt26, ?PRIM_BITSTRING_SLICE, [ + ctx, {free, BSBinaryReg2}, {free, BSOffsetReg0}, SizeBitsArg ]), - {MSt26, Dest, Rest7} = decode_dest(Rest6, MMod, MSt25), + {MSt28, Dest, Rest7} = decode_dest(Rest6, MMod, MSt27), ?TRACE("OP_BS_GET_BINARY2 ~p,~p,~p,~p,~p,~p,~p\n", [ Fail, Src, Live, Size, Unit, FlagsValue, Dest ]), - MSt27 = MMod:move_to_vm_register(MSt26, ResultTerm, Dest), - MSt28 = MMod:free_native_registers(MSt27, [ResultTerm, Dest]), - ?ASSERT_ALL_NATIVE_FREE(MSt28), - first_pass(Rest7, MMod, MSt28, State0); + MSt29 = MMod:move_to_vm_register(MSt28, ResultTerm, Dest), + MSt30 = MMod:free_native_registers(MSt29, [ResultTerm, Dest]), + ?ASSERT_ALL_NATIVE_FREE(MSt30), + first_pass(Rest7, MMod, MSt30, State0); % 120 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -1472,20 +1502,25 @@ first_pass(<>, MMod, MSt0, State0) -> is_integer(SizeReg) -> {MSt4, SizeReg * Unit}; true -> - MSt5 = MMod:mul(MSt4, SizeReg, Unit), + %% A negative size fails the match (nomatch), as on BEAM. It has + %% to be rejected before it is scaled by the unit: the scaled + %% size is added to the match state offset, and the capacity + %% check would then compare against a negative offset and let + %% the match proceed before the start of the bitstring. + MSt4a = cond_jump_to_label({SizeReg, '<', 0}, Fail, MMod, MSt4), + MSt5 = MMod:mul(MSt4a, SizeReg, Unit), {MSt5, SizeReg} end, {MSt7, BSBinaryReg} = MMod:get_array_element(MSt6, MatchStateRegPtr, 1), {MSt8, BSOffsetReg} = MMod:get_array_element(MSt7, MatchStateRegPtr, 2), MSt9 = MMod:add(MSt8, BSOffsetReg, NumBits), MSt10 = MMod:free_native_registers(MSt9, [NumBits]), - {MSt11, BSBinarySize} = term_binary_size({free, BSBinaryReg}, MMod, MSt10), - MSt12 = MMod:shift_left(MSt11, BSBinarySize, 3), - MSt13 = cond_jump_to_label({{free, BSBinarySize}, '<', BSOffsetReg}, Fail, MMod, MSt12), - MSt14 = MMod:move_to_array_element(MSt13, BSOffsetReg, MatchStateRegPtr, 2), - MSt15 = MMod:free_native_registers(MSt14, [BSOffsetReg, MatchStateRegPtr]), - ?ASSERT_ALL_NATIVE_FREE(MSt15), - first_pass(Rest5, MMod, MSt15, State0); + {MSt11, BSBinarySize} = term_bit_size({free, BSBinaryReg}, MMod, MSt10), + MSt12 = cond_jump_to_label({{free, BSBinarySize}, '<', BSOffsetReg}, Fail, MMod, MSt11), + MSt13 = MMod:move_to_array_element(MSt12, BSOffsetReg, MatchStateRegPtr, 2), + MSt14 = MMod:free_native_registers(MSt13, [BSOffsetReg, MatchStateRegPtr]), + ?ASSERT_ALL_NATIVE_FREE(MSt14), + first_pass(Rest5, MMod, MSt14, State0); % 121 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -1498,12 +1533,11 @@ first_pass(<>, MMod, MSt0, State0) -> {MSt4, BSOffsetReg} = MMod:get_array_element(MSt3, MatchStateRegPtr, 2), MSt5 = MMod:free_native_registers(MSt4, [MatchStateRegPtr]), MSt6 = MMod:add(MSt5, BSOffsetReg, Bits), - {MSt7, BSBinarySize} = term_binary_size({free, BSBinaryReg}, MMod, MSt6), - MSt8 = MMod:shift_left(MSt7, BSBinarySize, 3), - MSt9 = cond_jump_to_label({{free, BSBinarySize}, '!=', BSOffsetReg}, Fail, MMod, MSt8), - MSt10 = MMod:free_native_registers(MSt9, [BSOffsetReg]), - ?ASSERT_ALL_NATIVE_FREE(MSt10), - first_pass(Rest3, MMod, MSt10, State0); + {MSt7, BSBinarySize} = term_bit_size({free, BSBinaryReg}, MMod, MSt6), + MSt8 = cond_jump_to_label({{free, BSBinarySize}, '!=', BSOffsetReg}, Fail, MMod, MSt7), + MSt9 = MMod:free_native_registers(MSt8, [BSOffsetReg]), + ?ASSERT_ALL_NATIVE_FREE(MSt9), + first_pass(Rest3, MMod, MSt9, State0); % 124 first_pass( <>, MMod, MSt0, #state{import_resolver = ImportResolver} = State0 @@ -1572,16 +1606,15 @@ first_pass(<>, MMod, MSt0, State0) -> {MSt3, BSBinaryReg} = MMod:get_array_element(MSt2, MatchStateRegPtr, 1), {MSt4, BSOffsetReg} = MMod:get_array_element(MSt3, MatchStateRegPtr, 2), MSt5 = MMod:free_native_registers(MSt4, [MatchStateRegPtr]), - {MSt6, BSBinarySize} = term_binary_size({free, BSBinaryReg}, MMod, MSt5), - MSt7 = MMod:shift_left(MSt6, BSBinarySize, 3), - % BSBinarySize = binary_size * 8 - MSt8 = MMod:sub(MSt7, BSBinarySize, BSOffsetReg), + {MSt6, BSBinarySize} = term_bit_size({free, BSBinaryReg}, MMod, MSt5), + % BSBinarySize = source bit size + MSt7 = MMod:sub(MSt6, BSBinarySize, BSOffsetReg), % BSBinarySize = (binary_size * 8) - offset = remaining bits - MSt9 = MMod:free_native_registers(MSt8, [BSOffsetReg]), - {MSt10, BSBinarySize1} = MMod:and_(MSt9, {free, BSBinarySize}, Unit - 1), - MSt11 = cond_jump_to_label({{free, BSBinarySize1}, '!=', 0}, Fail, MMod, MSt10), - ?ASSERT_ALL_NATIVE_FREE(MSt11), - first_pass(Rest3, MMod, MSt11, State0); + MSt8 = MMod:free_native_registers(MSt7, [BSOffsetReg]), + MSt9 = bs_jump_unless_multiple_of_unit(BSBinarySize, Unit, Fail, MMod, MSt8), + MSt10 = MMod:free_native_registers(MSt9, [BSBinarySize]), + ?ASSERT_ALL_NATIVE_FREE(MSt10), + first_pass(Rest3, MMod, MSt10, State0); % 132 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -2370,73 +2403,92 @@ first_pass( MSt3 = MMod:add(MSt2, BinaryRegSize, BinaryLitSize), {MSt3, BinaryRegSize} end, - MSt5 = + {MSt5, TrimResultReg} = MMod:call_primitive(MSt4, ?PRIM_TRIM_LIVE_REGS, [ctx, Live]), + MSt6 = MMod:free_native_registers(MSt5, [TrimResultReg]), + %% A reused (append/private_append) binary must stay byte-aligned + MSt7 = if - is_integer(BinaryTotalSize) andalso BinaryTotalSize band 16#7 =/= 0 -> - MMod:call_primitive_last(MSt4, ?PRIM_RAISE_ERROR, [ + ReuseSourceBinary andalso is_integer(BinaryTotalSize) andalso + BinaryTotalSize rem 8 =/= 0 -> + MMod:call_primitive_last(MSt6, ?PRIM_RAISE_ERROR, [ ctx, jit_state, offset, ?UNSUPPORTED_ATOM ]); - is_integer(BinaryTotalSize) -> - MSt4; - true -> - MMod:if_block(MSt4, {BinaryTotalSize, '&', 16#7, '!=', 0}, fun(BlockSt) -> + ReuseSourceBinary andalso not is_integer(BinaryTotalSize) -> + MMod:if_block(MSt6, {BinaryTotalSize, '&', 16#7, '!=', 0}, fun(BlockSt) -> MMod:call_primitive_last(BlockSt, ?PRIM_RAISE_ERROR, [ ctx, jit_state, offset, ?UNSUPPORTED_ATOM ]) - end) + end); + true -> + MSt6 end, - {MSt6, TrimResultReg} = MMod:call_primitive(MSt5, ?PRIM_TRIM_LIVE_REGS, [ctx, Live]), - MSt7 = MMod:free_native_registers(MSt6, [TrimResultReg]), - {MSt12, BinaryTotalSizeInBytes, AllocSize} = + %% A non-byte-aligned total yields a bitstring + {MSt14, BinaryTotalSizeInBytes, AllocSize, WrapInfo} = if - is_integer(BinaryTotalSize) -> + ReuseSourceBinary andalso is_integer(BinaryTotalSize) -> {MSt7, (BinaryTotalSize div 8), - term_binary_heap_size((BinaryTotalSize div 8), MMod) + Alloc}; + term_binary_heap_size((BinaryTotalSize div 8), MMod) + Alloc, no_wrap}; + ReuseSourceBinary -> + {MSt8, Bytes} = MMod:shift_right(MSt7, {free, BinaryTotalSize}, 3), + {MSt9, BytesCopy} = MMod:copy_to_native_register(MSt8, Bytes), + {MSt10, AllocSizeReg} = term_binary_heap_size({free, BytesCopy}, MMod, MSt9), + MSt11 = + case Alloc of + 0 -> MSt10; + _ -> MMod:add(MSt10, AllocSizeReg, Alloc) + end, + {MSt11, Bytes, AllocSizeReg, no_wrap}; + is_integer(BinaryTotalSize) -> + Trailing = BinaryTotalSize rem 8, + ByteSize = (BinaryTotalSize + 7) div 8, + {Extra, Wrap} = + case Trailing of + 0 -> {0, no_wrap}; + _ -> {?TERM_BOXED_SUB_BINARY_SIZE, {wrap_lit, BinaryTotalSize}} + end, + {MSt7, ByteSize, term_binary_heap_size(ByteSize, MMod) + Alloc + Extra, Wrap}; true -> - {MSt8, BinaryTotalSizeBytes} = MMod:shift_right(MSt7, {free, BinaryTotalSize}, 3), - {MSt9, BinaryTotalSizeBytes0} = MMod:copy_to_native_register( - MSt8, BinaryTotalSizeBytes - ), - {MSt10, AllocSizeReg} = term_binary_heap_size( - {free, BinaryTotalSizeBytes0}, MMod, MSt9 - ), - case Alloc of - 0 -> - {MSt10, BinaryTotalSizeBytes, AllocSizeReg}; - _ -> - MSt11 = MMod:add(MSt10, AllocSizeReg, Alloc), - {MSt11, BinaryTotalSizeBytes, AllocSizeReg} - end + MSt8 = MMod:add(MSt7, BinaryTotalSize, 7), + {MSt9, CeilBytes} = MMod:shift_right(MSt8, {free, BinaryTotalSize}, 3), + {MSt10, CeilCopy} = MMod:copy_to_native_register(MSt9, CeilBytes), + {MSt11, AllocSizeReg} = term_binary_heap_size({free, CeilCopy}, MMod, MSt10), + MSt12 = MMod:add(MSt11, AllocSizeReg, ?TERM_BOXED_SUB_BINARY_SIZE), + MSt13 = + case Alloc of + 0 -> MSt12; + _ -> MMod:add(MSt12, AllocSizeReg, Alloc) + end, + {MSt13, CeilBytes, AllocSizeReg, wrap_dyn} end, - {MSt13, MemoryEnsureFreeReg} = MMod:call_primitive( - MSt12, ?PRIM_MEMORY_ENSURE_FREE_WITH_ROOTS, [ + {MSt15, MemoryEnsureFreeReg} = MMod:call_primitive( + MSt14, ?PRIM_MEMORY_ENSURE_FREE_WITH_ROOTS, [ ctx, jit_state, {free, AllocSize}, Live, ?MEMORY_CAN_SHRINK ] ), - MSt14 = handle_error_if({'(bool)', {free, MemoryEnsureFreeReg}, '==', false}, MMod, MSt13), - {MSt17, InitialCreatedBin} = + MSt16 = handle_error_if({'(bool)', {free, MemoryEnsureFreeReg}, '==', false}, MMod, MSt15), + {MSt19, InitialCreatedBin} = case ReuseSourceBinary of false -> % No reuse - create the binary now - {MSt15, CreatedBinResult} = MMod:call_primitive( - MSt14, ?PRIM_TERM_CREATE_EMPTY_BINARY, [ + {MSt17, CreatedBinResult} = MMod:call_primitive( + MSt16, ?PRIM_TERM_CREATE_EMPTY_BINARY, [ ctx, {free, BinaryTotalSizeInBytes} ] ), - MSt16 = MMod:if_block(MSt15, {CreatedBinResult, '==', ?TERM_INVALID_TERM}, fun( + MSt18 = MMod:if_block(MSt17, {CreatedBinResult, '==', ?TERM_INVALID_TERM}, fun( BSt0 ) -> MMod:call_primitive_last(BSt0, ?PRIM_RAISE_ERROR, [ ctx, jit_state, offset, ?OUT_OF_MEMORY_ATOM ]) end), - {MSt16, CreatedBinResult}; + {MSt18, CreatedBinResult}; true -> % Will reuse - defer creation until first segment - {MSt14, {private_append, BinaryTotalSizeInBytes}} + {MSt16, {private_append, BinaryTotalSizeInBytes}} end, % We redo the decoding. Rest7 should still be equal to previous value. - {Rest7, MSt18, FinalOffset, CreatedBin} = lists:foldl( + {Rest7, MSt20, FinalOffset, CreatedBin} = lists:foldl( fun(_Index, {AccRest0, AccMSt0, AccOffset0, AccCreatedBin}) -> {AtomTypeIndex, AccRest1} = decode_atom(AccRest0), AtomType = AtomResolver(AtomTypeIndex), @@ -2461,15 +2513,30 @@ first_pass( AccMSt5 = MMod:free_native_registers(AccMSt4, [Flags, Src, Size]), {AccRest6, AccMSt5, AccOffset1, AccCreatedBin1} end, - {Rest6, MSt17, 0, InitialCreatedBin}, + {Rest6, MSt19, 0, InitialCreatedBin}, lists:seq(1, NBSegments) ), ?TRACE("]\n", []), - MSt19 = MMod:free_native_registers(MSt18, [FinalOffset]), - MSt20 = MMod:move_to_vm_register(MSt19, CreatedBin, Dest), - MSt21 = MMod:free_native_registers(MSt20, [CreatedBin, Dest]), - ?ASSERT_ALL_NATIVE_FREE(MSt21), - first_pass(Rest7, MMod, MSt21, State1); + %% Wrap a non-byte-aligned result in a sub-binary carrying the trailing bit + %% count. FinalOffset holds the total inserted bit count for the dynamic case. + {MSt21, CreatedBin1} = + case WrapInfo of + no_wrap -> + {MSt20, CreatedBin}; + {wrap_lit, TotalBits} -> + MMod:call_primitive(MSt20, ?PRIM_BS_CREATE_BIN_WRAP, [ + ctx, {free, CreatedBin}, TotalBits + ]); + wrap_dyn -> + MMod:call_primitive(MSt20, ?PRIM_BS_CREATE_BIN_WRAP, [ + ctx, {free, CreatedBin}, FinalOffset + ]) + end, + MSt22 = MMod:free_native_registers(MSt21, [FinalOffset]), + MSt23 = MMod:move_to_vm_register(MSt22, CreatedBin1, Dest), + MSt24 = MMod:free_native_registers(MSt23, [CreatedBin1, Dest]), + ?ASSERT_ALL_NATIVE_FREE(MSt24), + first_pass(Rest7, MMod, MSt24, State1); % 178 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -2761,20 +2828,22 @@ first_pass_bs_create_bin_compute_size( end, {MSt5, AccLiteralSize0 + (SizeValue * SegmentUnit), AccSizeReg0, State0}; first_pass_bs_create_bin_compute_size( - AtomType, Src, ?ALL_ATOM, _SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 + AtomType, Src, ?ALL_ATOM, SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 ) when AtomType =:= binary orelse AtomType =:= append orelse AtomType =:= private_append -> MSt1 = verify_is_binary(Src, Fail, MMod, MSt0), - {MSt2, Reg} = MMod:copy_to_native_register(MSt1, Src), - {MSt3, Reg} = MMod:and_(MSt2, {free, Reg}, ?TERM_PRIMARY_CLEAR_MASK), - MSt4 = MMod:move_array_element(MSt3, Reg, 1, Reg), - MSt5 = MMod:shift_left(MSt4, Reg, 3), + {MSt2, Reg0} = MMod:copy_to_native_register(MSt1, Src), + {MSt3a, Reg} = term_bit_size({free, Reg0}, MMod, MSt2), + %% The whole source is taken, so its bit size must be a multiple of the + %% segment unit: a /binary segment (unit 8) rejects a partial bitstring, + %% a /bitstring one (unit 1) takes it. + MSt3 = bs_badarg_unless_multiple_of_unit(Reg, SegmentUnit, Fail, MMod, MSt3a), case AccSizeReg0 of undefined -> - {MSt5, AccLiteralSize0, Reg, State0}; + {MSt3, AccLiteralSize0, Reg, State0}; _ -> - MSt6 = MMod:add(MSt5, AccSizeReg0, Reg), - MSt7 = MMod:free_native_registers(MSt6, [Reg]), - {MSt7, AccLiteralSize0, AccSizeReg0, State0} + MSt4 = MMod:add(MSt3, AccSizeReg0, Reg), + MSt5 = MMod:free_native_registers(MSt4, [Reg]), + {MSt5, AccLiteralSize0, AccSizeReg0, State0} end; first_pass_bs_create_bin_compute_size( AtomType, Src, Size, SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 @@ -2784,17 +2853,22 @@ first_pass_bs_create_bin_compute_size( -> MSt1 = verify_is_binary(Src, Fail, MMod, MSt0), {MSt2, SizeValue} = term_to_int(Size, 0, MMod, MSt1), - {MSt2, AccLiteralSize0 + (SizeValue * SegmentUnit), AccSizeReg0, State0}; + RequiredBits = SizeValue * SegmentUnit, + %% Verify the source provides at least RequiredBits bits + {MSt3, SrcBitsReg0} = MMod:copy_to_native_register(MSt2, Src), + {MSt4, SrcBitsReg} = term_bit_size({free, SrcBitsReg0}, MMod, MSt3), + MSt5 = cond_raise_badarg_or_jump_to_fail_label( + {{free, SrcBitsReg}, '<', RequiredBits}, Fail, MMod, MSt4 + ), + {MSt5, AccLiteralSize0 + RequiredBits, AccSizeReg0, State0}; first_pass_bs_create_bin_compute_size( AtomType, Src, Size, SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 ) when AtomType =:= binary orelse AtomType =:= append orelse AtomType =:= private_append -> MSt1 = verify_is_binary(Src, Fail, MMod, MSt0), {MSt2, Reg0} = MMod:copy_to_native_register(MSt1, Size), - {MSt3, Reg1} = MMod:copy_to_native_register(MSt2, Src), - {MSt4, Reg1} = MMod:and_(MSt3, {free, Reg1}, ?TERM_PRIMARY_CLEAR_MASK), - MSt5 = MMod:move_array_element(MSt4, Reg1, 1, Reg1), - MSt6 = MMod:shift_left(MSt5, Reg1, 3), - MSt7 = MMod:if_block(MSt6, {{free, Reg0}, '!=', ?ALL_ATOM}, fun(BSt0) -> + {MSt3, Reg1a} = MMod:copy_to_native_register(MSt2, Src), + {MSt4, Reg1} = term_bit_size({free, Reg1a}, MMod, MSt3), + MSt5 = MMod:if_block(MSt4, {{free, Reg0}, '!=', ?ALL_ATOM}, fun(BSt0) -> {BSt1, SizeReg} = term_to_int(Size, Fail, MMod, BSt0), BSt2 = cond_raise_badarg_or_jump_to_fail_label( {SizeReg, '<', 0}, Fail, MMod, BSt1 @@ -2808,11 +2882,11 @@ first_pass_bs_create_bin_compute_size( end), case AccSizeReg0 of undefined -> - {MSt7, AccLiteralSize0, Reg1, State0}; + {MSt5, AccLiteralSize0, Reg1, State0}; _ -> - MSt8 = MMod:add(MSt7, AccSizeReg0, Reg1), - MSt9 = MMod:free_native_registers(MSt8, [Reg1]), - {MSt9, AccLiteralSize0, AccSizeReg0, State0} + MSt6 = MMod:add(MSt5, AccSizeReg0, Reg1), + MSt7 = MMod:free_native_registers(MSt6, [Reg1]), + {MSt7, AccLiteralSize0, AccSizeReg0, State0} end. first_pass_bs_create_bin_insert_value( @@ -3077,22 +3151,20 @@ first_pass_bs_match_ensure_at_least( true -> {Unit, Rest2} = decode_literal(Rest1), ?TRACE("{ensure_at_least,~p,~p},", [Stride, Unit]), - {MSt1, Reg} = MMod:get_array_element(MSt0, BSBinaryReg, 1), - MSt2 = MMod:shift_left(MSt1, Reg, 3), - % Reg is bs_bin_size * 8 - MSt3 = MMod:sub(MSt2, Reg, BSOffsetReg), + {MSt1, Reg} = term_bit_size({ptr, BSBinaryReg}, MMod, MSt0), + % Reg is the source bit size + MSt2 = MMod:sub(MSt1, Reg, BSOffsetReg), % Reg is (bs_bin_size * 8) - bs_offset = remaining bits - MSt4 = cond_jump_to_label({Reg, '<', Stride}, Fail, MMod, MSt3), + MSt3 = cond_jump_to_label({Reg, '<', Stride}, Fail, MMod, MSt2), % Also check unit alignment: (remaining - stride) % unit == 0 MSt7 = if Unit > 1 -> - MSt4b = MMod:sub(MSt4, Reg, Stride), - {MSt5, UnitReg} = MMod:and_(MSt4b, {free, Reg}, Unit - 1), - MSt6 = cond_jump_to_label({{free, UnitReg}, '!=', 0}, Fail, MMod, MSt5), - MSt6; + MSt4 = MMod:sub(MSt3, Reg, Stride), + MSt5 = bs_jump_unless_multiple_of_unit(Reg, Unit, Fail, MMod, MSt4), + MMod:free_native_registers(MSt5, [Reg]); true -> - MMod:free_native_registers(MSt4, [Reg]) + MMod:free_native_registers(MSt3, [Reg]) end, {J0 - 2, Rest2, MatchState, BSOffsetReg, MSt7} end. @@ -3109,14 +3181,13 @@ first_pass_bs_match_ensure_exactly( {J0, Rest0, MatchState, BSOffsetReg, MSt1}; true -> ?TRACE("{ensure_exactly,~p},", [Stride]), - {MSt1, Reg} = MMod:get_array_element(MSt0, BSBinaryReg, 1), - MSt2 = MMod:shift_left(MSt1, Reg, 3), - % Reg is bs_bin_size * 8 (use unit instead ??) - MSt3 = MMod:sub(MSt2, Reg, BSOffsetReg), + {MSt1, Reg} = term_bit_size({ptr, BSBinaryReg}, MMod, MSt0), + % Reg is the source bit size + MSt2 = MMod:sub(MSt1, Reg, BSOffsetReg), % Reg is (bs_bin_size * 8) - bs_offset - MSt4 = cond_jump_to_label({Reg, '!=', Stride}, Fail, MMod, MSt3), - MSt5 = MMod:free_native_registers(MSt4, [Reg]), - {J0 - 1, Rest1, MatchState, BSOffsetReg, MSt5} + MSt3 = cond_jump_to_label({Reg, '!=', Stride}, Fail, MMod, MSt2), + MSt4 = MMod:free_native_registers(MSt3, [Reg]), + {J0 - 1, Rest1, MatchState, BSOffsetReg, MSt4} end. first_pass_bs_match_integer( @@ -3125,46 +3196,38 @@ first_pass_bs_match_integer( {_Live, Rest1} = decode_literal(Rest0), {Flags, Rest2} = decode_compile_time_literal(Rest1, State0), {MSt1, FlagsValue} = decode_flags_list(Flags, MMod, MSt0), - {MSt2, Size, Rest3} = decode_typed_compact_term(Rest2, MMod, MSt0, State0), + {Size, Rest3} = decode_literal(Rest2), {Unit, Rest4} = decode_literal(Rest3), ?TRACE("{integer,~p,~p,~p, ", [Flags, Size, Unit]), - {MSt3, SizeReg} = term_to_int(Size, 0, MMod, MSt1), - {MSt6, NumBits} = - if - is_integer(SizeReg) -> - {MSt2, SizeReg * Unit}; - true -> - MSt3 = MMod:mul(SizeReg, Unit), - {MSt3, SizeReg} - end, - {MSt7, Result} = MMod:call_primitive(MSt6, ?PRIM_BITSTRING_EXTRACT_INTEGER, [ + NumBits = Size * Unit, + {MSt2, Result} = MMod:call_primitive(MSt1, ?PRIM_BITSTRING_EXTRACT_INTEGER, [ ctx, jit_state, BSBinaryReg, BSOffsetReg, NumBits, {free, FlagsValue} ]), - MSt8 = handle_error_if({Result, '==', 0}, MMod, MSt7), - MSt9 = cond_jump_to_label({Result, '==', ?FALSE_ATOM}, Fail, MMod, MSt8), - MSt10 = - case MMod:available_regs(MSt9) of + MSt3 = handle_error_if({Result, '==', 0}, MMod, MSt2), + MSt4 = cond_jump_to_label({Result, '==', ?FALSE_ATOM}, Fail, MMod, MSt3), + MSt5 = + case MMod:available_regs(MSt4) of [] -> - MMod:free_native_registers(MSt9, [BSOffsetReg]); + MMod:free_native_registers(MSt4, [BSOffsetReg]); _ -> - MSt9 + MSt4 end, - {MSt11, Dest, Rest5} = decode_dest(Rest4, MMod, MSt10), + {MSt6, Dest, Rest5} = decode_dest(Rest4, MMod, MSt5), ?TRACE("~p},", [Dest]), - MSt12 = MMod:move_to_vm_register(MSt11, Result, Dest), - MSt13 = MMod:free_native_registers(MSt12, [Result, Dest]), - case MMod:available_regs(MSt9) of + MSt7 = MMod:move_to_vm_register(MSt6, Result, Dest), + MSt8 = MMod:free_native_registers(MSt7, [Result, Dest]), + case MMod:available_regs(MSt4) of [] -> - {MSt14, MatchState} = MMod:and_(MSt13, {free, MatchState}, ?TERM_PRIMARY_CLEAR_MASK), - {MSt15, NewBSOffsetReg} = MMod:get_array_element(MSt14, MatchState, 2), - MSt16 = MMod:or_(MSt15, MatchState, ?TERM_PRIMARY_BOXED), - MSt17 = MMod:add(MSt16, NewBSOffsetReg, NumBits), - MSt18 = MMod:free_native_registers(MSt17, [NumBits]), - {J0 - 5, Rest5, MatchState, NewBSOffsetReg, MSt18}; + {MSt9, MatchState} = MMod:and_(MSt8, {free, MatchState}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt10, NewBSOffsetReg} = MMod:get_array_element(MSt9, MatchState, 2), + MSt11 = MMod:or_(MSt10, MatchState, ?TERM_PRIMARY_BOXED), + MSt12 = MMod:add(MSt11, NewBSOffsetReg, NumBits), + MSt13 = MMod:free_native_registers(MSt12, [NumBits]), + {J0 - 5, Rest5, MatchState, NewBSOffsetReg, MSt13}; _ -> - MSt14 = MMod:add(MSt13, BSOffsetReg, NumBits), - MSt15 = MMod:free_native_registers(MSt14, [NumBits]), - {J0 - 5, Rest5, MatchState, BSOffsetReg, MSt15} + MSt9 = MMod:add(MSt8, BSOffsetReg, NumBits), + MSt10 = MMod:free_native_registers(MSt9, [NumBits]), + {J0 - 5, Rest5, MatchState, BSOffsetReg, MSt10} end. first_pass_bs_match_binary( @@ -3185,42 +3248,32 @@ first_pass_bs_match_binary( {Unit, Rest4} = decode_literal(Rest3), ?TRACE("{binary,~p,~p,~p,~p", [Live, _Flags, Size, Unit]), MatchedBits = Size * Unit, - MSt1 = - if - MatchedBits rem 8 =:= 0 -> - cond_raise_badarg({BSOffsetReg, '&', 2#111, '!=', 0}, MMod, MSt0); - true -> - MMod:call_primitive_last(MSt0, ?PRIM_RAISE_ERROR, [ - ctx, jit_state, offset, ?BADARG_ATOM - ]) - end, - MatchedBytes = MatchedBits div 8, - {MSt2, BSOffseBytesReg} = MMod:shift_right(MSt1, BSOffsetReg, 3), - {MSt3, RemainingBytesReg} = MMod:get_array_element(MSt2, BSBinaryReg, 1), - MSt4 = MMod:sub(MSt3, RemainingBytesReg, BSOffseBytesReg), - MSt5 = cond_jump_to_label({RemainingBytesReg, '<', MatchedBytes}, Fail, MMod, MSt4), - MSt6 = MMod:free_native_registers(MSt5, [RemainingBytesReg]), - {MSt7, HeapSizeReg} = MMod:call_primitive(MSt6, ?PRIM_TERM_SUB_BINARY_HEAP_SIZE, [ - BSBinaryReg, MatchedBytes + % Capacity check in bits: a non-byte-aligned offset or length extracts a + % bitstring slice (copying when unaligned, sharing when aligned). + {MSt1, RemainingBitsReg} = term_bit_size({ptr, BSBinaryReg}, MMod, MSt0), + MSt2 = MMod:sub(MSt1, RemainingBitsReg, BSOffsetReg), + MSt3 = cond_jump_to_label({{free, RemainingBitsReg}, '<', MatchedBits}, Fail, MMod, MSt2), + {MSt4, HeapSizeReg} = MMod:call_primitive(MSt3, ?PRIM_BITSTRING_SLICE_HEAP_SIZE, [ + BSBinaryReg, BSOffsetReg, MatchedBits ]), - {MSt8, NewMatchState} = memory_ensure_free_with_extra_root( - MatchState, Live, {free, HeapSizeReg}, MMod, MSt7 + {MSt5, NewMatchState} = memory_ensure_free_with_extra_root( + MatchState, Live, {free, HeapSizeReg}, MMod, MSt4 ), % Restore BSBinaryReg as it may have been gc'd as well - {MSt9, MatchStateReg0} = MMod:copy_to_native_register(MSt8, NewMatchState), - {MSt10, MatchStateReg0} = MMod:and_(MSt9, {free, MatchStateReg0}, ?TERM_PRIMARY_CLEAR_MASK), - MSt11 = MMod:move_array_element(MSt10, MatchStateReg0, 1, BSBinaryReg), - MSt12 = MMod:free_native_registers(MSt11, [MatchStateReg0]), - {MSt13, ResultTerm} = MMod:call_primitive(MSt12, ?PRIM_TERM_MAYBE_CREATE_SUB_BINARY, [ - ctx, BSBinaryReg, {free, BSOffseBytesReg}, MatchedBytes + {MSt6, MatchStateReg0} = MMod:copy_to_native_register(MSt5, NewMatchState), + {MSt7, MatchStateReg0} = MMod:and_(MSt6, {free, MatchStateReg0}, ?TERM_PRIMARY_CLEAR_MASK), + MSt8 = MMod:move_array_element(MSt7, MatchStateReg0, 1, BSBinaryReg), + MSt9 = MMod:free_native_registers(MSt8, [MatchStateReg0]), + {MSt10, ResultTerm} = MMod:call_primitive(MSt9, ?PRIM_BITSTRING_SLICE, [ + ctx, BSBinaryReg, BSOffsetReg, MatchedBits ]), - {MSt14, BSBinaryReg} = MMod:and_(MSt13, {free, BSBinaryReg}, ?TERM_PRIMARY_CLEAR_MASK), - {MSt15, Dest, Rest5} = decode_dest(Rest4, MMod, MSt14), + {MSt11, BSBinaryReg} = MMod:and_(MSt10, {free, BSBinaryReg}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt12, Dest, Rest5} = decode_dest(Rest4, MMod, MSt11), ?TRACE("~p},", [Dest]), - MSt16 = MMod:move_to_vm_register(MSt15, ResultTerm, Dest), - MSt17 = MMod:free_native_registers(MSt16, [ResultTerm]), - MSt18 = MMod:add(MSt17, BSOffsetReg, MatchedBits), - {J0 - 5, Rest5, NewMatchState, BSOffsetReg, MSt18}. + MSt13 = MMod:move_to_vm_register(MSt12, ResultTerm, Dest), + MSt14 = MMod:free_native_registers(MSt13, [ResultTerm]), + MSt15 = MMod:add(MSt14, BSOffsetReg, MatchedBits), + {J0 - 5, Rest5, NewMatchState, BSOffsetReg, MSt15}. first_pass_bs_match_get_tail(MatchState, BSBinaryReg, BSOffsetReg, J0, Rest0, MMod, MSt0) -> {Live, Rest1} = decode_literal(Rest0), @@ -3239,29 +3292,21 @@ first_pass_bs_match_get_tail(MatchState, BSBinaryReg, BSOffsetReg, J0, Rest0, MM do_get_tail( MatchState, Live, BSOffsetReg, BSBinaryReg, MMod, MSt0 ) -> - MSt1 = cond_raise_badarg({BSOffsetReg, '&', 2#111, '!=', 0}, MMod, MSt0), - {MSt2, BSOffseBytesReg} = MMod:shift_right(MSt1, BSOffsetReg, 3), - {MSt3, TailBytesReg0} = MMod:get_array_element(MSt2, BSBinaryReg, 1), - MSt4 = MMod:sub(MSt3, TailBytesReg0, BSOffseBytesReg), - {MSt5, HeapSizeReg} = MMod:call_primitive(MSt4, ?PRIM_TERM_SUB_BINARY_HEAP_SIZE, [ - BSBinaryReg, {free, TailBytesReg0} + {MSt1, HeapSizeReg} = MMod:call_primitive(MSt0, ?PRIM_BITSTRING_GET_TAIL_HEAP_SIZE, [ + BSBinaryReg, BSOffsetReg ]), - {MSt6, NewMatchState} = memory_ensure_free_with_extra_root( - MatchState, Live, {free, HeapSizeReg}, MMod, MSt5 + {MSt2, NewMatchState} = memory_ensure_free_with_extra_root( + MatchState, Live, {free, HeapSizeReg}, MMod, MSt1 ), % Restore BSBinaryReg as it may have been gc'd as well - {MSt7, MatchStateReg0} = MMod:copy_to_native_register(MSt6, NewMatchState), - {MSt8, MatchStateReg0} = MMod:and_(MSt7, {free, MatchStateReg0}, ?TERM_PRIMARY_CLEAR_MASK), - MSt9 = MMod:move_array_element(MSt8, MatchStateReg0, 1, BSBinaryReg), - MSt10 = MMod:free_native_registers(MSt9, [MatchStateReg0]), - {MSt11, BSBinaryReg} = MMod:and_(MSt10, {free, BSBinaryReg}, ?TERM_PRIMARY_CLEAR_MASK), - {MSt12, TailBytesReg1} = MMod:get_array_element(MSt11, BSBinaryReg, 1), - MSt13 = MMod:sub(MSt12, TailBytesReg1, BSOffseBytesReg), - MSt14 = MMod:add(MSt13, BSBinaryReg, ?TERM_PRIMARY_BOXED), - {MSt15, ResultTerm} = MMod:call_primitive(MSt14, ?PRIM_TERM_MAYBE_CREATE_SUB_BINARY, [ - ctx, BSBinaryReg, {free, BSOffseBytesReg}, {free, TailBytesReg1} + {MSt3, MatchStateReg0} = MMod:copy_to_native_register(MSt2, NewMatchState), + {MSt4, MatchStateReg0} = MMod:and_(MSt3, {free, MatchStateReg0}, ?TERM_PRIMARY_CLEAR_MASK), + MSt5 = MMod:move_array_element(MSt4, MatchStateReg0, 1, BSBinaryReg), + MSt6 = MMod:free_native_registers(MSt5, [MatchStateReg0]), + {MSt7, ResultTerm} = MMod:call_primitive(MSt6, ?PRIM_BITSTRING_CREATE_TAIL, [ + ctx, BSBinaryReg, BSOffsetReg ]), - {MSt15, ResultTerm, NewMatchState}. + {MSt7, ResultTerm, NewMatchState}. first_pass_bs_match_equal_colon_equal( Fail, MatchState, BSBinaryReg, BSOffsetReg, J0, Rest0, MMod, MSt0 @@ -4503,7 +4548,10 @@ term_to_int({literal, Val}, _FailLabel, _MMod, MSt0) when is_integer(Val) -> % Optimized case: when we have type information showing this is an integer, skip the type check term_to_int({typed, Term, {t_integer, _Range}}, _FailLabel, MMod, MSt0) -> {MSt1, Reg} = MMod:move_to_native_register(MSt0, Term), - {MSt2, IntReg} = MMod:shift_right(MSt1, {free, Reg}, 4), + %% Untagging is an arithmetic shift: a small integer is stored sign-extended + %% in the upper bits, so a logical shift would turn a negative value into a + %% large positive one (the interpreter shifts a signed avm_int_t). + {MSt2, IntReg} = MMod:shift_right_arith(MSt1, {free, Reg}, 4), {MSt2, IntReg}; term_to_int({typed, Term, _NonIntegerType}, FailLabel, MMod, MSt0) -> % Type information shows it's not an integer, fall back to generic path @@ -4513,7 +4561,7 @@ term_to_int(Term, FailLabel, MMod, MSt0) -> MSt2 = cond_raise_badarg_or_jump_to_fail_label( {Reg, '&', ?TERM_IMMED_TAG_MASK, '!=', ?TERM_INTEGER_TAG}, FailLabel, MMod, MSt1 ), - {MSt3, IntReg} = MMod:shift_right(MSt2, {free, Reg}, 4), + {MSt3, IntReg} = MMod:shift_right_arith(MSt2, {free, Reg}, 4), {MSt3, IntReg}. first_pass_float3(Primitive, Rest0, MMod, MSt0, State0) -> @@ -5051,6 +5099,60 @@ term_put_tuple_element({free, TupleReg}, PosReg, {free, Value}, MMod, MSt0) -> MSt2 = MMod:move_to_array_element(MSt1, Value, TupleReg, PosReg, 1), MMod:free_native_registers(MSt2, [TupleReg, Value]). +%% Total bit size = binary_size * 8 + sub-binary trailing bit count. +%% {free, BinReg} takes a tagged boxed term and consumes it; {ptr, PtrReg} +%% takes an already-tag-cleared pointer and leaves it allocated for the caller. +term_bit_size({free, BinReg}, MMod, MSt0) -> + {MSt1, PtrReg} = MMod:and_(MSt0, {free, BinReg}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt2, BitsReg} = term_bit_size({ptr, PtrReg}, MMod, MSt1), + MSt3 = MMod:free_native_registers(MSt2, [PtrReg]), + {MSt3, BitsReg}; +term_bit_size({ptr, PtrReg}, MMod, MSt0) -> + {MSt1, BitsReg} = MMod:get_array_element(MSt0, PtrReg, 1), + MSt2 = MMod:shift_left(MSt1, BitsReg, 3), + {MSt3, TagReg} = MMod:get_array_element(MSt2, PtrReg, 0), + {MSt4, TagReg} = MMod:and_(MSt3, {free, TagReg}, ?TERM_BOXED_TAG_MASK), + MSt5 = MMod:if_block(MSt4, {{free, TagReg}, '==', ?TERM_BOXED_SUB_BINARY}, fun(BSt0) -> + {BSt1, TrailReg} = MMod:get_array_element(BSt0, PtrReg, 2), + {BSt2, TrailReg} = MMod:and_(BSt1, {free, TrailReg}, 16#7), + BSt3 = MMod:add(BSt2, BitsReg, TrailReg), + MMod:free_native_registers(BSt3, [TrailReg]) + end), + {MSt5, BitsReg}. + +%% Jump to Fail unless BitsReg is a multiple of Unit. Backends have no division, +%% so a power-of-two unit tests the remainder with a mask and any other unit goes +%% through a primitive; the interpreter tests `remaining_bits rem unit` for every +%% unit, and the two must agree. +bs_jump_unless_multiple_of_unit(_BitsReg, 1, _Fail, _MMod, MSt0) -> + MSt0; +bs_jump_unless_multiple_of_unit(BitsReg, Unit, Fail, MMod, MSt0) when Unit band (Unit - 1) =:= 0 -> + cond_jump_to_label({BitsReg, '&', Unit - 1, '!=', 0}, Fail, MMod, MSt0); +bs_jump_unless_multiple_of_unit(BitsReg, Unit, Fail, MMod, MSt0) -> + {MSt1, ResultReg} = MMod:call_primitive(MSt0, ?PRIM_BITSTRING_IS_MULTIPLE_OF, [ + BitsReg, Unit + ]), + cond_jump_to_label({'(bool)', {free, ResultReg}, '==', false}, Fail, MMod, MSt1). + +%% Same divisibility test as bs_jump_unless_multiple_of_unit, but raising badarg +%% when there is no fail label (construction, where BEAM raises rather than +%% failing a match). +bs_badarg_unless_multiple_of_unit(_BitsReg, Unit, _Fail, _MMod, MSt0) when Unit =:= 1 -> + MSt0; +bs_badarg_unless_multiple_of_unit(BitsReg, Unit, Fail, MMod, MSt0) when + Unit band (Unit - 1) =:= 0 +-> + cond_raise_badarg_or_jump_to_fail_label( + {BitsReg, '&', Unit - 1, '!=', 0}, Fail, MMod, MSt0 + ); +bs_badarg_unless_multiple_of_unit(BitsReg, Unit, Fail, MMod, MSt0) -> + {MSt1, ResultReg} = MMod:call_primitive(MSt0, ?PRIM_BITSTRING_IS_MULTIPLE_OF, [ + BitsReg, Unit + ]), + cond_raise_badarg_or_jump_to_fail_label( + {'(bool)', {free, ResultReg}, '==', false}, Fail, MMod, MSt1 + ). + %% @doc Get the stream module %% @return The stream module for jit on this platform -spec stream_module() -> module(). diff --git a/libs/jit/src/primitives.hrl b/libs/jit/src/primitives.hrl index edbff68864..b5ff7b0284 100644 --- a/libs/jit/src/primitives.hrl +++ b/libs/jit/src/primitives.hrl @@ -98,6 +98,12 @@ -define(PRIM_RAW_RAISE, 75). -define(PRIM_RAISE_ERROR_MFA, 76). -define(PRIM_TRY_CASE, 77). +-define(PRIM_BITSTRING_GET_TAIL_HEAP_SIZE, 78). +-define(PRIM_BITSTRING_CREATE_TAIL, 79). +-define(PRIM_BS_CREATE_BIN_WRAP, 80). +-define(PRIM_BITSTRING_SLICE_HEAP_SIZE, 81). +-define(PRIM_BITSTRING_SLICE, 82). +-define(PRIM_BITSTRING_IS_MULTIPLE_OF, 83). % Parameters to ?PRIM_MEMORY_ENSURE_FREE_WITH_ROOTS % -define(MEMORY_NO_SHRINK, 0). diff --git a/libs/jit/src/term.hrl b/libs/jit/src/term.hrl index 2a40a032c6..100b5485c4 100644 --- a/libs/jit/src/term.hrl +++ b/libs/jit/src/term.hrl @@ -41,6 +41,7 @@ -define(TERM_BOXED_REFC_BINARY, 16#20). -define(TERM_BOXED_HEAP_BINARY, 16#24). -define(TERM_BOXED_SUB_BINARY, 16#28). +-define(TERM_BOXED_SUB_BINARY_SIZE, 4). -define(TERM_BOXED_MAP, 16#2C). -define(TERM_BOXED_EXTERNAL_PID, 16#30). -define(TERM_BOXED_EXTERNAL_PORT, 16#34). diff --git a/src/libAtomVM/bif.c b/src/libAtomVM/bif.c index 42cae38ee8..d4f68b9e25 100644 --- a/src/libAtomVM/bif.c +++ b/src/libAtomVM/bif.c @@ -122,10 +122,10 @@ term bif_erlang_byte_size_1(Context *ctx, uint32_t fail_label, int live, term ar if (term_is_match_state(arg1)) { avm_int_t offset = term_get_match_state_offset(arg1); term src_bin = term_get_match_state_binary(arg1); - len = term_binary_size(src_bin) - offset / 8; + len = (term_bit_size(src_bin) - offset + 7) / 8; } else { - VALIDATE_VALUE_BIF(fail_label, arg1, term_is_binary); - len = term_binary_size(arg1); + VALIDATE_VALUE_BIF(fail_label, arg1, term_is_bitstring); + len = (term_bit_size(arg1) + 7) / 8; } return term_from_int(len); @@ -140,10 +140,10 @@ term bif_erlang_bit_size_1(Context *ctx, uint32_t fail_label, int live, term arg if (term_is_match_state(arg1)) { avm_int_t offset = term_get_match_state_offset(arg1); term src_bin = term_get_match_state_binary(arg1); - len = term_binary_size(src_bin) * 8 - offset; + len = term_bit_size(src_bin) - offset; } else { - VALIDATE_VALUE_BIF(fail_label, arg1, term_is_binary); - len = term_binary_size(arg1) * 8; + VALIDATE_VALUE_BIF(fail_label, arg1, term_is_bitstring); + len = term_bit_size(arg1); } return term_from_int(len); @@ -189,6 +189,14 @@ term bif_erlang_is_binary_1(Context *ctx, uint32_t fail_label, term arg1) return term_is_binary(arg1) ? TRUE_ATOM : FALSE_ATOM; } +term bif_erlang_is_bitstring_1(Context *ctx, uint32_t fail_label, term arg1) +{ + UNUSED(ctx); + UNUSED(fail_label); + + return term_is_bitstring(arg1) ? TRUE_ATOM : FALSE_ATOM; +} + term bif_erlang_is_boolean_1(Context *ctx, uint32_t fail_label, term arg1) { UNUSED(ctx); @@ -2100,9 +2108,12 @@ term bif_erlang_max_2(Context *ctx, uint32_t fail_label, term arg1, term arg2) term bif_erlang_size_1(Context *ctx, uint32_t fail_label, int live, term arg1) { - if (term_is_binary(arg1)) { - // For bitstrings, number of bytes is rounded down - return bif_erlang_byte_size_1(ctx, fail_label, live, arg1); + UNUSED(live); + + if (term_is_bitstring(arg1)) { + // For bitstrings, number of bytes is rounded down, + // unlike byte_size/1 which rounds up + return term_from_int(term_binary_size(arg1)); } else if (term_is_tuple(arg1)) { return bif_erlang_tuple_size_1(ctx, fail_label, arg1); } diff --git a/src/libAtomVM/bif.h b/src/libAtomVM/bif.h index 15e5971f1c..2992af53cd 100644 --- a/src/libAtomVM/bif.h +++ b/src/libAtomVM/bif.h @@ -51,6 +51,7 @@ term bif_erlang_length_1(Context *ctx, uint32_t fail_label, int live, term arg1) term bif_erlang_is_atom_1(Context *ctx, uint32_t fail_label, term arg1); term bif_erlang_is_binary_1(Context *ctx, uint32_t fail_label, term arg1); +term bif_erlang_is_bitstring_1(Context *ctx, uint32_t fail_label, term arg1); term bif_erlang_is_boolean_1(Context *ctx, uint32_t fail_label, term arg1); term bif_erlang_is_float_1(Context *ctx, uint32_t fail_label, term arg1); term bif_erlang_is_function_1(Context *ctx, uint32_t fail_label, term arg1); diff --git a/src/libAtomVM/bifs.gperf b/src/libAtomVM/bifs.gperf index 58ef245fdc..88c97aef90 100644 --- a/src/libAtomVM/bifs.gperf +++ b/src/libAtomVM/bifs.gperf @@ -49,7 +49,7 @@ erlang:bit_size/1, {.gcbif.base.type = GCBIFFunctionType, .gcbif.gcbif1_ptr = bi erlang:binary_part/3, {.gcbif.base.type = GCBIFFunctionType, .gcbif.gcbif3_ptr = bif_erlang_binary_part_3} erlang:get/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_get_1} erlang:is_atom/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_atom_1} -erlang:is_bitstring/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_binary_1} +erlang:is_bitstring/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_bitstring_1} erlang:is_binary/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_binary_1} erlang:is_boolean/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_boolean_1} erlang:is_float/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_float_1} diff --git a/src/libAtomVM/bitstring.c b/src/libAtomVM/bitstring.c index 9c279c6d05..d1403d6dd8 100644 --- a/src/libAtomVM/bitstring.c +++ b/src/libAtomVM/bitstring.c @@ -24,11 +24,6 @@ #include #include -static inline uint64_t from_le64(uint64_t value) -{ - return ((((value) &0xFF) << 56) | (((value) &0xFF00) << 40) | (((value) &0xFF0000) << 24) | (((value) &0xFF000000) << 8) | (((value) &0xFF00000000) >> 8) | (((value) &0xFF0000000000) >> 24) | (((value) &0xFF000000000000) >> 40) | (((value) &0xFF00000000000000) >> 56)); -} - bool bitstring_extract_any_integer(const uint8_t *src, size_t offset, avm_int_t n, enum BitstringFlags bs_flags, union maybe_unsigned_int64 *dst) { @@ -46,11 +41,23 @@ bool bitstring_extract_any_integer(const uint8_t *src, size_t offset, avm_int_t } if (bs_flags & LittleEndianIntegerMask) { - out = from_le64(out) >> (64 - n); + // Inverse of the little-endian insertion layout: the field stores the + // complete low-order bytes first (LSB byte first), then the remaining + // n rem 8 high-order bits. + size_t rem = (size_t) n & 0x7; + uint64_t raw = out; + uint64_t value = raw & ((((uint64_t) 1) << rem) - 1); + raw >>= rem; + for (size_t j = 0; j < ((size_t) n >> 3); ++j) { + value = (value << 8) | (raw & 0xFF); + raw >>= 8; + } + out = value; } if ((bs_flags & SignedInteger) && (out & ((uint64_t) 1) << (i - 1))) { - dst->u = ((uint64_t) 0xFFFFFFFFFFFFFFFF << i) | out; + // i == 64 leaves no sign bits to extend, and shifting by the type width is UB. + dst->u = (i < 64) ? (((uint64_t) 0xFFFFFFFFFFFFFFFF << i) | out) : out; } else { dst->u = out; } @@ -58,6 +65,23 @@ bool bitstring_extract_any_integer(const uint8_t *src, size_t offset, avm_int_t return true; } +// Write the low n bits of value into dst starting at bit offset, MSB first. +// Only set bits are written: the destination is assumed to be zero-filled. +static void insert_bits_msb_first(uint8_t *dst, size_t offset, uint64_t value, size_t n) +{ + for (size_t i = 0; i < n; ++i) { + size_t k = (n - 1) - i; + int bit_val = (value & (0x01ULL << k)) >> k; + if (bit_val) { + size_t bit_pos = offset + i; + size_t byte_pos = bit_pos >> 3; // div 8 + uint8_t *pos = dst + byte_pos; + int shift = 7 - (bit_pos & 7); // mod 8 + *pos ^= (0x01 << shift); + } + } +} + bool bitstring_insert_any_integer(uint8_t *dst, avm_int_t offset, avm_int64_t value, size_t n, enum BitstringFlags bs_flags) { // SignedInteger flag does not affect insertion (caller handles sign extension) @@ -78,36 +102,27 @@ bool bitstring_insert_any_integer(uint8_t *dst, avm_int_t offset, avm_int64_t va dst[byte_offset + i] = 0; } } + } else if (little_endian) { + // low-order bytes first (LSB byte first), then remaining high-order bits + size_t whole_bytes = n >> 3; + size_t rem = n & 0x7; + uint64_t uvalue = (uint64_t) value; + for (size_t i = 0; i < whole_bytes; ++i) { + insert_bits_msb_first(dst, offset + 8 * i, uvalue & 0xFF, 8); + uvalue >>= 8; + } + if (rem != 0) { + insert_bits_msb_first(dst, offset + 8 * whole_bytes, uvalue, rem); + } } else { - // Big-endian (or unaligned): write bits MSB first + // Big-endian: write bits MSB first uint64_t write_value = (uint64_t) value; - if (little_endian) { - // Byte-swap the value so MSB-first bit writing produces little-endian bytes - size_t num_bytes = (n + 7) >> 3; - uint64_t swapped = 0; - for (size_t i = 0; i < num_bytes; ++i) { - swapped = (swapped << 8) | (write_value & 0xFF); - write_value >>= 8; - } - write_value = swapped; - } - // value is truncated to 64 bits if (n > 8 * sizeof(value)) { offset += n - (8 * sizeof(value)); n = 8 * sizeof(value); } - for (size_t i = 0; i < n; ++i) { - size_t k = (n - 1) - i; - int bit_val = (write_value & (0x01ULL << k)) >> k; - if (bit_val) { - int bit_pos = offset + i; - int byte_pos = bit_pos >> 3; // div 8 - uint8_t *pos = (uint8_t *) (dst + byte_pos); - int shift = 7 - (bit_pos & 7); // mod 8 - *pos ^= (0x01 << shift); - } - } + insert_bits_msb_first(dst, offset, write_value, n); } return true; } @@ -324,39 +339,92 @@ void bitstring_copy_bits_incomplete_bytes(uint8_t *dst, size_t bits_offset, cons { size_t byte_offset = bits_offset / 8; size_t bit_offset = bits_offset - (8 * byte_offset); - if (bits_offset % 8 == 0 && bits_count >= 8) { + if (bit_offset == 0 && bits_count >= 8) { size_t bytes_count = bits_count / 8; memcpy(dst + byte_offset, src, bytes_count); src += bytes_count; byte_offset += bytes_count; bits_count -= bytes_count * 8; } - // Eventually copy bit by bit + if (bits_count == 0) { + return; + } + // Eventually copy bit by bit. Only the bytes that hold a copied bit may be + // read or written: the destination may be allocated to exactly the size of + // the bits it holds (a refc binary is), and the last requested bit may fall + // exactly on a byte boundary of either buffer. dst += byte_offset; uint8_t dest_byte = *dst; - uint8_t src_byte = *src++; - int dest_bit_ix = 7 - bit_offset; - int src_bit_ix = 7; - while (bits_count > 0) { - if (src_byte & (1 << src_bit_ix)) { + int dest_bit_ix = 7 - (int) bit_offset; + for (size_t i = 0; i < bits_count; i++) { + if (src[i / 8] & (1 << (7 - (i % 8)))) { dest_byte |= 1 << dest_bit_ix; } else { dest_byte &= ~(1 << dest_bit_ix); } if (dest_bit_ix == 0) { *dst++ = dest_byte; - dest_byte = *dst; - dest_bit_ix = 8; + if (i + 1 < bits_count) { + dest_byte = *dst; + } + dest_bit_ix = 7; + } else { + dest_bit_ix--; } - if (src_bit_ix == 0) { - src_byte = *src++; - src_bit_ix = 8; + } + // The last byte was already written if the final bit completed it + if (dest_bit_ix != 7) { + *dst = dest_byte; + } +} + +void bitstring_copy_bits_from(uint8_t *dst, const uint8_t *src, size_t src_offset, size_t bits_count) +{ + for (size_t i = 0; i < bits_count; i++) { + size_t s = src_offset + i; + if (src[s / 8] & (uint8_t) (1 << (7 - (s % 8)))) { + dst[i / 8] |= (uint8_t) (1 << (7 - (i % 8))); } - dest_bit_ix--; - src_bit_ix--; - bits_count--; } - *dst = dest_byte; +} + +size_t bitstring_slice_heap_size(term bs_bin, size_t offset, size_t len_bits) +{ + if (offset % 8 == 0 && len_bits % 8 == 0) { + return term_sub_binary_heap_size(bs_bin, len_bits / 8); + } + size_t words = term_binary_heap_size((len_bits + 7) / 8); + if (len_bits % 8 != 0) { + words += TERM_BOXED_SUB_BINARY_SIZE; + } + return words; +} + +term bitstring_slice(term bs_bin, size_t offset, size_t len_bits, Heap *heap, GlobalContext *glb) +{ + if (offset % 8 == 0 && len_bits % 8 == 0) { + return term_maybe_create_sub_binary(bs_bin, offset / 8, len_bits / 8, heap, glb); + } + size_t result_bytes = (len_bits + 7) / 8; + // term_create_empty_binary zero-fills, as bitstring_copy_bits_from requires + term bin = term_create_empty_binary(result_bytes, heap, glb); + uint8_t *dst = (uint8_t *) term_binary_data(bin); + bitstring_copy_bits_from(dst, (const uint8_t *) term_binary_data(bs_bin), offset, len_bits); + size_t trailing = len_bits % 8; + if (trailing != 0) { + return term_alloc_sub_binary_bits(bin, 0, len_bits / 8, (uint8_t) trailing, heap); + } + return bin; +} + +size_t bitstring_get_tail_heap_size(term bs_bin, size_t bs_offset) +{ + return bitstring_slice_heap_size(bs_bin, bs_offset, term_bit_size(bs_bin) - bs_offset); +} + +term bitstring_get_tail(term bs_bin, size_t bs_offset, Heap *heap, GlobalContext *glb) +{ + return bitstring_slice(bs_bin, bs_offset, term_bit_size(bs_bin) - bs_offset, heap, glb); } bool bitstring_extract_f16( diff --git a/src/libAtomVM/bitstring.h b/src/libAtomVM/bitstring.h index 2c38cc43f7..66af0b31f8 100644 --- a/src/libAtomVM/bitstring.h +++ b/src/libAtomVM/bitstring.h @@ -116,8 +116,8 @@ bool bitstring_insert_any_integer(uint8_t *dst, avm_int_t offset, avm_int64_t va static inline bool bitstring_extract_integer(term src_bin, size_t offset, avm_int_t n, enum BitstringFlags bs_flags, union maybe_unsigned_int64 *dst) { - unsigned long capacity = term_binary_size(src_bin); - if (8 * capacity - offset < (unsigned long) n) { + unsigned long capacity_bits = term_bit_size(src_bin); + if (capacity_bits - offset < (unsigned long) n) { return false; } @@ -486,6 +486,20 @@ static inline bool bitstring_match_utf32(term src_bin, size_t offset, int32_t *c */ void bitstring_copy_bits_incomplete_bytes(uint8_t *dst, size_t bits_offset, const uint8_t *src, size_t bits_count); +/** + * @brief Copy bits_count bits starting at bit src_offset of src into dst[0..]. + * + * @details Used to materialize a non-byte-aligned bitstring slice (e.g. a + * matched tail whose start is not a byte boundary) into a fresh byte-aligned + * buffer. The destination must be pre-zeroed; only set bits are written. + * + * @param dst destination buffer (pre-zeroed), receiving bits at offset 0 + * @param src source buffer + * @param src_offset offset in bits in the source buffer + * @param bits_count number of bits to copy + */ +void bitstring_copy_bits_from(uint8_t *dst, const uint8_t *src, size_t src_offset, size_t bits_count); + /** * @brief Copy bits_count bits from src to dst[bits_offset..] * @@ -519,6 +533,38 @@ bool bitstring_insert_f32( bool bitstring_insert_f64( term dst_bin, size_t offset, avm_float_t value, enum BitstringFlags bs_flags); +/** + * @brief Heap words needed to extract a len_bits slice at bit offset offset. + * @details Worst case: a byte-aligned offset and length shares storage via a + * sub-binary; otherwise the bits are copied into a fresh (possibly bitstring) + * binary. Pair with bitstring_slice. + */ +size_t bitstring_slice_heap_size(term bs_bin, size_t offset, size_t len_bits); + +/** + * @brief Extract a len_bits slice of bs_bin starting at bit offset offset. + * @details Byte-aligned offset+length shares storage via a sub-binary; a + * non-byte-aligned offset or length copies the bits into a fresh bitstring. + * Heap must already be reserved (see bitstring_slice_heap_size). + */ +term bitstring_slice(term bs_bin, size_t offset, size_t len_bits, Heap *heap, GlobalContext *glb); + +/** + * @brief Heap words needed to extract the tail of a match at bit offset bs_offset. + * @details Worst case: a byte-aligned start and length shares storage via a + * sub-binary; otherwise the remaining bits are copied into a fresh (possibly + * bitstring) binary. Pair with bitstring_get_tail. + */ +size_t bitstring_get_tail_heap_size(term bs_bin, size_t bs_offset); + +/** + * @brief Materialize the tail of a match at bit offset bs_offset. + * @details Byte-aligned start+length shares storage via a sub-binary; a + * non-byte-aligned start or length copies the remaining bits into a fresh + * bitstring. Heap must already be reserved (see bitstring_get_tail_heap_size). + */ +term bitstring_get_tail(term bs_bin, size_t bs_offset, Heap *heap, GlobalContext *glb); + #ifdef __cplusplus } #endif diff --git a/src/libAtomVM/ets_multimap.c b/src/libAtomVM/ets_multimap.c index a16cac8b32..17408f9872 100644 --- a/src/libAtomVM/ets_multimap.c +++ b/src/libAtomVM/ets_multimap.c @@ -708,7 +708,7 @@ static uint32_t hash_term_incr(term t, uint32_t h, GlobalContext *global) return hash_local_reference(t, h, global); } else if (term_is_external_reference(t)) { return hash_external_reference(t, h, global); - } else if (term_is_binary(t)) { + } else if (term_is_bitstring(t)) { return hash_binary(t, h, global); } else if (term_is_tuple(t)) { size_t arity = term_get_tuple_arity(t); diff --git a/src/libAtomVM/external_term.c b/src/libAtomVM/external_term.c index e0da90a042..dc0d44f5ae 100644 --- a/src/libAtomVM/external_term.c +++ b/src/libAtomVM/external_term.c @@ -53,6 +53,7 @@ #define STRING_EXT 107 #define LIST_EXT 108 #define BINARY_EXT 109 +#define BIT_BINARY_EXT 77 #define SMALL_BIG_EXT 110 #define NEW_FUN_EXT 112 #define EXPORT_EXT 113 @@ -70,6 +71,7 @@ #define STRING_EXT_BASE_SIZE 3 #define LIST_EXT_BASE_SIZE 5 #define BINARY_EXT_BASE_SIZE 5 +#define BIT_BINARY_EXT_BASE_SIZE 6 #define MAP_EXT_BASE_SIZE 5 #define SMALL_ATOM_EXT_BASE_SIZE 2 @@ -380,17 +382,33 @@ static int serialize_term(uint8_t *buf, term t, GlobalContext *glb) } return k; - } else if (term_is_binary(t)) { - if (!IS_NULL_PTR(buf)) { - buf[0] = BINARY_EXT; - } - size_t len = term_binary_size(t); - if (!IS_NULL_PTR(buf)) { - const uint8_t *data = (const uint8_t *) term_binary_data(t); - WRITE_32_UNALIGNED(buf + 1, len); - memcpy(buf + 5, data, len); + } else if (term_is_bitstring(t)) { + if (term_is_binary(t)) { + size_t len = term_binary_size(t); + if (!IS_NULL_PTR(buf)) { + buf[0] = BINARY_EXT; + const uint8_t *data = (const uint8_t *) term_binary_data(t); + WRITE_32_UNALIGNED(buf + 1, len); + memcpy(buf + 5, data, len); + } + return 5 + len; + } else { + size_t total_bits = term_bit_size(t); + size_t nbytes = (total_bits + 7) / 8; + uint8_t last_byte_bits = (uint8_t) (total_bits % 8); + if (!IS_NULL_PTR(buf)) { + buf[0] = BIT_BINARY_EXT; + const uint8_t *data = (const uint8_t *) term_binary_data(t); + WRITE_32_UNALIGNED(buf + 1, nbytes); + buf[5] = last_byte_bits; + memcpy(buf + 6, data, nbytes); + // Zero the insignificant low bits of the final byte (nbytes is + // at least 1 here since last_byte_bits is 1..7) so equal + // bitstrings have equal external encodings, as OTP does. + buf[6 + nbytes - 1] &= (uint8_t) (0xFF << (8 - last_byte_bits)); + } + return 6 + nbytes; } - return 5 + len; } else if (term_is_map(t)) { size_t size = term_get_map_size(t); @@ -828,6 +846,32 @@ static term parse_external_terms(const uint8_t *external_term_buf, size_t *eterm } } + case BIT_BINARY_EXT: { + uint32_t binary_size = READ_32_UNALIGNED(external_term_buf + 1); + uint8_t last_byte_bits = external_term_buf[5]; + // last_byte_bits is the number of significant bits in the trailing + // byte: 1..8 for a non-empty binary (8 means byte-aligned), 0 for + // an empty binary. Reject anything else, matching OTP. + if (UNLIKELY(last_byte_bits > 8 || ((last_byte_bits == 0) != (binary_size == 0)))) { + return term_invalid_term(); + } + *eterm_size = 6 + binary_size; + term bin; + if (copy) { + bin = term_from_literal_binary((uint8_t *) external_term_buf + 6, binary_size, heap, glb); + } else { + bin = term_from_const_binary((uint8_t *) external_term_buf + 6, binary_size, heap, glb); + } + if (UNLIKELY(term_is_invalid_term(bin))) { + return bin; + } + // 8 significant bits (or an empty binary) means byte-aligned. + if (last_byte_bits >= 8 || binary_size == 0) { + return bin; + } + return term_alloc_sub_binary_bits(bin, 0, binary_size - 1, last_byte_bits, heap); + } + case EXPORT_EXT: { size_t buf_pos = 1; size_t element_size; @@ -1296,6 +1340,43 @@ static int calculate_heap_usage(const uint8_t *external_term_buf, size_t remaini } } + case BIT_BINARY_EXT: { + if (UNLIKELY(remaining < BIT_BINARY_EXT_BASE_SIZE)) { + return INVALID_TERM_SIZE; + } + uint32_t binary_size = READ_32_UNALIGNED(external_term_buf + 1); + uint8_t last_byte_bits = external_term_buf[5]; + // See BIT_BINARY_EXT in parse_external_terms: 1..8 significant bits + // for a non-empty binary, 0 for an empty one. + if (UNLIKELY(last_byte_bits > 8 || ((last_byte_bits == 0) != (binary_size == 0)))) { + return INVALID_TERM_SIZE; + } + remaining -= BIT_BINARY_EXT_BASE_SIZE; + if (UNLIKELY(remaining < binary_size)) { + return INVALID_TERM_SIZE; + } + *eterm_size = BIT_BINARY_EXT_BASE_SIZE + binary_size; + +#if TERM_BYTES == 4 + int size_in_terms = ((binary_size + 4 - 1) >> 2); +#elif TERM_BYTES == 8 + int size_in_terms = ((binary_size + 8 - 1) >> 3); +#else +#error +#endif + + size_t words; + if (copy && term_binary_size_is_heap_binary(binary_size)) { + words = 2 + size_in_terms; + } else { + words = TERM_BOXED_REFC_BINARY_SIZE; + } + if (last_byte_bits < 8 && binary_size != 0) { + words += TERM_BOXED_SUB_BINARY_SIZE; + } + return words; + } + case EXPORT_EXT: { if (UNLIKELY(remaining < 1)) { return INVALID_TERM_SIZE; diff --git a/src/libAtomVM/jit.c b/src/libAtomVM/jit.c index 8d8bd5b4f8..f273725bd7 100644 --- a/src/libAtomVM/jit.c +++ b/src/libAtomVM/jit.c @@ -1510,7 +1510,10 @@ static term jit_bitstring_extract_integer( return extract_bigint( ctx, jit_state, int_bytes + byte_offset, n / 8, bitstring_flags_to_intn_opts(bs_flags)); } else { - return FALSE_ATOM; + // A >64-bit field at a non-byte-aligned offset (or non-byte-multiple + // size) is not extracted yet, as on the construction side. + set_error(ctx, jit_state, 0, UNSUPPORTED_ATOM); + return term_invalid_term(); } } @@ -1551,6 +1554,44 @@ static term jit_term_maybe_create_sub_binary(Context *ctx, term binary, size_t o return term_maybe_create_sub_binary(binary, offset, len, &ctx->heap, ctx->global); } +static size_t jit_bitstring_get_tail_heap_size(term *bs_bin_ptr, size_t bs_offset) +{ + term bs_bin = (term) (((uintptr_t) bs_bin_ptr) | TERM_PRIMARY_BOXED); + return bitstring_get_tail_heap_size(bs_bin, bs_offset); +} + +static term jit_bs_create_bin_wrap(Context *ctx, term byte_binary, size_t total_bits) +{ + if (total_bits % 8 == 0) { + return byte_binary; + } + return term_alloc_sub_binary_bits(byte_binary, 0, total_bits / 8, (uint8_t) (total_bits % 8), &ctx->heap); +} + +static term jit_bitstring_create_tail(Context *ctx, term bs_bin, size_t bs_offset) +{ + return bitstring_get_tail(bs_bin, bs_offset, &ctx->heap, ctx->global); +} + +static size_t jit_bitstring_slice_heap_size(term *bs_bin_ptr, size_t offset, size_t len_bits) +{ + term bs_bin = (term) (((uintptr_t) bs_bin_ptr) | TERM_PRIMARY_BOXED); + return bitstring_slice_heap_size(bs_bin, offset, len_bits); +} + +// Heap is assumed already reserved (see jit_bitstring_slice_heap_size); does not GC. +static term jit_bitstring_slice(Context *ctx, term bs_bin, size_t offset, size_t len_bits) +{ + return bitstring_slice(bs_bin, offset, len_bits, &ctx->heap, ctx->global); +} + +// Backends have no division, so an `all` segment with a non-power-of-two unit +// cannot test its remainder with an & (unit - 1) mask. +static bool jit_bitstring_is_multiple_of(size_t bits, size_t unit) +{ + return unit != 0 && bits % unit == 0; +} + static int jit_term_find_map_pos(Context *ctx, term map, term key) { return term_find_map_pos(map, key, ctx->global); @@ -1641,6 +1682,9 @@ static bool jit_bitstring_insert_integer(term bin, size_t offset, term value, si avm_uint64_t int_value = term_maybe_unbox_int64(value); return bitstring_insert_integer(bin, offset, int_value, n, flags); + } else if ((offset % 8 != 0) || (n % 8 != 0)) { + // Reject bitstrings for bigint + return false; } else { const intn_digit_t *big_src_value = NULL; size_t big_len = 0; @@ -1689,23 +1733,21 @@ static void jit_bitstring_copy_module_str(Context *ctx, JITState *jit_state, ter static int jit_bitstring_copy_binary(Context *ctx, JITState *jit_state, term t, size_t offset, term src, term size) { TRACE("jit_bitstring_copy_binary: t=%p offset=%d src=%p size=%p\n", (void *) t, (int) offset, (void *) src, (void *) size); - if (offset % 8) { - set_error(ctx, jit_state, 0, UNSUPPORTED_ATOM); - return -1; - } - uint8_t *dst = (uint8_t *) term_binary_data(t) + (offset / 8); + uint8_t *dst = (uint8_t *) term_binary_data(t); const uint8_t *bin = (const uint8_t *) term_binary_data(src); - size_t binary_size = term_binary_size(src); + size_t src_bits = term_bit_size(src); + size_t copy_bits = src_bits; if (size != ALL_ATOM) { - binary_size = (size_t) term_to_int(size); - if (binary_size % 8) { - set_error(ctx, jit_state, 0, UNSUPPORTED_ATOM); + copy_bits = (size_t) term_to_int(size); + if (UNLIKELY(copy_bits > src_bits)) { + // Capacity is verified by the emitted size-computation pass; this + // is a defensive check only. + set_error(ctx, jit_state, 0, BADARG_ATOM); return -1; } - binary_size = binary_size / 8; } - memcpy(dst, bin, binary_size); - return binary_size * 8; + bitstring_copy_bits(dst, offset, bin, copy_bits); + return (int) copy_bits; } static Context *jit_apply(Context *ctx, JITState *jit_state, int offset, term module, term function, unsigned int arity) @@ -1880,7 +1922,8 @@ static bool jit_bitstring_match_module_str(Context *ctx, JITState *jit_state, te size_t remaining = 0; const uint8_t *str = module_get_str(jit_state->module, str_id, &remaining); - if (term_binary_size(bs_bin) * 8 - bs_offset < MIN(remaining * 8, bits)) { + size_t capacity_bits = term_bit_size(bs_bin); + if (bs_offset > capacity_bits || capacity_bits - bs_offset < MIN(remaining * 8, bits)) { return false; } @@ -2067,7 +2110,13 @@ const ModuleNativeInterface module_native_interface = { jit_bitstring_insert_float, jit_raw_raise, jit_raise_error_mfa, - jit_try_case + jit_try_case, + jit_bitstring_get_tail_heap_size, + jit_bitstring_create_tail, + jit_bs_create_bin_wrap, + jit_bitstring_slice_heap_size, + jit_bitstring_slice, + jit_bitstring_is_multiple_of }; #endif diff --git a/src/libAtomVM/jit.h b/src/libAtomVM/jit.h index 5a95fa75da..cb68838c53 100644 --- a/src/libAtomVM/jit.h +++ b/src/libAtomVM/jit.h @@ -249,6 +249,12 @@ struct ModuleNativeInterface Context *(*raise_error_mfa)( Context *ctx, JITState *jit_state, int offset, int function_atom_index, int arity); void (*try_case)(Context *ctx); + size_t (*bitstring_get_tail_heap_size)(term *bs_bin_ptr, size_t bs_offset); + term (*bitstring_create_tail)(Context *ctx, term bs_bin, size_t bs_offset); + term (*bs_create_bin_wrap)(Context *ctx, term byte_binary, size_t total_bits); + size_t (*bitstring_slice_heap_size)(term *bs_bin_ptr, size_t offset, size_t len_bits); + term (*bitstring_slice)(Context *ctx, term bs_bin, size_t offset, size_t len_bits); + bool (*bitstring_is_multiple_of)(size_t bits, size_t unit); }; extern const ModuleNativeInterface module_native_interface; diff --git a/src/libAtomVM/nifs.c b/src/libAtomVM/nifs.c index e1926134c1..ca8e30983d 100644 --- a/src/libAtomVM/nifs.c +++ b/src/libAtomVM/nifs.c @@ -172,6 +172,7 @@ static term nif_erlang_binary_to_atom_1(Context *ctx, int argc, term argv[]); static term nif_erlang_binary_to_float_1(Context *ctx, int argc, term argv[]); static term nif_erlang_binary_to_integer(Context *ctx, int argc, term argv[]); static term nif_erlang_binary_to_list_1(Context *ctx, int argc, term argv[]); +static term nif_erlang_bitstring_to_list_1(Context *ctx, int argc, term argv[]); static term nif_erlang_binary_to_existing_atom_1(Context *ctx, int argc, term argv[]); static term nif_erlang_concat_2(Context *ctx, int argc, term argv[]); static term nif_erlang_display_1(Context *ctx, int argc, term argv[]); @@ -404,6 +405,11 @@ static const struct Nif binary_to_list_nif = { .nif_ptr = nif_erlang_binary_to_list_1 }; +static const struct Nif bitstring_to_list_nif = { + .base.type = NIFFunctionType, + .nif_ptr = nif_erlang_bitstring_to_list_1 +}; + static const struct Nif binary_to_existing_atom_1_nif = { .base.type = NIFFunctionType, .nif_ptr = nif_erlang_binary_to_existing_atom_1 @@ -2485,20 +2491,46 @@ static term nif_erlang_list_to_float_1(Context *ctx, int argc, term argv[]) } static term nif_erlang_binary_to_list_1(Context *ctx, int argc, term argv[]) +{ + // binary_to_list/1 requires a byte-aligned binary and raises badarg on a + // non-byte-aligned bitstring, matching OTP; bitstring_to_list/1 shares the + // rest of the implementation. + VALIDATE_VALUE(argv[0], term_is_binary); + return nif_erlang_bitstring_to_list_1(ctx, argc, argv); +} + +static term nif_erlang_bitstring_to_list_1(Context *ctx, int argc, term argv[]) { UNUSED(argc); term value = argv[0]; - VALIDATE_VALUE(value, term_is_binary); + VALIDATE_VALUE(value, term_is_bitstring); int bin_size = term_binary_size(value); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, bin_size * 2, 1, &value, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + // bitstring_to_list/1 on a non-byte-aligned bitstring yields the whole bytes + // followed by a final bitstring element holding the trailing partial byte + // (e.g. bitstring_to_list(<<1:1>>) == [<<1:1>>]). + uint8_t trailing_bits = (uint8_t) (term_bit_size(value) % 8); + + size_t heap_needed = (size_t) bin_size * 2; + if (trailing_bits != 0) { + heap_needed += term_binary_heap_size(1) + TERM_BOXED_SUB_BINARY_SIZE + CONS_SIZE; + } + if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_needed, 1, &value, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } const uint8_t *bin_data = (const uint8_t *) term_binary_data(value); term prev = term_nil(); + if (trailing_bits != 0) { + // The partial byte sits just past the whole bytes; wrap its high + // trailing_bits into a fresh bitstring as the final list element. + term tbin = term_create_empty_binary(1, &ctx->heap, ctx->global); + ((uint8_t *) term_binary_data(tbin))[0] = bin_data[bin_size]; + term tsub = term_alloc_sub_binary_bits(tbin, 0, 0, trailing_bits, &ctx->heap); + prev = term_list_prepend(tsub, prev, &ctx->heap); + } for (int i = bin_size - 1; i >= 0; i--) { prev = term_list_prepend(term_from_int11(bin_data[i]), prev, &ctx->heap); } @@ -3462,7 +3494,7 @@ static term nif_erlang_split_binary(Context *ctx, int argc, term argv[]) term bin_term = argv[0]; term pos_term = argv[1]; - VALIDATE_VALUE(bin_term, term_is_binary); + VALIDATE_VALUE(bin_term, term_is_bitstring); VALIDATE_VALUE(pos_term, term_is_integer); int32_t size = term_binary_size(bin_term); @@ -3472,12 +3504,26 @@ static term nif_erlang_split_binary(Context *ctx, int argc, term argv[]) RAISE_ERROR(BADARG_ATOM); } - size_t alloc_heap_size = term_sub_binary_heap_size(bin_term, pos) + term_sub_binary_heap_size(bin_term, size - pos) + TUPLE_SIZE(2); + // split_binary/2 accepts a bitstring: the second part keeps the trailing + // bits, e.g. split_binary(<<1:9>>, 1) == {<<0>>, <<1:1>>} + uint8_t trailing_bits = (uint8_t) (term_bit_size(bin_term) % 8); + + size_t second_part_heap_size = trailing_bits == 0 + ? term_sub_binary_heap_size(bin_term, size - pos) + : TERM_BOXED_SUB_BINARY_SIZE; + size_t alloc_heap_size = term_sub_binary_heap_size(bin_term, pos) + second_part_heap_size + TUPLE_SIZE(2); if (UNLIKELY(memory_ensure_free_with_roots(ctx, alloc_heap_size, 1, &bin_term, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } term sub_binary_a = term_maybe_create_sub_binary(bin_term, 0, pos, &ctx->heap, ctx->global); - term sub_binary_b = term_maybe_create_sub_binary(bin_term, pos, size - pos, &ctx->heap, ctx->global); + term sub_binary_b; + if (trailing_bits == 0) { + sub_binary_b = term_maybe_create_sub_binary(bin_term, pos, size - pos, &ctx->heap, ctx->global); + } else { + // only a sub-binary can carry trailing bits + size_t offset = term_get_sub_binary_offset(bin_term); + sub_binary_b = term_alloc_sub_binary_bits(bin_term, offset + pos, size - pos, trailing_bits, &ctx->heap); + } term tuple = term_alloc_tuple(2, &ctx->heap); term_put_tuple_element(tuple, 0, sub_binary_a); term_put_tuple_element(tuple, 1, sub_binary_b); diff --git a/src/libAtomVM/nifs.gperf b/src/libAtomVM/nifs.gperf index a7281dbc7e..7541e70b00 100644 --- a/src/libAtomVM/nifs.gperf +++ b/src/libAtomVM/nifs.gperf @@ -65,6 +65,7 @@ erlang:binary_to_float/1, &binary_to_float_nif erlang:binary_to_integer/1, &binary_to_integer_nif erlang:binary_to_integer/2, &binary_to_integer_nif erlang:binary_to_list/1, &binary_to_list_nif +erlang:bitstring_to_list/1, &bitstring_to_list_nif erlang:binary_to_existing_atom/1, &binary_to_existing_atom_1_nif erlang:delete_element/2, &delete_element_nif erlang:erase/0, &erase_0_nif diff --git a/src/libAtomVM/opcodesswitch.h b/src/libAtomVM/opcodesswitch.h index 889ff319a8..2baef81175 100644 --- a/src/libAtomVM/opcodesswitch.h +++ b/src/libAtomVM/opcodesswitch.h @@ -1064,14 +1064,14 @@ static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) } \ } -#define VERIFY_IS_BINARY(t, opcode_name, label) \ - if (UNLIKELY(!term_is_binary(t))) { \ - TRACE(opcode_name ": " #t " is not a binary\n"); \ - if (label == 0) { \ - RAISE_ERROR(BADARG_ATOM); \ - } else { \ - JUMP_TO_LABEL(mod, label); \ - } \ +#define VERIFY_IS_BITSTRING(t, opcode_name, label) \ + if (UNLIKELY(!term_is_bitstring(t))) { \ + TRACE(opcode_name ": " #t " is not a bitstring\n"); \ + if (label == 0) { \ + RAISE_ERROR(BADARG_ATOM); \ + } else { \ + JUMP_TO_LABEL(mod, label); \ + } \ } #define VERIFY_IS_MATCH_STATE(t, opcode_name, label) \ @@ -1084,10 +1084,10 @@ static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) } \ } -#define VERIFY_IS_MATCH_OR_BINARY(t, opcode_name) \ - if (UNLIKELY(!(term_is_binary(t) || term_is_match_state(t)))) { \ - TRACE(opcode_name ": " #t " is not a binary or match context.\n"); \ - RAISE_ERROR(BADARG_ATOM); \ +#define VERIFY_IS_MATCH_OR_BINARY(t, opcode_name) \ + if (UNLIKELY(!(term_is_bitstring(t) || term_is_match_state(t)))) { \ + TRACE(opcode_name ": " #t " is not a bitstring or match context.\n"); \ + RAISE_ERROR(BADARG_ATOM); \ } #define CALL_FUN(fun, args_count) \ @@ -1239,6 +1239,39 @@ static bool sort_kv_pairs(struct kv_pair *kv, int size, GlobalContext *global) } #endif +/** + * @brief Scale a dynamic segment size by its unit, in bits. + * + * @details Matching opcodes take a segment size that is scaled by the segment + * unit. The size comes from a register and can be negative or large enough for + * the product to overflow, so it cannot be multiplied blindly: converting a + * negative value to `size_t` and multiplying can wrap back to a small size that + * then passes the capacity check and matches, where BEAM fails the match. + * + * @param size the segment size, as a term (must be any integer) + * @param unit the segment unit, in bits + * @param max_bits the number of bits available for this segment + * @param size_bits on success, the scaled size, in bits + * @returns \c true if the scaled size is representable and fits within + * \c max_bits, \c false if the match should fail + */ +static inline bool bs_scaled_size_bits(term size, uint32_t unit, size_t max_bits, size_t *size_bits) +{ + if (!term_is_integer(size)) { + // a size that doesn't fit in a small integer cannot fit in max_bits + return false; + } + avm_int_t size_val = term_to_int(size); + if (size_val < 0) { + return false; + } + if (unit != 0 && (size_t) size_val > max_bits / unit) { + return false; + } + *size_bits = (size_t) size_val * unit; + return true; +} + static term maybe_alloc_boxed_integer_fragment(Context *ctx, avm_int64_t value) { #if BOXED_TERMS_REQUIRED_FOR_INT64 > 1 @@ -3398,9 +3431,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_INTEGER(size, "bs_init_bits", 0); avm_int_t size_val = term_to_int(size); - if (size_val % 8 != 0) { - TRACE("bs_init_bits: size_val (" AVM_INT_FMT ") is not evenly divisible by 8\n", size_val); - RAISE_ERROR(UNSUPPORTED_ATOM); + if (UNLIKELY(size_val < 0)) { + TRACE("bs_init_bits: size_val (" AVM_INT_FMT ") is negative\n", size_val); + RAISE_ERROR(BADARG_ATOM); } if (flags_value != 0) { TRACE("bs_init_bits: neither signed nor native or little endian encoding supported.\n"); @@ -3409,18 +3442,34 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_init_bits/6, fail=%i size=" AVM_INT_FMT " words=%u live=%u dreg=%c%i\n", fail, size_val, (unsigned) words, (unsigned) live, T_DEST_REG(dreg)); + // A non-byte-aligned total size yields a bitstring: the whole + // bytes live in a binary and the result wraps it in a sub-binary + // carrying the trailing bit count, as bs_create_bin does. + size_t bs_total_bits = (size_t) size_val; + size_t bs_total_bytes = (bs_total_bits + 7) / 8; + size_t bs_trailing_bits = bs_total_bits % 8; + size_t bs_heap_size = words + term_binary_heap_size(bs_total_bytes); + if (bs_trailing_bits != 0) { + bs_heap_size += TERM_BOXED_SUB_BINARY_SIZE; + } TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, words + term_binary_heap_size(size_val / 8), live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, bs_heap_size, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - term t = term_create_empty_binary(size_val / 8, &ctx->heap, ctx->global); + term t = term_create_empty_binary(bs_total_bytes, &ctx->heap, ctx->global); if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } + // ctx->bs stays the whole binary: the bs_put_* opcodes check + // their capacity in whole bytes, and the trailing partial byte + // is part of it. ctx->bs = t; ctx->bs_offset = 0; + if (bs_trailing_bits != 0) { + t = term_alloc_sub_binary_bits(t, 0, bs_total_bits / 8, (uint8_t) bs_trailing_bits, &ctx->heap); + } WRITE_REGISTER(dreg, t); break; } @@ -3445,37 +3494,55 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); - VERIFY_IS_BINARY(src, "bs_append", 0); + VERIFY_IS_BITSTRING(src, "bs_append", 0); VERIFY_IS_INTEGER(size, "bs_append", 0); avm_int_t size_val = term_to_int(size); - if (size_val % 8 != 0) { - TRACE("bs_append: size_val (" AVM_INT_FMT ") is not evenly divisible by 8\n", size_val); - RAISE_ERROR(UNSUPPORTED_ATOM); + if (UNLIKELY(size_val < 0)) { + TRACE("bs_append: size_val (" AVM_INT_FMT ") is negative\n", size_val); + RAISE_ERROR(BADARG_ATOM); } - if (unit != 8) { - TRACE("bs_append: unit is not equal to 8; unit=%u\n", (unsigned) unit); - RAISE_ERROR(UNSUPPORTED_ATOM); + + // Sizes are bit-granular: size_val is the number of bits to + // append and the source may be a non-byte-aligned bitstring, + // whose trailing bits belong in the result and shift every + // segment written after them. The source size must be a + // multiple of the segment unit, as on BEAM: a /binary segment + // (unit 8) rejects a partial source, a /bitstring one (unit 1) + // accepts it. + size_t src_bits = term_bit_size(src); + if (UNLIKELY(unit == 0 || (src_bits % unit) != 0)) { + TRACE("bs_append: source bit size (%zu) is not a multiple of unit %u\n", src_bits, (unsigned) unit); + RAISE_ERROR(BADARG_ATOM); } - size_t src_size = term_binary_size(src); + size_t bs_total_bits = src_bits + (size_t) size_val; + size_t bs_total_bytes = (bs_total_bits + 7) / 8; + size_t bs_trailing_bits = bs_total_bits % 8; + size_t bs_heap_size = term_binary_heap_size(bs_total_bytes); + if (bs_trailing_bits != 0) { + bs_heap_size += TERM_BOXED_SUB_BINARY_SIZE; + } TRIM_LIVE_REGS(live); x_regs[live] = src; - if (UNLIKELY(memory_ensure_free_with_roots(ctx, term_binary_heap_size(src_size + size_val / 8), live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, bs_heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } TRACE("bs_append/8, fail=%u size=" AVM_INT_FMT " unit=%u src=0x%" TERM_X_FMT " dreg=%c%i\n", (unsigned) fail, size_val, (unsigned) unit, src, T_DEST_REG(dreg)); src = x_regs[live]; - term t = term_create_empty_binary(src_size + size_val / 8, &ctx->heap, ctx->global); + term t = term_create_empty_binary(bs_total_bytes, &ctx->heap, ctx->global); if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - memcpy((void *) term_binary_data(t), (void *) term_binary_data(src), src_size); + bitstring_copy_bits((uint8_t *) term_binary_data(t), 0, (const uint8_t *) term_binary_data(src), src_bits); ctx->bs = t; - ctx->bs_offset = src_size * 8; + ctx->bs_offset = src_bits; + if (bs_trailing_bits != 0) { + t = term_alloc_sub_binary_bits(t, 0, bs_total_bits / 8, (uint8_t) bs_trailing_bits, &ctx->heap); + } WRITE_REGISTER(dreg, t); break; } @@ -3496,29 +3563,56 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); - VERIFY_IS_BINARY(src, "bs_private_append", 0); + VERIFY_IS_BITSTRING(src, "bs_private_append", 0); VERIFY_IS_INTEGER(size, "bs_private_append", 0); avm_int_t size_val = term_to_int(size); - if (size_val % 8 != 0) { - TRACE("bs_private_append: size_val (%li) is not evenly divisible by 8\n", (long int) size_val); - RAISE_ERROR(UNSUPPORTED_ATOM); + if (UNLIKELY(size_val < 0)) { + TRACE("bs_private_append: size_val (%li) is negative\n", (long int) size_val); + RAISE_ERROR(BADARG_ATOM); + } + + size_t src_bits = term_bit_size(src); + if (UNLIKELY(unit == 0 || (src_bits % unit) != 0)) { + TRACE("bs_private_append: source bit size (%zu) is not a multiple of unit %u\n", src_bits, (unsigned) unit); + RAISE_ERROR(BADARG_ATOM); } - size_t src_size = term_binary_size(src); - if (UNLIKELY(memory_ensure_free_opt(ctx, term_binary_heap_size(src_size + size_val / 8), MEMORY_NO_GC) != MEMORY_GC_OK)) { + size_t bs_total_bits = src_bits + (size_t) size_val; + size_t bs_total_bytes = (bs_total_bits + 7) / 8; + size_t bs_trailing_bits = bs_total_bits % 8; + // The binary can only be reused in place when it owns whole + // bytes: term_reuse_binary grows a byte-sized buffer, and the + // source trailing bits would land at the wrong offset. + bool bs_can_reuse = (src_bits % 8) == 0; + size_t bs_heap_size = term_binary_heap_size(bs_total_bytes); + if (bs_trailing_bits != 0) { + bs_heap_size += TERM_BOXED_SUB_BINARY_SIZE; + } + if (UNLIKELY(memory_ensure_free_opt(ctx, bs_heap_size, MEMORY_NO_GC) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } DECODE_COMPACT_TERM(src, src_pc) - term t = term_reuse_binary(src, src_size + size_val / 8, &ctx->heap, ctx->global); + term t; + if (bs_can_reuse) { + t = term_reuse_binary(src, bs_total_bytes, &ctx->heap, ctx->global); + } else { + t = term_create_empty_binary(bs_total_bytes, &ctx->heap, ctx->global); + if (LIKELY(!term_is_invalid_term(t))) { + bitstring_copy_bits((uint8_t *) term_binary_data(t), 0, (const uint8_t *) term_binary_data(src), src_bits); + } + } if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } ctx->bs = t; - ctx->bs_offset = src_size * 8; + ctx->bs_offset = src_bits; TRACE("bs_private_append/6, fail=%u size=" AVM_INT_FMT " unit=%u src=0x%" TERM_X_FMT " dreg=%c%i\n", (unsigned) fail, size_val, (unsigned) unit, src, T_DEST_REG(dreg)); + if (bs_trailing_bits != 0) { + t = term_alloc_sub_binary_bits(t, 0, bs_total_bits / 8, (uint8_t) bs_trailing_bits, &ctx->heap); + } WRITE_REGISTER(dreg, t); break; } @@ -3593,43 +3687,47 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term src; DECODE_COMPACT_TERM(src, pc); - VERIFY_IS_BINARY(src, "bs_put_binary", 0); - unsigned long size_val = 0; + VERIFY_IS_BITSTRING(src, "bs_put_binary", 0); + // Sizes and the destination offset are bit-granular: a partial + // bitstring source contributes its trailing bits and shifts + // every segment written after it. + size_t src_bits = term_bit_size(src); + size_t size_bits = 0; if (term_is_integer(size)) { - avm_int_t bit_size = term_to_int(size) * unit; - if (bit_size % 8 != 0) { - TRACE("bs_put_binary: Bit size must be evenly divisible by 8.\n"); - RAISE_ERROR(UNSUPPORTED_ATOM); + avm_int_t signed_size_value = term_to_int(size); + if (UNLIKELY(signed_size_value < 0)) { + TRACE("bs_put_binary: size is negative\n"); + RAISE_ERROR(BADARG_ATOM); + } + if (UNLIKELY(unit != 0 && (size_t) signed_size_value > src_bits / unit)) { + TRACE("bs_put_binary: size larger than the source bitstring\n"); + RAISE_ERROR(BADARG_ATOM); } - size_val = bit_size / 8; + size_bits = (size_t) signed_size_value * unit; } else if (size == ALL_ATOM) { - size_val = term_binary_size(src); + size_bits = src_bits; } else { TRACE("bs_put_binary: Unsupported size term type in put binary: %p\n", (void *) size); RAISE_ERROR(BADARG_ATOM); } - if (size_val > term_binary_size(src)) { - TRACE("bs_put_binary: binary data size (%li) larger than source binary size (%li)\n", (long) size_val, (long) term_binary_size(src)); + if (UNLIKELY(size_bits > src_bits)) { + TRACE("bs_put_binary: binary data size (%zu bits) larger than source binary size (%zu bits)\n", size_bits, src_bits); RAISE_ERROR(BADARG_ATOM); } if (flags_value != 0) { TRACE("bs_put_binary: neither signed nor native or little endian encoding supported.\n"); RAISE_ERROR(UNSUPPORTED_ATOM); } - - if (ctx->bs_offset % 8 != 0) { - TRACE("bs_put_binary: Unsupported bit syntax operation. Writing binaries must be byte-aligend.\n"); - RAISE_ERROR(UNSUPPORTED_ATOM); + size_t bs_capacity_bits = term_binary_size(ctx->bs) * 8; + if (UNLIKELY(ctx->bs_offset > bs_capacity_bits || bs_capacity_bits - ctx->bs_offset < size_bits)) { + TRACE("bs_put_binary: insufficient capacity to write binary\n"); + RAISE_ERROR(BADARG_ATOM); } - TRACE("bs_put_binary/5, fail=%u size=%li unit=%u flags=%x src=0x%x\n", (unsigned) fail, size_val, (unsigned) unit, (int) flags_value, (unsigned int) src); + TRACE("bs_put_binary/5, fail=%u size=%zu bits unit=%u flags=%x src=0x%x\n", (unsigned) fail, size_bits, (unsigned) unit, (int) flags_value, (unsigned int) src); - int result = term_bs_insert_binary(ctx->bs, ctx->bs_offset, src, size_val); - if (UNLIKELY(result)) { - TRACE("bs_put_binary: Failed to insert binary into binary: %i\n", result); - RAISE_ERROR(BADARG_ATOM); - } - ctx->bs_offset += 8 * size_val; + bitstring_copy_bits((uint8_t *) term_binary_data(ctx->bs), ctx->bs_offset, (const uint8_t *) term_binary_data(src), size_bits); + ctx->bs_offset += size_bits; break; } @@ -3700,7 +3798,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) uint32_t offset; DECODE_LITERAL(offset, pc); - if (UNLIKELY(!term_is_binary(ctx->bs))) { + if (UNLIKELY(!term_is_bitstring(ctx->bs))) { TRACE("bs_put_string: Bad state. ctx->bs is not a binary.\n"); RAISE_ERROR(BADARG_ATOM); } @@ -3751,7 +3849,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_INTEGER(src, "bs_put_utf8/3", 0); avm_int_t src_value = term_to_int(src); TRACE("bs_put_utf8/3 flags=%x, src=0x%lx\n", (int) flags, (long) src_value); - if (UNLIKELY(!term_is_binary(ctx->bs))) { + if (UNLIKELY(!term_is_bitstring(ctx->bs))) { TRACE("bs_put_utf8/3: Bad state. ctx->bs is not a binary.\n"); RAISE_ERROR(BADARG_ATOM); } @@ -3799,7 +3897,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_INTEGER(src, "bs_put_utf16/3", 0); avm_int_t src_value = term_to_int(src); TRACE("bs_put_utf16/3 flags=%x, src=" AVM_INT_FMT "\n", (int) flags, src_value); - if (UNLIKELY(!term_is_binary(ctx->bs))) { + if (UNLIKELY(!term_is_bitstring(ctx->bs))) { TRACE("bs_put_utf16: Bad state. ctx->bs is not a binary.\n"); RAISE_ERROR(BADARG_ATOM); } @@ -3828,7 +3926,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_INTEGER(src, "bs_put_utf32/3", 0); avm_int_t src_value = term_to_int(src); TRACE("bs_put_utf32/3 flags=%x, src=" AVM_INT_FMT "\n", (int) flags, src_value); - if (UNLIKELY(!term_is_binary(ctx->bs))) { + if (UNLIKELY(!term_is_bitstring(ctx->bs))) { TRACE("bs_put_utf32/3: Bad state. ctx->bs is not a binary.\n"); RAISE_ERROR(BADARG_ATOM); } @@ -4062,7 +4160,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_DEST_REGISTER_GC_SAFE(dreg, pc); TRACE("bs_start_match3/4, fail=%i src=0x%" TERM_X_FMT " live=%u dreg=%c%i\n", fail, src, live, T_DEST_REG_GC_SAFE(dreg)); - if (!(term_is_binary(src) || term_is_match_state(src))) { + if (!(term_is_bitstring(src) || term_is_match_state(src))) { pc = mod->labels[fail]; } else { // MEMORY_CAN_SHRINK because bs_start_match is classified as gc in beam_ssa_codegen.erl @@ -4115,29 +4213,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_get_tail/3 src=0x%" TERM_X_FMT " dreg=%c%i live=%u\n", src, T_DEST_REG_GC_SAFE(dreg), live); if (bs_offset == 0) { - WRITE_REGISTER_GC_SAFE(dreg, bs_bin); - } else { - if (bs_offset % 8 != 0) { - TRACE("bs_get_tail: Unsupported alignment.\n"); - RAISE_ERROR(UNSUPPORTED_ATOM); - } else { - size_t start_pos = bs_offset / 8; - size_t src_size = term_binary_size(bs_bin); - size_t new_bin_size = src_size - start_pos; - size_t heap_size = term_sub_binary_heap_size(bs_bin, src_size - start_pos); - - TRIM_LIVE_REGS(live); - x_regs[live] = src; - if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { - RAISE_ERROR(OUT_OF_MEMORY_ATOM); - } - src = x_regs[live]; - bs_bin = term_get_match_state_binary(src); - term t = term_maybe_create_sub_binary(bs_bin, start_pos, new_bin_size, &ctx->heap, ctx->global); - WRITE_REGISTER_GC_SAFE(dreg, t); + size_t heap_size = bitstring_get_tail_heap_size(bs_bin, bs_offset); + TRIM_LIVE_REGS(live); + x_regs[live] = src; + if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); } + src = x_regs[live]; + bs_bin = term_get_match_state_binary(src); + term t = bitstring_get_tail(bs_bin, bs_offset, &ctx->heap, ctx->global); + WRITE_REGISTER_GC_SAFE(dreg, t); } break; } @@ -4181,7 +4268,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(BADARG_ATOM); } - if (term_binary_size(bs_bin) * 8 - bs_offset < MINI(remaining * 8, bits)) { + if (term_bit_size(bs_bin) - bs_offset < MINI(remaining * 8, bits)) { TRACE("bs_match_string: failed to match (binary is shorter)\n"); JUMP_TO_ADDRESS(mod->labels[fail]); } else { @@ -4246,17 +4333,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(flags_value, pc) VERIFY_IS_MATCH_STATE(src, "bs_skip_bits2", 0); - VERIFY_IS_INTEGER(size, "bs_skip_bits2", 0); + VERIFY_IS_ANY_INTEGER(size, "bs_skip_bits2", 0); // Ignore flags value as skipping bits is the same whatever the endianness - avm_int_t size_val = term_to_int(size); + TRACE("bs_skip_bits2/5, fail=%u src=%p size=0x%lx unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned long) term_to_int(size), (unsigned) unit, (int) flags_value); - TRACE("bs_skip_bits2/5, fail=%u src=%p size=0x%lx unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned long) size_val, (unsigned) unit, (int) flags_value); - - size_t increment = size_val * unit; avm_int_t bs_offset = term_get_match_state_offset(src); term bs_bin = term_get_match_state_binary(src); - if ((bs_offset + increment) > term_binary_size(bs_bin) * 8) { - TRACE("bs_skip_bits2: Insufficient capacity to skip bits: %lu, inc: %zu\n", (unsigned long) bs_offset, increment); + size_t bs_capacity = term_bit_size(bs_bin); + size_t increment; + if ((size_t) bs_offset > bs_capacity + || !bs_scaled_size_bits(size, unit, bs_capacity - bs_offset, &increment)) { + TRACE("bs_skip_bits2: Insufficient capacity to skip bits: %lu\n", (unsigned long) bs_offset); JUMP_TO_ADDRESS(mod->labels[fail]); } else { term_set_match_state_offset(src, bs_offset + increment); @@ -4280,7 +4367,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); - if ((term_binary_size(bs_bin) * 8 - bs_offset) % unit != 0) { + if ((term_bit_size(bs_bin) - bs_offset) % unit != 0) { TRACE("bs_test_unit: Available bits in source not evenly divisible by unit\n"); JUMP_TO_ADDRESS(mod->labels[fail]); } @@ -4304,8 +4391,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); - if ((term_binary_size(bs_bin) * 8 - bs_offset) != (unsigned int) bits) { - TRACE("bs_test_tail2: Expected exactly %u bits remaining, but remaining=%u\n", (unsigned) bits, (unsigned) (term_binary_size(bs_bin) * 8 - bs_offset)); + if ((term_bit_size(bs_bin) - bs_offset) != (unsigned int) bits) { + TRACE("bs_test_tail2: Expected exactly %u bits remaining, but remaining=%u\n", (unsigned) bits, (unsigned) (term_bit_size(bs_bin) - bs_offset)); JUMP_TO_ADDRESS(mod->labels[fail]); } break; @@ -4328,16 +4415,22 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(flags_value, pc) VERIFY_IS_MATCH_STATE(src, "bs_get_integer", 0); - VERIFY_IS_INTEGER(size, "bs_get_integer", 0); - - avm_int_t size_val = term_to_int(size); + VERIFY_IS_ANY_INTEGER(size, "bs_get_integer", 0); - TRACE("bs_get_integer2/7, fail=%u src=%p live=%u size=%u unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned) size_val, (unsigned) live, (unsigned) unit, (int) flags_value); + TRACE("bs_get_integer2/7, fail=%u src=%p live=%u size=%u unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned) term_to_int(size), (unsigned) live, (unsigned) unit, (int) flags_value); - avm_int_t increment = size_val * unit; union maybe_unsigned_int64 value; term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); + size_t bs_capacity = term_bit_size(bs_bin); + size_t increment_bits; + if ((size_t) bs_offset > bs_capacity + || !bs_scaled_size_bits(size, unit, bs_capacity - bs_offset, &increment_bits)) { + TRACE("bs_get_integer2: size is negative or exceeds the remaining capacity\n"); + JUMP_TO_ADDRESS(mod->labels[fail]); + } + // bounded by the remaining capacity, so it fits in an avm_int_t + avm_int_t increment = (avm_int_t) increment_bits; term t; if (increment <= 64) { bool status = bitstring_extract_integer(bs_bin, bs_offset, increment, flags_value, &value); @@ -4361,8 +4454,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } } } else if ((bs_offset % 8 == 0) && (increment % 8 == 0) && (increment <= INTN_MAX_UNSIGNED_BITS_SIZE)) { - unsigned long capacity = term_binary_size(bs_bin); - if (8 * capacity - bs_offset < (unsigned long) increment) { + unsigned long capacity_bits = term_bit_size(bs_bin); + if (capacity_bits - bs_offset < (unsigned long) increment) { JUMP_TO_ADDRESS(mod->labels[fail]); } size_t byte_offset = bs_offset / 8; @@ -4401,16 +4494,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(flags_value, pc); VERIFY_IS_MATCH_STATE(src, "bs_get_float", 0); - VERIFY_IS_INTEGER(size, "bs_get_float", 0); - - avm_int_t size_val = term_to_int(size); + VERIFY_IS_ANY_INTEGER(size, "bs_get_float", 0); - TRACE("bs_get_float2/7, fail=%u src=%p size=%u unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned) size_val, (unsigned) unit, (int) flags_value); + TRACE("bs_get_float2/7, fail=%u src=%p unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned) unit, (int) flags_value); - avm_int_t increment = size_val * unit; avm_float_t value; term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); + size_t bs_capacity = term_bit_size(bs_bin); + size_t increment_bits; + if ((size_t) bs_offset > bs_capacity + || !bs_scaled_size_bits(size, unit, bs_capacity - bs_offset, &increment_bits)) { + TRACE("bs_get_float2: size is negative or exceeds the remaining capacity\n"); + JUMP_TO_ADDRESS(mod->labels[fail]); + } + // both bounded by the remaining capacity, so they fit in an avm_int_t + avm_int_t size_val = term_to_int(size); + avm_int_t increment = (avm_int_t) increment_bits; bool status; switch (size_val) { case 16: @@ -4466,40 +4566,49 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); - if (unit != 8) { - TRACE("bs_get_binary2: Unsupported: unit must be 8.\n"); + if (flags_value != 0) { + TRACE("bs_get_binary2: neither signed nor native or little endian encoding supported.\n"); RAISE_ERROR(UNSUPPORTED_ATOM); } - avm_int_t size_val = 0; - if (term_is_integer(size)) { - size_val = term_to_int(size); + + size_t bs_capacity = term_bit_size(bs_bin); + if ((size_t) bs_offset > bs_capacity) { + TRACE("bs_get_binary2: match state offset is past the end of the bitstring\n"); + JUMP_TO_ADDRESS(mod->labels[fail]); + } + size_t remaining_bits = bs_capacity - bs_offset; + size_t size_bits; + if (term_is_any_integer(size)) { + // A negative or overflowing size fails the match, as on BEAM + if (!bs_scaled_size_bits(size, unit, remaining_bits, &size_bits)) { + TRACE("bs_get_binary2: size is negative or exceeds the remaining capacity\n"); + JUMP_TO_ADDRESS(mod->labels[fail]); + } } else if (size == ALL_ATOM) { - size_val = term_binary_size(bs_bin) - bs_offset / 8; + // all takes the whole remainder, which must be a multiple + // of the segment unit + if (remaining_bits % unit != 0) { + TRACE("bs_get_binary2: remainder is not a multiple of unit\n"); + JUMP_TO_ADDRESS(mod->labels[fail]); + } + size_bits = remaining_bits; } else { TRACE("bs_get_binary2: size is neither an integer nor the atom `all`\n"); RAISE_ERROR(BADARG_ATOM); } - if (bs_offset % unit != 0) { - TRACE("bs_get_binary2: Unsupported. Offset on binary read must be aligned on byte boundaries.\n"); - RAISE_ERROR(BADARG_ATOM); - } - if (flags_value != 0) { - TRACE("bs_get_binary2: neither signed nor native or little endian encoding supported.\n"); - RAISE_ERROR(UNSUPPORTED_ATOM); - } TRACE("bs_get_binary2/7, fail=%u src=%p live=%u unit=%u\n", (unsigned) fail, (void *) bs_bin, (unsigned) live, (unsigned) unit); - if ((unsigned int) (bs_offset / unit + size_val) > term_binary_size(bs_bin)) { - TRACE("bs_get_binary2: insufficient capacity -- bs_offset = %d, size_val = %d\n", (int) bs_offset, (int) size_val); + if (size_bits > remaining_bits) { + TRACE("bs_get_binary2: insufficient capacity -- bs_offset = %d, size_bits = %d\n", (int) bs_offset, (int) size_bits); JUMP_TO_ADDRESS(mod->labels[fail]); } else { - term_set_match_state_offset(src, bs_offset + size_val * unit); + term_set_match_state_offset(src, bs_offset + size_bits); TRIM_LIVE_REGS(live); // there is always room for a MAX_REG + 1 register, used as working register x_regs[live] = bs_bin; - size_t heap_size = term_sub_binary_heap_size(bs_bin, size_val); + size_t heap_size = bitstring_slice_heap_size(bs_bin, bs_offset, size_bits); if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } @@ -4509,7 +4618,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) bs_bin = x_regs[live]; - term t = term_maybe_create_sub_binary(bs_bin, bs_offset / unit, size_val, &ctx->heap, ctx->global); + term t = bitstring_slice(bs_bin, bs_offset, size_bits, &ctx->heap, ctx->global); WRITE_REGISTER(dreg, t); } break; @@ -4750,7 +4859,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_bitstr/2, label=%i, arg1=%" TERM_X_FMT "\n", label, arg1); - if (!term_is_binary(arg1)) { + if (!term_is_bitstring(arg1)) { pc = mod->labels[label]; } @@ -5424,10 +5533,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // no_fail: we know it's a binary or a match_state // resume: we know it's a match_state - if (term_is_invalid_term(fail_atom) && !(term_is_binary(src) || term_is_match_state(src))) { + if (term_is_invalid_term(fail_atom) && !(term_is_bitstring(src) || term_is_match_state(src))) { pc = mod->labels[fail_label]; } else { - assert(term_is_binary(src) || term_is_match_state(src)); + assert(term_is_bitstring(src) || term_is_match_state(src)); TRIM_LIVE_REGS(live); x_regs[live] = src; @@ -5637,14 +5746,30 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case APPEND_ATOM: case BINARY_ATOM: case PRIVATE_APPEND_ATOM: { - VERIFY_IS_BINARY(src, "bs_create_bin/6", fail); + VERIFY_IS_BITSTRING(src, "bs_create_bin/6", fail); if (size == ALL_ATOM) { - // We only support src as a binary of bytes here. - segment_size = term_binary_size(src); - segment_unit = 8; + size_t src_bits = term_bit_size(src); + // The whole source is taken, so its bit size + // must be a multiple of the segment unit: a + // /binary segment (unit 8) rejects a partial + // bitstring, a /bitstring one (unit 1) takes it. + if (UNLIKELY(segment_unit == 0 || (src_bits % segment_unit) != 0)) { + if (fail == 0) { + RAISE_ERROR(BADARG_ATOM); + } else { + JUMP_TO_LABEL(mod, fail); + } + } if (atom_type == PRIVATE_APPEND_ATOM && j == 0) { + // Reusing the accumulator requires a byte-aligned source + if (UNLIKELY(src_bits % 8 != 0)) { + TRACE("bs_create_bin/6: private_append on a non-byte-aligned bitstring is not supported\n"); + RAISE_ERROR(UNSUPPORTED_ATOM); + } reuse_binary = true; } + segment_size = src_bits; + segment_unit = 1; } else { VERIFY_IS_INTEGER(size, "bs_create_bin/6", fail); avm_int_t signed_size_value = term_to_int(size); @@ -5656,13 +5781,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } } avm_int_t size_in_bits = signed_size_value * segment_unit; - if (size_in_bits % 8) { - TRACE("bs_create_bin/6: size in bits (%d) is not evenly divisible by 8\n", (int) size_in_bits); - RAISE_ERROR(UNSUPPORTED_ATOM); - } - avm_int_t size_in_bytes = size_in_bits / 8; - size_t binary_size = term_binary_size(src); - if ((size_t) size_in_bytes > binary_size) { + if ((size_t) size_in_bits > term_bit_size(src)) { if (fail == 0) { RAISE_ERROR(BADARG_ATOM); } else { @@ -5680,18 +5799,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } binary_size += segment_unit * segment_size; } - // Allocate and build binary in second iteration - if (binary_size % 8) { - TRACE("bs_create_bin/6: total binary size (%d) is not evenly divisible by 8\n", (int) binary_size); + // Allocate and build binary in second iteration. A non-byte-aligned + // total size yields a bitstring: the whole bytes are stored in a heap + // binary and wrapped in a sub-binary carrying the trailing bit count. + size_t trailing_bits = binary_size % 8; + size_t binary_bytes = (binary_size + 7) / 8; + if (UNLIKELY(trailing_bits != 0 && reuse_binary)) { + TRACE("bs_create_bin/6: non-byte-aligned append is not supported\n"); RAISE_ERROR(UNSUPPORTED_ATOM); } TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, alloc + term_binary_heap_size(binary_size / 8), live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + size_t bs_heap_size = alloc + term_binary_heap_size(binary_bytes); + if (trailing_bits != 0) { + bs_heap_size += TERM_BOXED_SUB_BINARY_SIZE; + } + if (UNLIKELY(memory_ensure_free_with_roots(ctx, bs_heap_size, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } term t; if (!reuse_binary) { - t = term_create_empty_binary(binary_size / 8, &ctx->heap, ctx->global); + t = term_create_empty_binary(binary_bytes, &ctx->heap, ctx->global); if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } @@ -5799,6 +5926,13 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) "binary\n"); RAISE_ERROR(BADARG_ATOM); } + } else if ((offset % 8 != 0) || ((size_value * segment_unit) % 8 != 0)) { + // intn_to_integer_bytes writes whole, byte-aligned + // bytes; a non-byte-aligned bit offset or field + // width would misplace the bits. Reject rather than + // silently miscompile (bignums >64 bits only). + TRACE("bs_create_bin/6: non-byte-aligned big integer segment unsupported\n"); + RAISE_ERROR(UNSUPPORTED_ATOM); } else { // when building a binary, `signed` flag is implicit intn_from_integer_options_t intn_flags @@ -5868,21 +6002,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case APPEND_ATOM: case BINARY_ATOM: case PRIVATE_APPEND_ATOM: { - if (offset % 8) { - TRACE("bs_create_bin/6: current offset (%d) is not evenly divisible by 8\n", (int) offset); - RAISE_ERROR(UNSUPPORTED_ATOM); - } - size_t src_size = term_binary_size(src); + size_t src_bits = term_bit_size(src); if (reuse_binary && j == 0) { + // The size pass verified the reused source is byte-aligned t = term_reuse_binary(src, binary_size / 8, &ctx->heap, ctx->global); if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - segment_size = src_size * 8; + segment_size = src_bits; break; } - uint8_t *dst = (uint8_t *) term_binary_data(t) + (offset / 8); + uint8_t *dst = (uint8_t *) term_binary_data(t); const uint8_t *bin = (const uint8_t *) term_binary_data(src); + size_t copy_bits = src_bits; if (size != ALL_ATOM) { VERIFY_IS_INTEGER(size, "bs_create_bin/6", fail); avm_int_t signed_size_value = term_to_int(size); @@ -5890,19 +6022,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_create_bin/6: size value less than 0: %i\n", (int) signed_size_value); RAISE_ERROR(BADARG_ATOM); } - // We checked earlier it's a multiple of 8 - size_value = ((size_t) signed_size_value) * segment_unit / 8; - if (size_value > src_size) { + copy_bits = ((size_t) signed_size_value) * segment_unit; + if (copy_bits > src_bits) { if (fail == 0) { RAISE_ERROR(BADARG_ATOM); } else { JUMP_TO_LABEL(mod, fail); } } - src_size = size_value; } - memcpy(dst, bin, src_size); - segment_size = src_size * 8; + bitstring_copy_bits(dst, offset, bin, copy_bits); + segment_size = copy_bits; break; } default: @@ -5910,6 +6040,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } offset += segment_size; } + if (trailing_bits != 0) { + t = term_alloc_sub_binary_bits(t, 0, binary_size / 8, (uint8_t) trailing_bits, &ctx->heap); + } WRITE_REGISTER_GC_SAFE(dreg, t); break; } @@ -6036,14 +6169,13 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) int unit; DECODE_LITERAL(unit, pc); j++; - size_t bs_bin_size = term_binary_size(bs_bin); if (UNLIKELY(stride < 0)) { RAISE_ERROR(BADARG_ATOM); } size_t unsigned_stride = (size_t) stride; - size_t remaining = (bs_bin_size * 8) - bs_offset; + size_t remaining = term_bit_size(bs_bin) - bs_offset; if (remaining < unsigned_stride || (remaining - unsigned_stride) % unit != 0) { - TRACE("bs_match/3: ensure_at_least failed -- bs_bin_size = %d, bs_offset = %d, stride = %d, unit = %d\n", (int) bs_bin_size, (int) bs_offset, (int) stride, (int) unit); + TRACE("bs_match/3: ensure_at_least failed -- bs_offset = %d, stride = %d, unit = %d\n", (int) bs_offset, (int) stride, (int) unit); goto bs_match_jump_to_fail; } break; @@ -6057,9 +6189,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(BADARG_ATOM); } size_t unsigned_stride = (size_t) stride; - size_t bs_bin_size = term_binary_size(bs_bin); - if ((bs_bin_size * 8) - bs_offset != unsigned_stride) { - TRACE("bs_match/3: ensure_exactly failed -- bs_bin_size = %lu, bs_offset = %lu, stride = %lu\n", (unsigned long) bs_bin_size, (unsigned long) bs_offset, (unsigned long) stride); + if (term_bit_size(bs_bin) - bs_offset != unsigned_stride) { + TRACE("bs_match/3: ensure_exactly failed -- bs_offset = %lu, stride = %lu\n", (unsigned long) bs_offset, (unsigned long) stride); goto bs_match_jump_to_fail; } break; @@ -6074,16 +6205,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) j++; avm_int_t flags_value; DECODE_FLAGS_LIST(flags_value, flags, opcode) - term size; - DECODE_COMPACT_TERM(size, pc); + // The size of a bs_match integer command is always a + // literal (a BEAM invariant asserted by + // beam_validator: a variable-size segment is emitted + // as a separate bs_get_integer2), as for the binary + // command below. + int size; + DECODE_LITERAL(size, pc); j++; int unit; DECODE_LITERAL(unit, pc); j++; // context_clean_registers(ctx, live); // TODO: check if needed - VERIFY_IS_INTEGER(size, "bs_match/3", fail); - avm_int_t size_val = term_to_int(size); - avm_int_t increment = size_val * unit; + avm_int_t increment = (avm_int_t) size * unit; union maybe_unsigned_int64 value; term t; if (increment <= 64) { @@ -6118,7 +6252,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) HANDLE_ERROR(); } } else { - goto bs_match_jump_to_fail; + // A >64-bit field at a non-byte-aligned offset (or a + // non-byte-multiple size) is not extracted yet, as on + // the construction side. + RAISE_ERROR(UNSUPPORTED_ATOM); } DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); @@ -6145,15 +6282,11 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(unit, pc); j++; int matched_bits = size * unit; - if (bs_offset % 8 != 0 || matched_bits % 8 != 0) { - TRACE("bs_match/3: Unsupported. Offset on binary read must be aligned on byte boundaries.\n"); - RAISE_ERROR(BADARG_ATOM); - } - if ((bs_offset + matched_bits) > term_binary_size(bs_bin) * 8) { + if ((bs_offset + matched_bits) > term_bit_size(bs_bin)) { TRACE("bs_match/3: insufficient capacity\n"); goto bs_match_jump_to_fail; } - size_t heap_size = term_sub_binary_heap_size(bs_bin, matched_bits / 8); + size_t heap_size = bitstring_slice_heap_size(bs_bin, bs_offset, matched_bits); TRIM_LIVE_REGS(live); x_regs[live] = match_state; if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { @@ -6161,7 +6294,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } match_state = x_regs[live]; bs_bin = term_get_match_state_binary(match_state); - term t = term_maybe_create_sub_binary(bs_bin, bs_offset / 8, matched_bits / 8, &ctx->heap, ctx->global); + term t = bitstring_slice(bs_bin, bs_offset, matched_bits, &ctx->heap, ctx->global); DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); j++; @@ -6177,15 +6310,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) int unit; DECODE_LITERAL(unit, pc); j++; - // TODO: rewrite this bit once bitstrings are supported - if (bs_offset % 8 != 0) { - TRACE("bs_match/3: Unsupported. Offset on binary read must be aligned on byte boundaries.\n"); - RAISE_ERROR(BADARG_ATOM); - } - size_t total_bytes = term_binary_size(bs_bin); - size_t bs_offset_bytes = bs_offset / 8; - size_t tail_bytes = total_bytes - bs_offset_bytes; - size_t heap_size = term_sub_binary_heap_size(bs_bin, tail_bytes); + size_t heap_size = bitstring_get_tail_heap_size(bs_bin, bs_offset); TRIM_LIVE_REGS(live); x_regs[live] = match_state; if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { @@ -6193,7 +6318,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } match_state = x_regs[live]; bs_bin = term_get_match_state_binary(match_state); - term t = term_maybe_create_sub_binary(bs_bin, bs_offset_bytes, tail_bytes, &ctx->heap, ctx->global); + term t = bitstring_get_tail(bs_bin, bs_offset, &ctx->heap, ctx->global); DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); j++; diff --git a/src/libAtomVM/term.c b/src/libAtomVM/term.c index f8b1a0cce3..517c7f8a7d 100644 --- a/src/libAtomVM/term.c +++ b/src/libAtomVM/term.c @@ -346,11 +346,14 @@ int term_funprint(PrinterFun *fun, term t, const GlobalContext *global) ret += printed; return ret; - } else if (term_is_binary(t)) { + } else if (term_is_bitstring(t)) { int len = term_binary_size(t); const unsigned char *binary_data = (const unsigned char *) term_binary_data(t); + uint8_t trailing_bits = (uint8_t) (term_bit_size(t) % 8); - int is_printable = 1; + // Print no empty string ahead of the trailing bits field of a + // sub-byte bitstring (<<1:1>>, not <<""1:1>>) + int is_printable = (len != 0) || (trailing_bits == 0); for (int i = 0; i < len; i++) { if (!isprint(binary_data[i])) { is_printable = 0; @@ -391,6 +394,16 @@ int term_funprint(PrinterFun *fun, term t, const GlobalContext *global) ret += printed; } } + if (trailing_bits != 0) { + // Print the trailing partial byte as a final Value:Size field. + uint8_t trailing_value = (uint8_t) (binary_data[len] >> (8 - trailing_bits)); + int printed = fun->print(fun, "%s%u:%u", len != 0 ? "," : "", + (unsigned int) trailing_value, (unsigned int) trailing_bits); + if (UNLIKELY(printed < 0)) { + return printed; + } + ret += printed; + } int printed = fun->print(fun, ">>"); if (UNLIKELY(printed < 0)) { return printed; @@ -955,25 +968,36 @@ TermCompareResult term_compare(term t, term other, TermCompareOpts opts, GlobalC break; } case TERM_TYPE_INDEX_BINARY: { - int t_size = term_binary_size(t); - int other_size = term_binary_size(other); + size_t t_bits = term_bit_size(t); + size_t other_bits = term_bit_size(other); - const char *t_data = term_binary_data(t); - const char *other_data = term_binary_data(other); + const unsigned char *t_data = (const unsigned char *) term_binary_data(t); + const unsigned char *other_data = (const unsigned char *) term_binary_data(other); - int cmp_size = (t_size > other_size) ? other_size : t_size; + size_t cmp_bits = (t_bits > other_bits) ? other_bits : t_bits; + size_t cmp_bytes = cmp_bits / 8; - int memcmp_result = memcmp(t_data, other_data, cmp_size); - if (memcmp_result == 0) { - if (t_size == other_size) { - CMP_POP_AND_CONTINUE(); - break; - } else { - result = (t_size > other_size) ? TermGreaterThan : TermLessThan; + int memcmp_result = memcmp(t_data, other_data, cmp_bytes); + if (memcmp_result != 0) { + result = (memcmp_result > 0) ? TermGreaterThan : TermLessThan; + goto unequal; + } + // Compare the trailing partial byte over its valid bits only. + size_t rem_bits = cmp_bits % 8; + if (rem_bits != 0) { + unsigned char mask = (unsigned char) (0xFF << (8 - rem_bits)); + unsigned char tb = t_data[cmp_bytes] & mask; + unsigned char ob = other_data[cmp_bytes] & mask; + if (tb != ob) { + result = (tb > ob) ? TermGreaterThan : TermLessThan; goto unequal; } + } + if (t_bits == other_bits) { + CMP_POP_AND_CONTINUE(); + break; } else { - result = (memcmp_result > 0) ? TermGreaterThan : TermLessThan; + result = (t_bits > other_bits) ? TermGreaterThan : TermLessThan; goto unequal; } } @@ -1287,13 +1311,18 @@ static term find_binary(term binary_or_state) } term term_alloc_sub_binary(term binary_or_state, size_t offset, size_t len, Heap *heap) +{ + return term_alloc_sub_binary_bits(binary_or_state, offset, len, 0, heap); +} + +term term_alloc_sub_binary_bits(term binary_or_state, size_t offset, size_t len, uint8_t trailing_bits, Heap *heap) { term *boxed = memory_heap_alloc(heap, TERM_BOXED_SUB_BINARY_SIZE); term binary = find_binary(binary_or_state); boxed[0] = ((TERM_BOXED_SUB_BINARY_SIZE - 1) << 6) | TERM_BOXED_SUB_BINARY; boxed[1] = (term) len; - boxed[2] = (term) offset; + boxed[2] = (term) ((offset << 3) | trailing_bits); boxed[3] = binary; return ((term) boxed) | TERM_PRIMARY_BOXED; diff --git a/src/libAtomVM/term.h b/src/libAtomVM/term.h index df1d411354..6cedb18488 100644 --- a/src/libAtomVM/term.h +++ b/src/libAtomVM/term.h @@ -338,6 +338,22 @@ term term_alloc_refc_binary(size_t size, bool is_const, Heap *heap, GlobalContex */ term term_alloc_sub_binary(term binary, size_t offset, size_t len, Heap *heap); +/** + * @brief Allocate a sub-binary that may be non-byte-aligned (a bitstring). + * + * @details Same as term_alloc_sub_binary but carries a trailing partial-byte + * bit count (0..7). When trailing_bits is 0 the result is an ordinary + * byte-aligned binary; when non-zero the total bit size is len*8 + trailing_bits + * and the last referenced byte is only partially valid. + * @param binary the referenced binary + * @param offset the offset (in bytes) into the referenced binary + * @param len the number of whole bytes of the sub-binary + * @param trailing_bits number of valid bits (0..7) in the byte following the whole bytes + * @param heap the heap to allocate the binary in + * @return a term (reference) pointing to the newly allocated sub-binary. + */ +term term_alloc_sub_binary_bits(term binary, size_t offset, size_t len, uint8_t trailing_bits, Heap *heap); + /** * @brief Gets a pointer to a term stored on the heap * @@ -468,10 +484,41 @@ static inline size_t term_boxed_size(term t) return term_get_size_from_boxed_header(*boxed_value); } +/** + * @brief Checks if a term is a bitstring + * + * @details Returns \c true if a term is a bitstring (a binary or a + * sub-binary carrying trailing bits), otherwise \c false. This is the + * predicate behind the is_bitstring/1 BIF. + * @param t the term that will be checked. + * @return \c true if check succeeds, \c false otherwise. + */ +static inline bool term_is_bitstring(term t) +{ + /* boxed: 10 */ + if ((t & TERM_PRIMARY_MASK) == TERM_PRIMARY_BOXED) { + const term *boxed_value = term_to_const_term_ptr(t); + int masked_value = boxed_value[0] & TERM_BOXED_TAG_MASK; + switch (masked_value) { + case TERM_BOXED_REFC_BINARY: + case TERM_BOXED_HEAP_BINARY: + case TERM_BOXED_SUB_BINARY: + return true; + default: + return false; + } + } + + return false; +} + /** * @brief Checks if a term is a binary * - * @details Returns \c true if a term is a binary stored on the heap, otherwise \c false. + * @details Returns \c true if a term is a byte-aligned binary, otherwise + * \c false. A sub-binary carrying trailing bits (a non-byte-aligned + * bitstring) is not a binary; use \c term_is_bitstring to accept it. This + * is the predicate behind the is_binary/1 BIF. * @param t the term that will be checked. * @return \c true if check succeeds, \c false otherwise. */ @@ -484,8 +531,10 @@ static inline bool term_is_binary(term t) switch (masked_value) { case TERM_BOXED_REFC_BINARY: case TERM_BOXED_HEAP_BINARY: - case TERM_BOXED_SUB_BINARY: return true; + case TERM_BOXED_SUB_BINARY: + // boxed_value[2] packs (byte_offset << 3) | trailing_bits + return (boxed_value[2] & 0x7) == 0; default: return false; } @@ -1891,7 +1940,7 @@ static inline size_t term_binary_heap_size(size_t size) */ static inline unsigned long term_binary_size(term t) { - TERM_DEBUG_ASSERT(term_is_binary(t)); + TERM_DEBUG_ASSERT(term_is_bitstring(t)); const term *boxed_value = term_to_const_term_ptr(t); return boxed_value[1]; @@ -1934,7 +1983,7 @@ static inline struct RefcBinary *term_resource_refc_binary_ptr(term resource) */ static inline const char *term_binary_data(term t) { - TERM_DEBUG_ASSERT(term_is_binary(t)); + TERM_DEBUG_ASSERT(term_is_bitstring(t)); const term *boxed_value = term_to_const_term_ptr(t); if (term_is_refc_binary(t)) { @@ -1948,7 +1997,8 @@ static inline const char *term_binary_data(term t) } } if (term_is_sub_binary(t)) { - return term_binary_data(boxed_value[3]) + boxed_value[2]; // offset + // boxed_value[2] packs (byte_offset << 3) | trailing_bits + return term_binary_data(boxed_value[3]) + (boxed_value[2] >> 3); } return (const char *) (boxed_value + 2); } @@ -2035,7 +2085,7 @@ static inline term term_maybe_create_sub_binary(term binary, size_t offset, size return term_alloc_sub_binary(binary, offset, len, heap); } else if (term_is_sub_binary(binary) && len >= SUB_BINARY_MIN) { const term *boxed_value = term_to_const_term_ptr(binary); - return term_alloc_sub_binary(boxed_value[3], boxed_value[2] + offset, len, heap); + return term_alloc_sub_binary(boxed_value[3], (boxed_value[2] >> 3) + offset, len, heap); } else { const char *data = term_binary_data(binary); return term_from_literal_binary(data + offset, len, heap, glb); @@ -2116,47 +2166,6 @@ static inline BinaryPosLen term_nomatch_binary_pos_len(void) return (BinaryPosLen){ .pos = -1, .len = -1 }; } -/** - * @brief Insert an binary into a binary (using bit syntax). - * - * @details Insert the data from the input binary, starting - * at the bit position starting in offset. - * @param t a term pointing to binary data. Fails if t is not a binary term. - * @param offset the bitwise offset in t at which to start writing the integer value - * @param src binary source to insert binary data into. - * @param n the number of low-order bits from value to write. - * @return 0 on success; non-zero value if: - * t is not a binary term - * n is greater than the number of bits in an integer - * there is insufficient capacity in the binary to write these bits - * In general, none of these conditions should apply, if this function is being - * called in the context of generated bit syntax instructions. - */ -static inline int term_bs_insert_binary(term t, int offset, term src, int n) -{ - if (!term_is_binary(t)) { - fprintf(stderr, "Target is not a binary\n"); - return -1; - } - if (!term_is_binary(src)) { - fprintf(stderr, "Source is not a binary\n"); - return -2; - } - if (offset % 8 != 0) { - fprintf(stderr, "Offset not aligned on a byte boundary\n"); - return -3; - } - unsigned long capacity = term_binary_size(t); - if (capacity < (unsigned long) (offset / 8 + n)) { - fprintf(stderr, "Insufficient capacity to write binary\n"); - return -4; - } - uint8_t *dst_pos = (uint8_t *) term_binary_data(t) + offset / 8; - uint8_t *src_pos = (uint8_t *) term_binary_data(src); - memcpy(dst_pos, src_pos, n); - return 0; -} - /** * @brief Get a ref term from ref ticks * @@ -2955,6 +2964,40 @@ static inline term term_get_sub_binary_ref(term t) return boxed_value[3]; } +/** + * @brief Number of trailing (partial-byte) bits of a sub-binary (0..7). + * + * @details A byte-aligned sub-binary returns 0. A non-byte-aligned bitstring + * returns 1..7, meaning its total bit size is term_binary_size(t)*8 + this. + */ +static inline uint8_t term_get_sub_binary_num_trailing_bits(term t) +{ + const term *boxed_value = term_to_const_term_ptr(t); + return (uint8_t) (boxed_value[2] & 0x7); +} + +/** + * @brief Offset (in bytes) of a sub-binary into its parent binary. + */ +static inline size_t term_get_sub_binary_offset(term t) +{ + const term *boxed_value = term_to_const_term_ptr(t); + return (size_t) (boxed_value[2] >> 3); +} + +/** + * @brief Total size in bits of any bitstring (binary or sub-binary). + */ +static inline size_t term_bit_size(term t) +{ + size_t bits = term_binary_size(t) * 8; + const term *boxed_value = term_to_const_term_ptr(t); + if ((boxed_value[0] & TERM_BOXED_TAG_MASK) == TERM_BOXED_SUB_BINARY) { + bits += boxed_value[2] & 0x7; + } + return bits; +} + /** * @brief Create a resource on the heap. * @details This function creates a resource (obtained from `enif_alloc_resource`) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3c7f880c49..c0b608c7d5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,6 +22,7 @@ cmake_minimum_required (VERSION 3.13) project (tests) add_executable(test-erlang test.c) +add_executable(test-bitstring test-bitstring.c) add_executable(test-enif test-enif.c) add_executable(test-heap test-heap.c) add_executable(test-jit_stream_flash test-jit_stream_flash.c ../src/libAtomVM/jit_stream_flash.c) @@ -29,6 +30,7 @@ add_executable(test-mailbox test-mailbox.c) add_executable(test-structs test-structs.c) target_compile_features(test-erlang PUBLIC c_std_11) +target_compile_features(test-bitstring PUBLIC c_std_11) target_compile_features(test-enif PUBLIC c_std_11) target_compile_features(test-heap PUBLIC c_std_11) target_compile_features(test-jit_stream_flash PUBLIC c_std_11) @@ -37,6 +39,7 @@ target_compile_features(test-structs PUBLIC c_std_11) if(CMAKE_COMPILER_IS_GNUCC) target_compile_options(test-erlang PUBLIC -Wall -pedantic -Wextra -ggdb) + target_compile_options(test-bitstring PUBLIC -Wall -pedantic -Wextra -ggdb) target_compile_options(test-enif PUBLIC -Wall -pedantic -Wextra -ggdb) target_compile_options(test-heap PUBLIC -Wall -pedantic -Wextra -ggdb) target_compile_options(test-jit_stream_flash PUBLIC -Wall -pedantic -Wextra -ggdb) @@ -51,6 +54,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") if (HAVE_CLOCK_GETTIME) find_library(LIBRT rt REQUIRED) target_link_libraries(test-erlang PRIVATE ${LIBRT}) + target_link_libraries(test-bitstring PRIVATE ${LIBRT}) target_link_libraries(test-enif PRIVATE ${LIBRT}) target_link_libraries(test-heap PRIVATE ${LIBRT}) target_link_libraries(test-jit_stream_flash PRIVATE ${LIBRT}) @@ -65,6 +69,7 @@ endif() include(MbedTLS) if (MbedTLS_FOUND) target_link_libraries(test-erlang PRIVATE MbedTLS::mbedtls) + target_link_libraries(test-bitstring PRIVATE MbedTLS::mbedtls) target_link_libraries(test-enif PRIVATE MbedTLS::mbedtls) target_link_libraries(test-heap PRIVATE MbedTLS::mbedtls) target_link_libraries(test-jit_stream_flash PRIVATE MbedTLS::mbedtls) @@ -82,6 +87,7 @@ if((${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") OR (${CMAKE_SYSTEM_NAME} STREQUAL "FreeBSD") OR (${CMAKE_SYSTEM_NAME} STREQUAL "DragonFly")) target_include_directories(test-erlang PRIVATE ../src/platforms/generic_unix/lib) + target_include_directories(test-bitstring PRIVATE ../src/platforms/generic_unix/lib) target_include_directories(test-enif PRIVATE ../src/platforms/generic_unix/lib) target_include_directories(test-heap PRIVATE ../src/platforms/generic_unix/lib) target_include_directories(test-jit_stream_flash PRIVATE ../src/platforms/generic_unix/lib) @@ -92,12 +98,14 @@ else() endif() target_include_directories(test-erlang PRIVATE ../src/libAtomVM) +target_include_directories(test-bitstring PRIVATE ../src/libAtomVM) target_include_directories(test-enif PRIVATE ../src/libAtomVM) target_include_directories(test-heap PRIVATE ../src/libAtomVM) target_include_directories(test-jit_stream_flash PRIVATE ../src/libAtomVM ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(test-mailbox PRIVATE ../src/libAtomVM) target_include_directories(test-structs PRIVATE ../src/libAtomVM) target_link_libraries(test-erlang PRIVATE libAtomVM libAtomVM${PLATFORM_LIB_SUFFIX}) +target_link_libraries(test-bitstring PRIVATE libAtomVM libAtomVM${PLATFORM_LIB_SUFFIX}) target_link_libraries(test-enif PRIVATE libAtomVM libAtomVM${PLATFORM_LIB_SUFFIX}) target_link_libraries(test-heap PRIVATE libAtomVM libAtomVM${PLATFORM_LIB_SUFFIX}) # test-jit_stream_flash includes jit_stream_flash.c and provides its own mock platform implementation @@ -128,12 +136,14 @@ endif() if (COVERAGE) include(CodeCoverage) append_coverage_compiler_flags_to_target(test-erlang) + append_coverage_compiler_flags_to_target(test-bitstring) append_coverage_compiler_flags_to_target(test-enif) append_coverage_compiler_flags_to_target(test-heap) append_coverage_compiler_flags_to_target(test-jit_stream_flash) append_coverage_compiler_flags_to_target(test-mailbox) append_coverage_compiler_flags_to_target(test-structs) append_coverage_linker_flags_to_target(test-erlang) + append_coverage_linker_flags_to_target(test-bitstring) append_coverage_linker_flags_to_target(test-enif) append_coverage_linker_flags_to_target(test-heap) append_coverage_linker_flags_to_target(test-jit_stream_flash) diff --git a/tests/erlang_tests/CMakeLists.txt b/tests/erlang_tests/CMakeLists.txt index a4b0f82b08..e3110f0b0e 100644 --- a/tests/erlang_tests/CMakeLists.txt +++ b/tests/erlang_tests/CMakeLists.txt @@ -455,6 +455,8 @@ compile_erlang(test_bs) compile_erlang(test_bs_int) compile_erlang(test_bs_int_any_flags) compile_erlang(test_bs_int_unaligned) +compile_erlang(test_bs_gettail) +compile_erlang(test_bitstring_to_list) compile_erlang(test_bs_start_match_live) compile_erlang(test_bs_utf) compile_erlang(test_catch) @@ -1017,6 +1019,8 @@ set(erlang_test_beams test_bs_int.beam test_bs_int_any_flags.beam test_bs_int_unaligned.beam + test_bs_gettail.beam + test_bitstring_to_list.beam test_bs_start_match_live.beam test_bs_utf.beam test_catch.beam diff --git a/tests/erlang_tests/test_binary_to_term.erl b/tests/erlang_tests/test_binary_to_term.erl index 17d507c4a1..fb3bbad6f4 100644 --- a/tests/erlang_tests/test_binary_to_term.erl +++ b/tests/erlang_tests/test_binary_to_term.erl @@ -82,6 +82,14 @@ start() -> 115, 116>> ), test_reverse(<<"foobar">>, <<131, 109, 0, 0, 0, 6, 102, 111, 111, 98, 97, 114>>), + test_reverse(<<1:1>>, <<131, 77, 0, 0, 0, 1, 1, 128>>), + test_reverse(<<5:31>>, <<131, 77, 0, 0, 0, 4, 7, 0, 0, 0, 10>>), + test_reverse(<<255, 5:7>>, <<131, 77, 0, 0, 0, 2, 7, 255, 10>>), + % BIT_BINARY_EXT: insignificant padding bits in the last byte are accepted + % on decode (as OTP does) and canonicalized (zeroed) on re-encode. + DirtyBitstring = erlang:binary_to_term(id(<<131, 77, 0, 0, 0, 1, 1, 255>>)), + true = DirtyBitstring =:= <<1:1>>, + <<131, 77, 0, 0, 0, 1, 1, 128>> = erlang:term_to_binary(DirtyBitstring), test_reverse(<<":アトムＶＭ">>, <<131, 109, 0, 0, 0, 6, 58, 162, 200, 224, 54, 45>>), test_reverse("", <<131, 106>>), test_reverse("foobar", <<131, 107, 0, 6, 102, 111, 111, 98, 97, 114>>), @@ -157,6 +165,7 @@ start() -> ok = test_safe_option(), ok = test_invalid_export_fun_encoding(), ok = test_atom_utf8_ext_node(), + ok = test_invalid_bit_binary(), 0. test_reverse(T, Interop) -> @@ -1304,6 +1313,28 @@ test_atom_utf8_ext_node() -> true = is_reference(Ref), ok. +test_invalid_bit_binary() -> + %% BIT_BINARY_EXT (tag 77) layout: <<77, Len:32, Bits:8, Data:Len/bytes>>. + %% Bits is the number of significant bits in the trailing byte. For a + %% non-empty binary it must be in 1..8 (8 meaning a byte-aligned last + %% byte), and an empty binary must carry 0. Anything else is a malformed + %% encoding and must be rejected, matching OTP. + + %% Bits = 0 with a non-empty binary + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 1, 0, 128>>) end), + %% Bits = 9 (> 8) + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 1, 9, 128>>) end), + %% Bits = 255 (> 8) + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 1, 255, 128>>) end), + %% Non-zero trailing bits with an empty binary + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 0, 5>>) end), + + %% Valid encodings still decode. Bits = 8 is a byte-aligned last byte. + <<255>> = binary_to_term(<<131, 77, 0, 0, 0, 1, 8, 255>>), + <<1:1>> = binary_to_term(<<131, 77, 0, 0, 0, 1, 1, 128>>), + <<255, 5:7>> = binary_to_term(<<131, 77, 0, 0, 0, 2, 7, 255, 10>>), + ok. + make_binterm_fun(Id) -> fun() -> Bin = ?MODULE:get_binary(Id), diff --git a/tests/erlang_tests/test_bitstring_to_list.erl b/tests/erlang_tests/test_bitstring_to_list.erl new file mode 100644 index 0000000000..e0b249fdb5 --- /dev/null +++ b/tests/erlang_tests/test_bitstring_to_list.erl @@ -0,0 +1,68 @@ +% +% This file is part of AtomVM. +% +% Copyright 2026 Paul Guyot +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. +% +% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +% + +-module(test_bitstring_to_list). + +-export([start/0, id/1]). + +-define(ID(X), ?MODULE:id(X)). + +start() -> + [1, 2, 3] = erlang:bitstring_to_list(?ID(<<1, 2, 3>>)), + [] = erlang:bitstring_to_list(?ID(<<>>)), + [$a, $b, $c] = erlang:bitstring_to_list(?ID(<<"abc">>)), + Bytes = [0, 1, 2, 127, 128, 200, 254, 255], + Bytes = erlang:bitstring_to_list(?ID(<<0, 1, 2, 127, 128, 200, 254, 255>>)), + ok = raises_badarg(fun() -> erlang:bitstring_to_list(?ID([1, 2, 3])) end), + ok = raises_badarg(fun() -> erlang:bitstring_to_list(?ID(not_a_bitstring)) end), + %% binary_to_list/1 requires a byte-aligned binary (unlike bitstring_to_list/1). + ok = raises_badarg(fun() -> erlang:binary_to_list(?ID(<<1:1>>)) end), + ok = raises_badarg(fun() -> erlang:binary_to_list(?ID(<<255, 5:7>>)) end), + %% Non-byte-aligned bitstrings: the trailing partial byte is returned as a + %% final bitstring element. beam_core_to_ssa relies on this when compiling a + %% match against a non-byte-aligned literal segment (e.g. <<1:1,_:63>>). + [B1] = erlang:bitstring_to_list(?ID(<<1:1>>)), + true = is_bitstring(B1), + false = is_binary(B1), + 1 = bit_size(B1), + 1 = extract(B1), + [B3] = erlang:bitstring_to_list(?ID(<<3:3>>)), + 3 = bit_size(B3), + 3 = extract(B3), + [255, B7] = erlang:bitstring_to_list(?ID(<<255, 5:7>>)), + 7 = bit_size(B7), + 5 = extract(B7), + 0. + +extract(Bitstring) -> + Size = bit_size(Bitstring), + <> = Bitstring, + Value. + +raises_badarg(Fun) -> + try Fun() of + Ret -> {unexpected, Ret} + catch + error:badarg -> ok; + C:E -> {unexpected, C, E} + end. + +id(X) -> + X. diff --git a/tests/erlang_tests/test_bs.erl b/tests/erlang_tests/test_bs.erl index 76d0950b0b..b23c08b09c 100644 --- a/tests/erlang_tests/test_bs.erl +++ b/tests/erlang_tests/test_bs.erl @@ -37,7 +37,16 @@ start() -> ok = test_create_with_invalid_int_value(), ok = test_create_with_invalid_int_size(), ok = test_create_with_int_unit(), - ok = test_create_with_unsupported_unaligned_int_size(), + ok = test_create_with_unaligned_int_size(), + ok = test_bitstring_compare(), + ok = test_bitstring_bif_guards(), + ok = test_bitstring_segments(), + ok = test_little_endian_unaligned(), + ok = test_bs_match_string_trailing(), + ok = test_dynamic_size_extraction(), + ok = test_signed_int_unaligned(), + ok = test_big_int_unaligned_unsupported(), + ok = test_non_pow2_unit(), ok = test_create_with_int_little_endian(), ok = test_create_with_int_signed(), ok = test_create_with_invalid_binary_value(), @@ -104,6 +113,7 @@ start() -> ok = test_bs_skip_bits2_little(), ok = test_bs_variable_size_bitstring(), + ok = test_negative_dynamic_size(), ok = test_float(), 0. @@ -145,8 +155,283 @@ test_create_with_int_unit() -> ), ok. -test_create_with_unsupported_unaligned_int_size() -> - atom_unsupported(fun() -> create_int_binary(16#FFFF, id(28)) end). +test_create_with_unaligned_int_size() -> + B = create_int_binary(16#FFFF, id(28)), + 28 = bit_size(B), + 4 = byte_size(B), + false = is_binary(B), + true = is_bitstring(B), + ok. + +test_bitstring_compare() -> + false = id(<<5:31>>) =:= id(<<5:24>>), + false = id(<<5:31>>) == id(<<5:24>>), + true = id(<<5:3>>) =:= id(<<5:3>>), + true = id(<<5:3>>) == id(<<5:3>>), + true = id(<<(id(5)):31>>) =:= id(<<5:31>>), + false = id(<<5:3>>) < id(<<4>>), + true = id(<<4>>) < id(<<5:3>>), + true = id(<<255>>) < id(<<255, 0:1>>), + true = id(<<255, 0:1>>) < id(<<255, 1:1>>), + true = id(<<1:1>>) > id(<<0:7>>), + true = id(<<>>) < id(<<0:1>>), + [<<>>, <<254>>, <<255>>, <<255, 0:1>>, <<255, 1:1>>] = sort_bitstrings( + id([<<255, 1:1>>, <<255>>, <<255, 0:1>>, <<254>>, <<>>]) + ), + ok. + +% BIFs and NIFs that operate on binaries must badarg on a non-byte-aligned +% bitstring instead of silently truncating it to whole bytes; a few +% (byte_size/1, bit_size/1, size/1, split_binary/2) accept bitstrings. +test_bitstring_bif_guards() -> + Bits9 = id(<<1:9>>), + Bits1 = id(<<1:1>>), + 2 = byte_size(Bits9), + 9 = bit_size(Bits9), + % size/1 rounds down to whole bytes, unlike byte_size/1 which rounds up + 1 = size(Bits9), + 0 = size(Bits1), + % is_bitstring accepts a non-byte-aligned bitstring, is_binary does not + true = erlang:is_bitstring(Bits9), + false = erlang:is_binary(Bits9), + true = erlang:is_bitstring(id(<<1, 2>>)), + true = erlang:is_binary(id(<<1, 2>>)), + false = erlang:is_bitstring(id({})), + % binary_part/3 and binary:part/3 badarg on a non-byte-aligned bitstring + % only since OTP 27; OTP 26 truncated to whole bytes. AtomVM follows the + % modern behavior. + HasBitstringPartGuard = + erlang:system_info(machine) =:= "ATOM" orelse + list_to_integer(erlang:system_info(otp_release)) >= 27, + case HasBitstringPartGuard of + true -> + expect_error(fun() -> binary_part(Bits9, 0, 1) end, badarg), + expect_error(fun() -> binary:part(Bits9, 0, 1) end, badarg); + false -> + ok + end, + % split_binary/2 keeps the trailing bits in the second part + {<<>>, <<0, 1:1>>} = split_binary(Bits9, 0), + {<<0>>, <<1:1>>} = split_binary(Bits9, 1), + expect_error(fun() -> split_binary(Bits9, 2) end, badarg), + expect_error(fun() -> iolist_to_binary(Bits1) end, badarg), + expect_error(fun() -> list_to_binary([Bits1]) end, badarg), + expect_error(fun() -> iolist_size(Bits1) end, badarg), + expect_error(fun() -> binary_to_atom(Bits1, utf8) end, badarg), + expect_error(fun() -> binary_to_list(Bits1) end, badarg), + expect_error(fun() -> binary_to_term(Bits1) end, badarg), + expect_error(fun() -> erlang:crc32(Bits1) end, badarg), + expect_error(fun() -> binary:at(Bits9, 0) end, badarg), + expect_error(fun() -> binary:copy(Bits1) end, badarg), + expect_error(fun() -> binary:split(Bits9, id(<<0>>)) end, badarg), + expect_error(fun() -> binary:first(Bits9) end, badarg), + expect_error(fun() -> binary:last(Bits9) end, badarg), + ok. + +% A bitstring used as a segment source must be copied bit-granularly; these +% used to silently truncate the source to whole bytes. Expected values are +% written in byte layout so they build through the byte-aligned path even if +% the compiler does not constant-fold them. +test_bitstring_segments() -> + Bits1 = id(<<1:1>>), + <<1:1>> = copy_bitstring(Bits1), + <<213, 1:1>> = append_to_bitstring(Bits1), + <<5:3>> = copy_bitstring(id(<<5:3>>)), + % explicit bit-sized bitstring segments + <<2:3>> = take_bits(id(<<2:3>>)), + <<5:4>> = sized_then_bit(id(<<2:3>>)), + <> = id(<<5:3>>), + <<2:2>> = T2, + % mixed bitstring segments and a byte segment at an unaligned offset + <<13:4>> = mix_bitstrings(Bits1, id(<<5:3>>)), + <<255, 255, 1:1>> = sandwich(Bits1), + ok. + +copy_bitstring(B) -> <>. +append_to_bitstring(B) -> <>. +take_bits(B) -> <>. +sized_then_bit(B) -> <>. +mix_bitstrings(P, Q) -> <

>. +sandwich(P) -> <<255, P/bits, 255>>. + +% Little-endian integers whose width is not a multiple of 8 lay out complete +% low-order bytes first, then the remaining high-order bits (OTP layout). +% Expected values are written in byte layout (verified on OTP) so they do not +% depend on the little-endian runtime path under test. +test_little_endian_unaligned() -> + ok = check_le_cases([ + {1, 1, <<1:1>>, <<11:4>>}, + {4, 16#A, <<10:4>>, <<90:7>>}, + {7, 16#55, <<85:7>>, <<181, 1:2>>}, + {9, 16#155, <<85, 1:1>>, <<170, 11:4>>}, + {12, 16#ABC, <<188, 10:4>>, <<183, 74:7>>}, + {15, 16#5A5A, <<90, 90:7>>, <<171, 86, 2:2>>}, + {63, 16#123456789ABCDEF, <<239, 205, 171, 137, 103, 69, 35, 1:7>>, + <<189, 249, 181, 113, 44, 232, 164, 96, 1:2>>} + ]), + % signed little-endian round trip of a negative value + <<251, 15:4>> = make_le(id(-5), id(12)), + <> = id(<<251, 15:4>>), + -5 = S, + ok. + +check_le_cases([]) -> + ok; +check_le_cases([{W, V, Plain, Prefixed} | T]) -> + % construction, at bit offset 0 and after a 3-bit prefix + Plain = make_le(id(V), id(W)), + Prefixed = make_le_prefixed(id(V), id(W)), + % extraction (the inverse mapping), from byte-layout literals + <> = id(Plain), + V = X, + <<_:3, Y:W/little>> = id(Prefixed), + V = Y, + check_le_cases(T). + +make_le(V, W) -> <>. +make_le_prefixed(V, W) -> <<5:3, V:W/little>>. + +% Literal matches that consume trailing bits must measure the source capacity +% in bits, not whole bytes. +test_bs_match_string_trailing() -> + ok = match_9(id(<<255, 1:1>>)), + nomatch = match_9(id(<<255, 0:1>>)), + nomatch = match_9(id(<<255>>)), + ok = match_long(id(<<"hello!!!", 5:3>>)), + nomatch = match_long(id(<<"hello!!!", 4:3>>)), + ok. + +match_9(<<255, 1:1>>) -> ok; +match_9(_) -> nomatch. + +match_long(<<"hello!!!", 5:3>>) -> ok; +match_long(_) -> nomatch. + +% Dynamic-size binary/bitstring segment extraction compiles to bs_get_binary2 +% (OTP 26 through at least 29 emit it; fixed sizes go through bs_match). +% Sizes are bit-granular and the source offset may be unaligned. +test_dynamic_size_extraction() -> + {<<1, 2>>, <<3>>} = dyn_binary(id(2), id(<<1, 2, 3>>)), + nope = dyn_binary(id(4), id(<<1, 2, 3>>)), + {<<5:3>>, <<1:1>>} = dyn_bits(id(3), id(<<5:3, 1:1>>)), + {<<255, 1:1>>, <<5:3>>} = dyn_bits(id(9), id(<<255, 1:1, 5:3>>)), + nope = dyn_bits(id(5), id(<<5:3, 1:1>>)), + % dynamic size at an unaligned offset + {<<2:2>>, <<1:1>>} = dyn_bits_after3(id(2), id(<<5:3, 2:2, 1:1>>)), + % an all-remaining binary tail fails on a non-byte-aligned remainder + nope = bin_tail_of(id(<<1:12>>)), + % a negative dynamic size fails the match, it does not raise + nope = dyn_binary(id(-1), id(<<1, 2, 3>>)), + nope = dyn_bits(id(-1), id(<<5:3, 1:1>>)), + % a bound segment with a unit other than 8 is only supported since + % bit-granular extraction; a negative size must still be rejected before it + % is scaled, as it is for the units the parent commit already covers + nope = dyn_binary_unit64(id(-1), id(<<1>>)), + nope = dyn_binary_unit64(id(-(1 bsl 58)), id(<<1>>)), + nope = dyn_binary_unit64(id(-(1 bsl 58) - 1), id(<<1>>)), + nope = u3(id(-(1 bsl 62)), id(<<5:6, 1:3>>)), + ok. + +% A non-power-of-two unit on a variable-size binary segment (BEAM emits +% bs_get_binary2 with that unit); the JIT must handle what the interpreter does. +test_non_pow2_unit() -> + {<<5:6>>, <<1:3>>} = u3(id(2), id(<<5:6, 1:3>>)), + nope = u3(id(4), id(<<5:6, 1:3>>)), + % `all` with a non-power-of-two unit: the remainder must be tested with a + % real remainder, not a mask, and the JIT must agree with the interpreter + <<5:6, 1:3>> = u3_all(id(<<5:6, 1:3>>)), + <<1:1, 2:2>> = u3_all(id(<<1:1, 2:2>>)), + <<>> = u3_all(id(<<>>)), + nope = u3_all(id(<<5:6, 1:2>>)), + nope = u3_all(id(<<1>>)), + % same, at a non-byte-aligned starting offset + <<3:3, 1:3>> = u3_all_after5(id(<<9:5, 3:3, 1:3>>)), + nope = u3_all_after5(id(<<9:5, 3:2>>)), + ok. + +u3(N, B) -> + case B of + <> -> {X, R}; + _ -> nope + end. + +u3_all(B) -> + case B of + <> -> X; + _ -> nope + end. + +u3_all_after5(B) -> + case B of + <<_:5, X/binary-unit:3>> -> X; + _ -> nope + end. + +% A signed integer of the full 64-bit width at a non-byte-aligned offset must +% sign-extend correctly (guards a shift by the whole type width in the extractor). +test_signed_int_unaligned() -> + AllOnes = id(<<16#0F, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#F0>>), + -1 = sig64_at4(AllOnes), + 18446744073709551615 = uns64_at4(AllOnes), + -1000000000000 = sig64_at4(id(<<0:4, -1000000000000:64/signed, 0:4>>)), + 42 = sig64_at4(id(<<0:4, 42:64/signed, 0:4>>)), + ok. + +% A >64-bit integer field at a non-byte-aligned offset is not supported yet +% (BEAM matches it); AtomVM raises unsupported, as the construction side does. +test_big_int_unaligned_unsupported() -> + atom_unsupported(fun() -> big72_at1(id(<<0:1, 42:72, 0:7>>)) end). + +big72_at1(B) -> + <<_:1, X:72, _:7>> = B, + X. + +sig64_at4(B) -> + <<_:4, X:64/signed, _:4>> = B, + X. + +uns64_at4(B) -> + <<_:4, X:64/unsigned, _:4>> = B, + X. + +dyn_binary(N, B) -> + case B of + <> -> {A, Rest}; + _ -> nope + end. + +dyn_bits(N, B) -> + case B of + <> -> {X, R}; + _ -> nope + end. + +dyn_binary_unit64(N, B) -> + case B of + <> -> {X, R}; + _ -> nope + end. + +dyn_bits_after3(N, B) -> + case B of + <<_:3, X:N/bits, R/bits>> -> {X, R}; + _ -> nope + end. + +bin_tail_of(B) -> + case B of + <> -> X; + _ -> nope + end. + +sort_bitstrings(L) -> sort_bitstrings(L, []). + +sort_bitstrings([], Sorted) -> Sorted; +sort_bitstrings([H | T], Sorted) -> sort_bitstrings(T, insert_bitstring(Sorted, H)). + +insert_bitstring([], B) -> [B]; +insert_bitstring([H | T], B) when B < H -> [B, H | T]; +insert_bitstring([H | T], B) -> [H | insert_bitstring(T, B)]. test_create_with_int_little_endian() -> <<2, 1>> = create_int_binary_little_endian(16#0102, 16), @@ -185,7 +470,10 @@ test_create_with_binary_size_out_of_range() -> expect_error(fun() -> create_binary_binary(<<"foo">>, id(4)) end, badarg). test_create_with_unsupported_binary_unit() -> - atom_unsupported(fun() -> create_binary_binary_unit_3(<<"foo">>, id(3)) end). + % A binary segment whose size in bits is not a multiple of 8 takes a + % bit-granular prefix of the source (9 bits of <<"foo">> here). + <<102, 0:1>> = create_binary_binary_unit_3(<<"foo">>, id(3)), + ok. % Things are very broken here, we get {badmatch, <<16#FFFFFFFF:32>>} but % this term isn't equal to {badmatch, <<16#FFFFFFFF:32>>} @@ -275,9 +563,10 @@ test_get_with_int_signed() -> ok. test_get_with_unaligned_binary() -> - atom_unsupported(fun() -> get_int_then_binary(<<1, 2, 3, 4>>, id(4), id(1)) end, fun(T) -> - T =:= badarg - end). + % A dynamic-size binary segment may start at an unaligned offset (the + % extracted slice is copied in that case). + {0, <<16>>} = get_int_then_binary(<<1, 2, 3, 4>>, id(4), id(1)), + ok. create_int_binary_unit_3(Value, Size) -> <>. @@ -689,6 +978,58 @@ check_x86_64_jt(<<>>) -> ok; check_x86_64_jt(<<16#e9, _Offset:32/little, Tail/binary>>) -> check_x86_64_jt(Tail); check_x86_64_jt(Bin) -> {unexpected, Bin}. +% A dynamic segment size comes from a register and can be negative. It must be +% rejected before it is scaled by the segment unit: the scaled size is compared +% against the remaining capacity and added to the match offset, so scaling +% first lets a negative size wrap to a small one and match, or move the match +% offset before the start of the binary. +test_negative_dynamic_size() -> + B = id(<<1>>), + nope = skip_unit64(id(-1), B), + nope = skip_unit64(id(-(1 bsl 58)), B), + nope = skip_unit64(id(-(1 bsl 58) - 1), B), + nope = skip_unit8(id(-1), B), + nope = skip_unit8(id(-(1 bsl 61)), B), + nope = int_unit64(id(-1), B), + nope = int_unit64(id(-(1 bsl 58)), B), + nope = float_unit64(id(-1), id(<<1, 2, 3, 4, 5, 6, 7, 8>>)), + nope = bin_unit8(id(-1), B), + nope = bin_unit8(id(-(1 bsl 61)), B), + % the same segments still match when the size is valid + <<>> = skip_unit8(id(1), B), + {<<1>>, <<>>} = bin_unit8(id(1), B), + ok. + +skip_unit64(N, B) -> + case B of + <<_:N/binary-unit:64, R/binary>> -> R; + _ -> nope + end. + +skip_unit8(N, B) -> + case B of + <<_:N/binary-unit:8, R/binary>> -> R; + _ -> nope + end. + +bin_unit8(N, B) -> + case B of + <> -> {X, R}; + _ -> nope + end. + +int_unit64(N, B) -> + case B of + <> -> X; + _ -> nope + end. + +float_unit64(N, B) -> + case B of + <> -> X; + _ -> nope + end. + id(X) -> ?MODULE:ext_id(X). ext_id(X) -> X. diff --git a/tests/erlang_tests/test_bs_gettail.erl b/tests/erlang_tests/test_bs_gettail.erl new file mode 100644 index 0000000000..83252918e3 --- /dev/null +++ b/tests/erlang_tests/test_bs_gettail.erl @@ -0,0 +1,42 @@ +% +% This file is part of AtomVM. +% +% Copyright 2026 Paul Guyot +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. +% +% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +% + +-module(test_bs_gettail). +-export([start/0, id/1]). + +%% Matching-only (get_tail) validation: byte-aligned construction, then match a +%% non-byte-aligned prefix capturing the remainder as a bitstring, then re-match. +start() -> + Bin = <<(id(16#12345678)):32>>, + <> = Bin, + 152709948 = I, + 31 = bit_size(Bin) - bit_size(V), + 1 = bit_size(V), + false = is_binary(V), + true = is_bitstring(V), + <<0:1>> = V, + Bin2 = <<(id(16#12345679)):32>>, + <> = Bin2, + false = is_binary(W), + true = is_bitstring(W), + <<1:1>> = W, + 0. + +id(X) -> X. diff --git a/tests/erlang_tests/test_bs_int_unaligned.erl b/tests/erlang_tests/test_bs_int_unaligned.erl index 25ce755d35..20709a4db4 100644 --- a/tests/erlang_tests/test_bs_int_unaligned.erl +++ b/tests/erlang_tests/test_bs_int_unaligned.erl @@ -32,14 +32,9 @@ start() -> %% expressions that traverse a byte boundary ok = test_pack_unpack(13, 3, 1, 7, false), ok = test_pack_unpack(3, 13, 1, 7, false), - %% expressions not aligned on 8 bit boundary (expect failure with AtomVM) - ExpectFailure = - case erlang:system_info(machine) of - "BEAM" -> false; - _ -> true - end, - ok = test_pack_unpack(1, 1, 1, 1, ExpectFailure), - ok = test_pack_unpack(3, 13, 1, 1, ExpectFailure), + %% expressions not aligned on 8 bit boundary + ok = test_pack_unpack(1, 1, 1, 1, false), + ok = test_pack_unpack(3, 13, 1, 1, false), 0. test_pack_unpack(ALen, BLen, CLen, DLen, ExpectFailure) -> diff --git a/tests/erlang_tests/test_no_bs_create_bin.erl b/tests/erlang_tests/test_no_bs_create_bin.erl index 0868c345af..180fd9980b 100644 --- a/tests/erlang_tests/test_no_bs_create_bin.erl +++ b/tests/erlang_tests/test_no_bs_create_bin.erl @@ -39,7 +39,8 @@ put_utf16/1, put_utf32/1, build_from_list/1, - bin_comprehension/1 + bin_comprehension/1, + ext_id/1 ]). -if(?OTP_RELEASE =< 27). @@ -62,6 +63,7 @@ start() -> ok = test_put_utf32(), ok = test_private_append(), ok = test_bin_comprehension(), + ok = test_bitstring_source(), 0. %% bs_add + bs_init_bits + bs_put_integer with dynamic size. @@ -133,6 +135,81 @@ test_bin_comprehension() -> <<42>> = bin_comprehension([42]), ok. +%% bs_append / bs_private_append / bs_put_binary with a non-byte-aligned +%% source. These opcodes are bit-granular: a partial bitstring keeps its +%% trailing bits in the result and every segment written after it shifts by +%% that many bits, exactly as bs_create_bin does. A /binary segment (unit 8) +%% still rejects a partial source with badarg, as on BEAM. +test_bitstring_source() -> + %% a byte-aligned source is unaffected + <<1, 2, 3, 4>> = concat_bins(id(<<1, 2>>), id(<<3, 4>>)), + <<1, 2, 3>> = build_from_list(id([1, 2, 3])), + <<1, 2>> = copy_bitstring(id(<<1, 2>>)), + <<>> = copy_bitstring(id(<<>>)), + <<1, 2, 3>> = build_from_list_onto(id(<<>>), [1, 2, 3]), + + Bits1 = id(<<1:1>>), + Bits3 = id(<<5:3>>), + Bits9 = id(<<16#AB:8, 1:1>>), + + %% bs_append: a partial source is copied whole, alone and followed by a + %% segment that shifts by the trailing bit count + <<1:1>> = copy_bitstring(Bits1), + <<5:3>> = copy_bitstring(Bits3), + <<16#AB:8, 1:1>> = copy_bitstring(Bits9), + <<1:1, 16#AB:8>> = append_byte_to_bitstring(Bits1, 16#AB), + <<5:3, 16#AB:8>> = append_byte_to_bitstring(Bits3, 16#AB), + <<16#AB:8, 1:1, 16#CD:8>> = append_byte_to_bitstring(Bits9, 16#CD), + + %% bs_put_binary with an explicit bit size, and with a partial source + %% written at an already unaligned destination offset + <<5:3>> = put_bitstring_sized(Bits3, 3), + <<5:3, 1:1>> = append_bits_to_bitstring(Bits3, id(<<1:1>>)), + <<16#AB:8, 1:1, 5:3>> = append_bits_to_bitstring(Bits9, Bits3), + + %% bs_private_append: the accumulator itself is a partial bitstring, so it + %% cannot be grown in place + <<1:1, 1, 2, 3>> = build_from_list_onto(Bits1, [1, 2, 3]), + <<5:3>> = build_from_list_onto(Bits3, []), + + %% a /binary segment rejects a partial source rather than truncating it + badarg = expect_error(fun() -> append_binary_to(Bits1, id(<<1, 2>>)) end), + ok. + +copy_bitstring(Bits) -> + <>. + +append_binary_to(Bits, Bin) -> + <>. + +expect_error(Fun) -> + try + Fun(), + no_error + catch + error:Reason -> Reason + end. + +append_byte_to_bitstring(Bits, Byte) -> + <>. + +append_bits_to_bitstring(Bits, More) -> + <>. + +put_bitstring_sized(Bits, Size) -> + <>. + +build_from_list_onto(Acc, []) -> + Acc; +build_from_list_onto(Acc, [H | T]) -> + build_from_list_onto(<>, T). + +id(X) -> + ?MODULE:ext_id(X). + +ext_id(X) -> + X. + make_bin(Size, Val) -> <>. diff --git a/tests/libs/jit/jit_tests.erl b/tests/libs/jit/jit_tests.erl index f731bdac50..c1ad460843 100644 --- a/tests/libs/jit/jit_tests.erl +++ b/tests/libs/jit/jit_tests.erl @@ -217,8 +217,10 @@ term_to_int_verify_is_match_state_typed_optimization_x86_64_test() -> ), % Check the reading of x[1] is immediatly followed by a shift right. + % Untagging is an arithmetic shift (sar), so that a negative small integer + % stays negative instead of becoming a large positive value. % 15c: 4c 8b 5f 38 mov 0x38(%rdi),%r11 - % 160: 49 c1 eb 04 shr $0x4,%r11 + % 160: 49 c1 fb 04 sar $0x4,%r11 % As opposed to testing its type % 15c: 4c 8b 5f 38 mov 0x38(%rdi),%r11 @@ -227,10 +229,10 @@ term_to_int_verify_is_match_state_typed_optimization_x86_64_test() -> % 167: 41 80 fa 0f cmp $0xf,%r10b % 16b: 74 05 je 0x172 % 16d: e9 ab 00 00 00 jmpq 0x21d - % 172: 49 c1 eb 04 shr $0x4,%r11 + % 172: 49 c1 fb 04 sar $0x4,%r11 ?assertMatch( {_, 8}, - binary:match(CompiledCode, <<16#4c, 16#8b, 16#5f, 16#38, 16#49, 16#c1, 16#eb, 16#04>>) + binary:match(CompiledCode, <<16#4c, 16#8b, 16#5f, 16#38, 16#49, 16#c1, 16#fb, 16#04>>) ), % Check call to bs_start_match3 is followed by a skip of verify_is_boxed diff --git a/tests/test-bitstring.c b/tests/test-bitstring.c new file mode 100644 index 0000000000..f10e9fb1af --- /dev/null +++ b/tests/test-bitstring.c @@ -0,0 +1,210 @@ +/* + * This file is part of AtomVM. + * + * Copyright 2026 Paul Guyot + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later + */ + +#include +#include +#include + +#include "bitstring.h" +#include "context.h" +#include "globalcontext.h" +#include "memory.h" +#include "term.h" +#include "utils.h" + +// A copy must touch exactly the destination bytes that hold the copied bits, +// and read exactly the source bytes that hold them. Both buffers are bracketed +// with guard bytes so an off-by-one byte is caught here rather than by a +// sanitizer on an unrelated allocation. +#define GUARD 0x5A + +struct guarded +{ + uint8_t before[4]; + uint8_t data[16]; + uint8_t after[4]; +}; + +static void guarded_init(struct guarded *g, uint8_t fill) +{ + memset(g->before, GUARD, sizeof(g->before)); + memset(g->data, fill, sizeof(g->data)); + memset(g->after, GUARD, sizeof(g->after)); +} + +static void guarded_check(const struct guarded *g) +{ + for (size_t i = 0; i < sizeof(g->before); i++) { + assert(g->before[i] == GUARD); + } + for (size_t i = 0; i < sizeof(g->after); i++) { + assert(g->after[i] == GUARD); + } +} + +// Reference implementation: copy bits_count bits from the start of src to +// bits_offset in dst, one bit at a time, touching nothing else. +static void reference_copy_bits( + uint8_t *dst, size_t bits_offset, const uint8_t *src, size_t bits_count) +{ + for (size_t i = 0; i < bits_count; i++) { + size_t dst_bit = bits_offset + i; + uint8_t mask = (uint8_t) (1U << (7 - (dst_bit % 8))); + if (src[i / 8] & (uint8_t) (1U << (7 - (i % 8)))) { + dst[dst_bit / 8] |= mask; + } else { + dst[dst_bit / 8] &= (uint8_t) ~mask; + } + } +} + +static void check_copy_bits(size_t bits_offset, size_t bits_count, uint8_t src_fill) +{ + struct guarded src; + guarded_init(&src, src_fill); + + struct guarded got; + struct guarded expected; + // Two different fills, so a byte the copy must not touch differs from the + // byte it would be overwritten with. + guarded_init(&got, 0xC3); + guarded_init(&expected, 0xC3); + + reference_copy_bits(expected.data, bits_offset, src.data, bits_count); + bitstring_copy_bits(got.data, bits_offset, src.data, bits_count); + + // The bytes past the last copied bit must be untouched, and no guard byte + // on either side of either buffer may have moved. + assert(memcmp(got.data, expected.data, sizeof(got.data)) == 0); + guarded_check(&got); + guarded_check(&src); +} + +void test_copy_bits_boundaries(void) +{ + // Exact byte boundaries at an unaligned destination: the last copied bit + // completes the destination byte, so nothing past it may be read or written. + check_copy_bits(1, 7, 0xFF); + check_copy_bits(1, 7, 0x00); + // The source is consumed to its last bit and must not be read past it. + check_copy_bits(1, 8, 0xFF); + check_copy_bits(1, 8, 0x00); + // A zero-bit copy touches neither buffer. + check_copy_bits(0, 0, 0xFF); + check_copy_bits(1, 0, 0xFF); + check_copy_bits(7, 0, 0xFF); + + // Sweep offsets and counts, aligned and unaligned, including whole bytes. + for (size_t offset = 0; offset < 16; offset++) { + for (size_t count = 0; count <= 64; count++) { + check_copy_bits(offset, count, 0xFF); + check_copy_bits(offset, count, 0x00); + check_copy_bits(offset, count, 0xA5); + } + } +} + +// Copying into a destination sized to exactly the bits it holds, as a refc +// binary is: a byte of slack would hide a one-byte overrun. +void test_copy_bits_exact_allocation(void) +{ + for (size_t bits = 1; bits <= 24; bits++) { + size_t bytes = (bits + 7) / 8; + uint8_t *dst = calloc(bytes, 1); + assert(dst != NULL); + uint8_t src[4] = { 0xFF, 0xFF, 0xFF, 0xFF }; + // starting at bit 1 leaves the copy ending mid-byte or exactly on a + // byte boundary depending on the count + if (1 + bits <= bytes * 8) { + bitstring_copy_bits(dst, 1, src, bits); + } + free(dst); + } +} + +void test_print_bitstring(void) +{ + GlobalContext *glb = globalcontext_new(); + Context *ctx = context_new(glb); + + // Reserve every term this function builds in one go: memory_ensure_free may + // collect, and the terms below are plain C locals rather than roots, so a + // second reservation would leave the earlier ones dangling. + size_t heap_size = 3 * TERM_BOXED_SUB_BINARY_SIZE + 3 * term_binary_heap_size(1) + + 2 * term_binary_heap_size(2); + assert(memory_ensure_free(ctx, heap_size) == MEMORY_GC_OK); + + // <<1:1>>: no complete byte, a single trailing bit + term bin1 = term_create_empty_binary(1, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin1))[0] = 0x80; + term bits1 = term_alloc_sub_binary_bits(bin1, 0, 0, 1, &ctx->heap); + char buf[64]; + int len = term_snprint(buf, sizeof(buf), bits1, glb); + assert(len > 0); + assert(strcmp(buf, "<<1:1>>") == 0); + + // <<255, 5:7>>: a complete byte followed by a partial one + term bin2 = term_create_empty_binary(2, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin2))[0] = 0xFF; + ((uint8_t *) term_binary_data(bin2))[1] = (uint8_t) (5 << 1); + term bits2 = term_alloc_sub_binary_bits(bin2, 0, 1, 7, &ctx->heap); + len = term_snprint(buf, sizeof(buf), bits2, glb); + assert(len > 0); + assert(strcmp(buf, "<<255,5:7>>") == 0); + + // a printable byte followed by trailing bits: the printable prefix must not + // be rendered as a quoted string that swallows the trailing field + term bin3 = term_create_empty_binary(2, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin3))[0] = 'a'; + ((uint8_t *) term_binary_data(bin3))[1] = (uint8_t) (1 << 5); + term bits3 = term_alloc_sub_binary_bits(bin3, 0, 1, 3, &ctx->heap); + len = term_snprint(buf, sizeof(buf), bits3, glb); + assert(len > 0); + assert(strcmp(buf, "<<\"a\",1:3>>") == 0); + + // a byte-aligned bitstring still prints as a plain binary, and a printable + // one still prints as a quoted string + term bin4 = term_create_empty_binary(1, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin4))[0] = 0; + len = term_snprint(buf, sizeof(buf), bin4, glb); + assert(len > 0); + assert(strcmp(buf, "<<0>>") == 0); + + term bin5 = term_create_empty_binary(1, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin5))[0] = 42; + len = term_snprint(buf, sizeof(buf), bin5, glb); + assert(len > 0); + assert(strcmp(buf, "<<\"*\">>") == 0); + + context_destroy(ctx); + globalcontext_destroy(glb); +} + +int main(int argc, char **argv) +{ + UNUSED(argc); + UNUSED(argv); + + test_copy_bits_boundaries(); + test_copy_bits_exact_allocation(); + test_print_bitstring(); + + return EXIT_SUCCESS; +} diff --git a/tests/test.c b/tests/test.c index 342c142fd4..efe1bf2fc9 100644 --- a/tests/test.c +++ b/tests/test.c @@ -417,6 +417,8 @@ struct Test tests[] = { TEST_CASE(test_bs_int), TEST_CASE(test_bs_int_any_flags), TEST_CASE(test_bs_int_unaligned), + TEST_CASE(test_bs_gettail), + TEST_CASE(test_bitstring_to_list), TEST_CASE(test_bs_start_match_live), TEST_CASE(test_bs_utf), TEST_CASE_EXPECTED(bs_append_extra_words, 1),