diff --git a/.github/workflows/build-and-test-macos.yaml b/.github/workflows/build-and-test-macos.yaml index 1d18a21030..dc0dc1f940 100644 --- a/.github/workflows/build-and-test-macos.yaml +++ b/.github/workflows/build-and-test-macos.yaml @@ -259,6 +259,12 @@ jobs: run: | ./tests/test-term + - name: "Test: test-bitstring" + timeout-minutes: 10 + working-directory: build + run: | + ./tests/test-bitstring + - name: "Test: test_etest.avm" timeout-minutes: 5 working-directory: build diff --git a/.github/workflows/build-and-test-on-freebsd.yaml b/.github/workflows/build-and-test-on-freebsd.yaml index d7a3b49407..7cb8fd824f 100644 --- a/.github/workflows/build-and-test-on-freebsd.yaml +++ b/.github/workflows/build-and-test-on-freebsd.yaml @@ -164,6 +164,13 @@ jobs: cd build ./tests/test-term + - name: "Test: test-bitstring" + shell: freebsd {0} + run: | + cd $GITHUB_WORKSPACE; + cd build + ./tests/test-bitstring + - name: "Test: test_etest.avm" shell: freebsd {0} run: | diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 48de1aa824..43ae9e6a75 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -808,6 +808,19 @@ jobs: ulimit -c unlimited ./tests/test-erlang -s prime_smp + - name: "Test: test-bitstring with valgrind" + if: matrix.library-arch == '' + working-directory: build + run: | + ulimit -c unlimited + valgrind --error-exitcode=1 ./tests/test-bitstring + + - name: "Test: test-bitstring" + working-directory: build + run: | + ulimit -c unlimited + ./tests/test-bitstring + - name: "Test: test-enif with valgrind" if: matrix.library-arch == '' working-directory: build diff --git a/.github/workflows/build-linux-artifacts.yaml b/.github/workflows/build-linux-artifacts.yaml index cd65b9cc87..c060775a3c 100644 --- a/.github/workflows/build-linux-artifacts.yaml +++ b/.github/workflows/build-linux-artifacts.yaml @@ -232,6 +232,7 @@ jobs: cp ../build_tests/tests/libs/alisp/*.avm tests/libs/alisp/ && VERBOSE=1 make AtomVM && make test-erlang && + make test-bitstring && make test-enif && make test-heap && make test-mailbox && @@ -239,6 +240,8 @@ jobs: make test-term && file ./tests/test-erlang && ./tests/test-erlang -s prime_smp && + file ./tests/test-bitstring && + ./tests/test-bitstring && file ./tests/test-enif && ./tests/test-enif && file ./tests/test-heap && diff --git a/CHANGELOG.md b/CHANGELOG.md index d7d1ef0f1a..d6cd259293 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 The emscripten module object gained `trackedObjectsMap`, `nextTrackedObjectKey()` and the `onRunTrackedJs`, `onGetTrackedObjects` and `onTrackedObjectDelete` hooks, which embedders may override to customize what tracking means +- Added support for non-byte-aligned bitstrings ### Changed - `erlang:process_info/2` now accepts only pids of local processes, as Erlang/OTP does: @@ -109,6 +110,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed `term_is_uint32` accepting big integers whose low 64 bits are within range on 32-bit builds, which made `erlang:crc32/2`, `erlang:crc32_combine/3` and `crypto:pbkdf2_hmac/5` silently truncate huge integer arguments instead of raising `badarg` +- Fixed `erlang:list_to_binary/1`, `erlang:iolist_to_binary/1`, `erlang:iolist_size/1` and the + `unicode:characters_to_*` functions accepting an integer improper tail, such as `[1 | 2]` ## [0.7.0-alpha.1] - 2026-04-06 diff --git a/code-queries/allocations-exceeding-ensure-free.ql b/code-queries/allocations-exceeding-ensure-free.ql index e669704a20..961a5ef64c 100644 --- a/code-queries/allocations-exceeding-ensure-free.ql +++ b/code-queries/allocations-exceeding-ensure-free.ql @@ -950,6 +950,14 @@ predicate consumesContextBudget(FunctionCall efCall, FunctionCall consumer) { ensureFreeContextVar(efCall) } +/** + * Holds if `f` tears down a context (e.g. `context_destroy`): it frees the + * context rather than re-establishing a usable heap budget on it. + */ +predicate isContextTeardown(Function f) { + f.hasName("context_destroy") +} + /** * Holds if `efCall` is a redundant reserving ensure_free: no allocating call * uses its budget, and `supersedingCall` is a subsequent call that resets @@ -959,6 +967,8 @@ predicate consumesContextBudget(FunctionCall efCall, FunctionCall consumer) { pragma[nomagic] predicate isRedundantEnsureFree(FunctionCall efCall, FunctionCall supersedingCall) { isReservingEnsureFreeCall(efCall) and + // Test code calls ensure_free for its GC side effects + not efCall.getFile().getRelativePath().matches("tests/%") and // No allocating call uses this ensure_free's budget not exists(FunctionCall a | allocToBudget(a, efCall)) and // ...and no pointer-bumping consumer (memory_copy_term_tree) uses it either @@ -986,8 +996,10 @@ predicate isRedundantEnsureFree(FunctionCall efCall, FunctionCall supersedingCal // A function that internally calls ensure_free on the caller's // context (e.g., enif_make_resource), passed that same context as an // argument. Uses the ensure_free-only notion: own-heap setup does not - // reset this context's budget. + // reset this context's budget. Context teardown (context_destroy) is + // excluded: it frees the context rather than re-establishing a budget. not isEnsureFreeCall(supersedingCall) and + not isContextTeardown(supersedingCall.getTarget()) and transitivelyCallsEnsureFreeOnly(supersedingCall.getTarget()) and supersedingCall.getAnArgument().(VariableAccess).getTarget() = ctxVar ) and diff --git a/doc/src/apidocs/libatomvm/functions.rst b/doc/src/apidocs/libatomvm/functions.rst index e25a540eef..3e734b4a0c 100644 --- a/doc/src/apidocs/libatomvm/functions.rst +++ b/doc/src/apidocs/libatomvm/functions.rst @@ -242,7 +242,6 @@ Functions .. doxygenfunction:: term_binary_heap_size .. doxygenfunction:: term_binary_size_is_heap_binary .. doxygenfunction:: term_boxed_size -.. doxygenfunction:: term_bs_insert_binary .. doxygenfunction:: term_compare .. doxygenfunction:: term_create_empty_binary .. doxygenfunction:: term_create_uninitialized_binary diff --git a/doc/src/differences-with-beam.md b/doc/src/differences-with-beam.md index 687af7ee04..00c69790ba 100644 --- a/doc/src/differences-with-beam.md +++ b/doc/src/differences-with-beam.md @@ -146,17 +146,25 @@ Since `-0` is not allowed, the result is normalized to `0`. ### Bit syntax -AtomVM supports binaries, binary construction and binary pattern matching. Bit syntax (i.e. with -sizes not a multiple of 8) is only supported as long as they would not generate bitstrings, i.e. -binaries with a number if bits that are not multiple of 8. +AtomVM supports binaries and bitstrings, including construction and pattern matching of bitstrings +whose size in bits is not a multiple of 8: -The following is supported: + <> = <<1:3>> - <> = <> +There is one limitation, which follows from the 256-bit integer limit described above: a segment +holding an integer that does not fit in 64 bits is only supported when it is byte-aligned, meaning +both its offset in the bitstring and its size in bits are multiples of 8. Otherwise AtomVM raises +`unsupported`, where BEAM would construct or match the segment. -The following is not: +The following is supported, as the segment starts at offset 0 and is 72 bits long: - <> + <> = <<(1 bsl 70):72>> + +The following is not, as the 72-bit segment starts at bit offset 1: + + <<_:1, X:72, _:7>> = Bin + +Integers that do fit in 64 bits have no such restriction and may be used at any offset and size. ### Code reloading diff --git a/libs/estdlib/src/binary.erl b/libs/estdlib/src/binary.erl index 55e76af863..24b10cda13 100644 --- a/libs/estdlib/src/binary.erl +++ b/libs/estdlib/src/binary.erl @@ -77,11 +77,13 @@ copy(_Binary, _N) -> %% @end %%----------------------------------------------------------------------------- -spec decode_hex(Data :: <<_:_*16>>) -> binary(). -decode_hex(Data) -> +decode_hex(Data) when is_binary(Data) -> case byte_size(Data) rem 2 of 0 -> <<<<(binary_to_integer(B, 16))>> || <> <= Data>>; _ -> erlang:error(badarg) - end. + end; +decode_hex(_Data) -> + erlang:error(badarg). %%----------------------------------------------------------------------------- %% @param Data binary data to convert into hex encoded binary @@ -101,10 +103,12 @@ encode_hex(Data) -> %% @end %%----------------------------------------------------------------------------- -spec encode_hex(Data :: binary(), Case :: lowercase | uppercase) -> binary(). -encode_hex(Data, uppercase) -> +encode_hex(Data, uppercase) when is_binary(Data) -> <<(integer_to_binary(B, 16)) || <> <= Data>>; -encode_hex(Data, lowercase) -> - <<<<(hd(string:to_lower(integer_to_list(B, 16)))):8>> || <> <= Data>>. +encode_hex(Data, lowercase) when is_binary(Data) -> + <<<<(hd(string:to_lower(integer_to_list(B, 16)))):8>> || <> <= Data>>; +encode_hex(_Data, _Case) -> + erlang:error(badarg). %%----------------------------------------------------------------------------- %% @param Binaries non-empty list of binaries diff --git a/libs/estdlib/src/erlang.erl b/libs/estdlib/src/erlang.erl index b648e6ffe7..c02f797e86 100644 --- a/libs/estdlib/src/erlang.erl +++ b/libs/estdlib/src/erlang.erl @@ -62,6 +62,7 @@ list_to_atom/1, list_to_existing_atom/1, list_to_binary/1, + list_to_bitstring/1, list_to_integer/1, list_to_integer/2, list_to_tuple/1, @@ -156,6 +157,7 @@ hd/1, is_atom/1, is_binary/1, + is_bitstring/1, is_boolean/1, is_float/1, is_function/1, @@ -192,11 +194,15 @@ stacktrace/0, stacktrace_extrainfo/0, raise_stacktrace/0, - term_to_binary_option/0 + term_to_binary_option/0, + bitstring_list/0 ]). -type atom_encoding() :: latin1 | utf8 | unicode. +-type bitstring_list() :: + maybe_improper_list(byte() | bitstring() | bitstring_list(), bitstring() | []). + -type mem_type() :: binary. -type time_unit() :: second | millisecond | microsecond | nanosecond | native. -type timestamp() :: { @@ -830,6 +836,22 @@ list_to_existing_atom(_String) -> list_to_binary(_IOList) -> erlang:nif_error(undefined). +%%----------------------------------------------------------------------------- +%% @param BitstringList list to convert to bitstring +%% @returns a bitstring composed of bytes and bitstrings from the list +%% @doc Convert a list into a bitstring. +%% +%% Unlike `list_to_binary/1', the elements of the list may be +%% bitstrings that are not a whole number of bytes, and so may be the +%% result. +%% +%% Errors with `badarg' if the list is not a list of bytes and bitstrings. +%% @end +%%----------------------------------------------------------------------------- +-spec list_to_bitstring(BitstringList :: bitstring_list()) -> bitstring(). +list_to_bitstring(_BitstringList) -> + erlang:nif_error(undefined). + %%----------------------------------------------------------------------------- %% @param String string to convert to integer %% @returns an integer value from its string representation @@ -973,15 +995,16 @@ binary_to_list(_Binary) -> %%----------------------------------------------------------------------------- %% @param Bitstring Bitstring to convert to list -%% @returns a list of bytes from the bitstring +%% @returns a list of bytes, with a final element holding the trailing bits if +%% `Bitstring' is not a whole number of bytes %% @doc Convert a bitstring to a list of bytes. %% -%% Unlike Erlang/OTP, AtomVM only supports byte-aligned bitstrings (binaries), -%% so the returned list never has a trailing bitstring and this function -%% behaves like `binary_to_list/1'. +%% If the number of bits in `Bitstring' is not divisible by `8', the +%% last element of the list is a bitstring containing the trailing +%% `1..7' bits, e.g. `bitstring_to_list(<<1:1>>)' returns `[<<1:1>>]'. %% @end %%----------------------------------------------------------------------------- --spec bitstring_to_list(Bitstring :: bitstring()) -> [byte()]. +-spec bitstring_to_list(Bitstring :: bitstring()) -> [byte() | bitstring()]. bitstring_to_list(_Bitstring) -> erlang:nif_error(undefined). @@ -2013,6 +2036,19 @@ is_atom(_Term) -> is_binary(_Term) -> erlang:nif_error(undefined). +%%----------------------------------------------------------------------------- +%% @param Term the term to test +%% @returns `true' if `Term' is a bitstring; `false', otherwise. +%% @doc Return `true' if `Term' is a bitstring (including a binary); +%% `false', otherwise. +%% +%% This function may be used in a guard expression. +%% @end +%%----------------------------------------------------------------------------- +-spec is_bitstring(Term :: term()) -> boolean(). +is_bitstring(_Term) -> + erlang:nif_error(undefined). + %%----------------------------------------------------------------------------- %% @param Term the term to test %% @returns `true' if `Term' is a boolean (`true' or `false'); `false', otherwise. diff --git a/libs/estdlib/src/io_lib.erl b/libs/estdlib/src/io_lib.erl index 7930575d9b..cd875bebe2 100644 --- a/libs/estdlib/src/io_lib.erl +++ b/libs/estdlib/src/io_lib.erl @@ -316,6 +316,14 @@ format_spw(#format{control = w} = Format, L) when is_list(L) -> [$[, lists:join($,, [format_spw(Format, E) || E <- L]), $]]; format_spw(#format{control = s}, _) -> error(badarg); +format_spw(_Format, T) when is_bitstring(T) -> + TrailingBits = bit_size(T) rem 8, + WholeBits = bit_size(T) - TrailingBits, + <> = T, + Numbers = + [erlang:integer_to_list(B) || <> <= Whole] ++ + [[erlang:integer_to_list(Trailing), $:, erlang:integer_to_list(TrailingBits)]], + [$<, $<, lists:join($,, Numbers), $>, $>]; format_spw(_Format, T) when is_integer(T) -> erlang:integer_to_list(T); format_spw(_Format, T) when is_float(T) -> diff --git a/libs/jit/src/default_atoms.hrl b/libs/jit/src/default_atoms.hrl index ae9dbf7e5b..c206e5cd2a 100644 --- a/libs/jit/src/default_atoms.hrl +++ b/libs/jit/src/default_atoms.hrl @@ -37,6 +37,7 @@ -define(ALL_ATOM_INDEX, 15). -define(LOWERCASE_EXIT_ATOM_INDEX, 16). -define(BADRECORD_ATOM_INDEX, 17). +-define(SYSTEM_LIMIT_ATOM_INDEX, 18). -define(FALSE_ATOM, ((?FALSE_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM)). -define(TRUE_ATOM, ((?TRUE_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM)). @@ -68,6 +69,9 @@ ((?LOWERCASE_EXIT_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM) ). -define(BADRECORD_ATOM, ((?BADRECORD_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM)). +-define(SYSTEM_LIMIT_ATOM, + ((?SYSTEM_LIMIT_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM) +). -define(DEFAULT_ATOMS, #{ false => ?FALSE_ATOM, diff --git a/libs/jit/src/jit.erl b/libs/jit/src/jit.erl index f8d4f6a6df..fc89bf3e7c 100644 --- a/libs/jit/src/jit.erl +++ b/libs/jit/src/jit.erl @@ -801,10 +801,19 @@ first_pass(<>, MMod, MSt0, State0) -> {Label, Rest1} = decode_label(Rest0), {MSt1, Arg1, Rest2} = decode_compact_term(Rest1, MMod, MSt0, State0), ?TRACE("OP_IS_BINARY ~p, ~p\n", [Label, Arg1]), - MSt2 = verify_is_binary(Arg1, Label, MMod, MSt1), - MSt3 = MMod:free_native_registers(MSt2, [Arg1]), - ?ASSERT_ALL_NATIVE_FREE(MSt3), - first_pass(Rest2, MMod, MSt3, State0); + MSt2 = verify_is_bitstring(Arg1, Label, MMod, MSt1), + {MSt3, Reg} = MMod:move_to_native_register(MSt2, Arg1), + {MSt4, Reg} = MMod:and_(MSt3, {free, Reg}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt5, TagReg} = MMod:get_array_element(MSt4, Reg, 0), + {MSt6, TagReg} = MMod:and_(MSt5, {free, TagReg}, ?TERM_BOXED_TAG_MASK), + %% is_binary/1 is false for a non-byte-aligned bitstring + MSt7 = MMod:if_block(MSt6, {{free, TagReg}, '==', ?TERM_BOXED_SUB_BINARY}, fun(BSt0) -> + {BSt1, OffsetReg} = MMod:get_array_element(BSt0, Reg, 2), + cond_jump_to_label({{free, OffsetReg}, '&', 16#7, '!=', 0}, Label, MMod, BSt1) + end), + MSt8 = MMod:free_native_registers(MSt7, [Reg]), + ?ASSERT_ALL_NATIVE_FREE(MSt8), + first_pass(Rest2, MMod, MSt8, State0); % 55 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -1375,10 +1384,6 @@ first_pass(<>, MMod, MSt0, State0) -> {MSt5, BSOffsetReg0} = MMod:get_array_element(MSt4, MatchStateRegPtr, 2), MSt6 = if - Unit =/= 8 -> - MMod:call_primitive_last(MSt5, ?PRIM_RAISE_ERROR, [ - ctx, jit_state, offset, ?UNSUPPORTED_ATOM - ]); FlagsValue =/= 0 -> MMod:call_primitive_last(MSt5, ?PRIM_RAISE_ERROR, [ ctx, jit_state, offset, ?UNSUPPORTED_ATOM @@ -1386,48 +1391,46 @@ first_pass(<>, MMod, MSt0, State0) -> true -> MSt5 end, - MSt7 = MMod:if_block(MSt6, {BSOffsetReg0, '&', 16#7, '!=', 0}, fun(BlockSt) -> - MMod:call_primitive_last(BlockSt, ?PRIM_RAISE_ERROR, [ctx, jit_state, offset, ?BADARG_ATOM]) - end), - {MSt8, BSOffsetReg1} = MMod:shift_right(MSt7, {free, BSOffsetReg0}, 3), - {MSt9, BSBinaryReg0} = MMod:and_(MSt8, {free, BSBinaryReg0}, ?TERM_PRIMARY_CLEAR_MASK), - {MSt10, SizeReg} = MMod:get_array_element(MSt9, {free, BSBinaryReg0}, 1), - {MSt13, SizeValue} = + {MSt7, BinPtrReg} = MMod:and_(MSt6, {free, BSBinaryReg0}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt8, RemainingReg} = term_bit_size({ptr, BinPtrReg}, MMod, MSt7), + MSt9 = MMod:free_native_registers(MSt8, [BinPtrReg]), + MSt10 = MMod:sub(MSt9, RemainingReg, BSOffsetReg0), + {MSt14, SizeBits} = if Size =:= ?ALL_ATOM -> - MSt11 = MMod:sub(MSt10, SizeReg, BSOffsetReg1), - {MSt11, SizeReg}; + MSt11 = bs_fail_or_badarg_unless_multiple_of_unit( + RemainingReg, Unit, Fail, MMod, MSt10 + ), + {MSt11, RemainingReg}; is_integer(Size) -> - % SizeReg is binary size % Size is a tagged integer: (N bsl 4) bor 0xF - % SizeBytes is the raw byte count - SizeBytes = Size bsr 4, - MSt11 = MMod:sub(MSt10, SizeReg, SizeBytes), - MSt12 = cond_jump_to_label({{free, SizeReg}, '<', BSOffsetReg1}, Fail, MMod, MSt11), - {MSt12, SizeBytes}; + SizeBitsLit = (Size bsr 4) * Unit, + MSt11 = cond_jump_to_label( + {{free, RemainingReg}, '<', SizeBitsLit}, Fail, MMod, MSt10 + ), + {MSt11, SizeBitsLit}; true -> {MSt11, SizeValReg} = MMod:move_to_native_register(MSt10, Size), MSt12 = MMod:if_else_block( MSt11, {SizeValReg, '==', ?ALL_ATOM}, fun(BSt0) -> - BSt1 = MMod:sub(BSt0, SizeReg, BSOffsetReg1), - MMod:free_native_registers(BSt1, [SizeValReg]) + BSt1 = bs_fail_or_badarg_unless_multiple_of_unit( + RemainingReg, Unit, Fail, MMod, BSt0 + ), + MMod:move_to_native_register(BSt1, RemainingReg, SizeValReg) end, fun(BSt0) -> {BSt1, SizeValReg} = term_to_int(SizeValReg, Fail, MMod, BSt0), - BSt2 = cond_jump_to_label({SizeValReg, '<', 0}, Fail, MMod, BSt1), - BSt3 = MMod:sub(BSt2, SizeReg, SizeValReg), - BSt4 = cond_jump_to_label({SizeReg, '<', BSOffsetReg1}, Fail, MMod, BSt3), - BSt5 = MMod:move_to_native_register(BSt4, SizeValReg, SizeReg), - MMod:free_native_registers(BSt5, [SizeValReg]) + BSt2 = scale_size_by_unit(SizeValReg, Unit, Fail, MMod, BSt1), + cond_jump_to_label({RemainingReg, '<', SizeValReg}, Fail, MMod, BSt2) end ), - {MSt12, SizeReg} + MSt13 = MMod:free_native_registers(MSt12, [RemainingReg]), + {MSt13, SizeValReg} end, - {MSt14, NewOffsetReg} = MMod:copy_to_native_register(MSt13, BSOffsetReg1), - MSt15 = MMod:add(MSt14, NewOffsetReg, SizeValue), - MSt16 = MMod:shift_left(MSt15, NewOffsetReg, 3), + {MSt15, NewOffsetReg} = MMod:copy_to_native_register(MSt14, BSOffsetReg0), + MSt16 = MMod:add(MSt15, NewOffsetReg, SizeBits), % Write new offset MSt17 = MMod:move_to_array_element(MSt16, NewOffsetReg, MatchStateRegPtr, 2), MSt18 = MMod:free_native_registers(MSt17, [NewOffsetReg]), @@ -1435,23 +1438,31 @@ first_pass(<>, MMod, MSt0, State0) -> MSt20 = MMod:free_native_registers(MSt19, [TrimResultReg]), {MSt21, BSBinaryReg1} = MMod:get_array_element(MSt20, {free, MatchStateRegPtr}, 1), MSt22 = MMod:or_(MSt21, BSBinaryReg1, ?TERM_PRIMARY_BOXED), - {MSt23, HeapSizeReg} = MMod:call_primitive(MSt22, ?PRIM_TERM_SUB_BINARY_HEAP_SIZE, [ - BSBinaryReg1, SizeValue + {MSt23, TmpPtrReg} = MMod:copy_to_native_register(MSt22, BSBinaryReg1), + {MSt24, TmpPtrReg} = MMod:and_(MSt23, {free, TmpPtrReg}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt25, HeapSizeReg} = MMod:call_primitive(MSt24, ?PRIM_BITSTRING_SLICE_HEAP_SIZE, [ + {free, TmpPtrReg}, BSOffsetReg0, SizeBits ]), - {MSt24, BSBinaryReg2} = memory_ensure_free_with_extra_root( - BSBinaryReg1, Live, {free, HeapSizeReg}, MMod, MSt23 + {MSt26, BSBinaryReg2} = memory_ensure_free_with_extra_root( + BSBinaryReg1, Live, {free, HeapSizeReg}, MMod, MSt25 ), - {MSt25, ResultTerm} = MMod:call_primitive(MSt24, ?PRIM_TERM_MAYBE_CREATE_SUB_BINARY, [ - ctx, {free, BSBinaryReg2}, {free, BSOffsetReg1}, {free, SizeValue} + SizeBitsArg = + if + is_integer(SizeBits) -> SizeBits; + true -> {free, SizeBits} + end, + {MSt27, ResultTerm} = MMod:call_primitive(MSt26, ?PRIM_BITSTRING_SLICE, [ + ctx, {free, BSBinaryReg2}, {free, BSOffsetReg0}, SizeBitsArg ]), - {MSt26, Dest, Rest7} = decode_dest(Rest6, MMod, MSt25), + MSt28 = raise_out_of_memory_if_invalid(ResultTerm, MMod, MSt27), + {MSt29, Dest, Rest7} = decode_dest(Rest6, MMod, MSt28), ?TRACE("OP_BS_GET_BINARY2 ~p,~p,~p,~p,~p,~p,~p\n", [ Fail, Src, Live, Size, Unit, FlagsValue, Dest ]), - MSt27 = MMod:move_to_vm_register(MSt26, ResultTerm, Dest), - MSt28 = MMod:free_native_registers(MSt27, [ResultTerm, Dest]), - ?ASSERT_ALL_NATIVE_FREE(MSt28), - first_pass(Rest7, MMod, MSt28, State0); + MSt30 = MMod:move_to_vm_register(MSt29, ResultTerm, Dest), + MSt31 = MMod:free_native_registers(MSt30, [ResultTerm, Dest]), + ?ASSERT_ALL_NATIVE_FREE(MSt31), + first_pass(Rest7, MMod, MSt31, State0); % 120 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -1475,13 +1486,12 @@ first_pass(<>, MMod, MSt0, State0) -> {MSt8, BSOffsetReg} = MMod:get_array_element(MSt7, MatchStateRegPtr, 2), MSt9 = MMod:add(MSt8, BSOffsetReg, NumBits), MSt10 = MMod:free_native_registers(MSt9, [NumBits]), - {MSt11, BSBinarySize} = term_binary_size({free, BSBinaryReg}, MMod, MSt10), - MSt12 = MMod:shift_left(MSt11, BSBinarySize, 3), - MSt13 = cond_jump_to_label({{free, BSBinarySize}, '<', BSOffsetReg}, Fail, MMod, MSt12), - MSt14 = MMod:move_to_array_element(MSt13, BSOffsetReg, MatchStateRegPtr, 2), - MSt15 = MMod:free_native_registers(MSt14, [BSOffsetReg, MatchStateRegPtr]), - ?ASSERT_ALL_NATIVE_FREE(MSt15), - first_pass(Rest5, MMod, MSt15, State0); + {MSt11, BSBinarySize} = term_bit_size({free, BSBinaryReg}, MMod, MSt10), + MSt12 = cond_jump_to_label({{free, BSBinarySize}, '<', BSOffsetReg}, Fail, MMod, MSt11), + MSt13 = MMod:move_to_array_element(MSt12, BSOffsetReg, MatchStateRegPtr, 2), + MSt14 = MMod:free_native_registers(MSt13, [BSOffsetReg, MatchStateRegPtr]), + ?ASSERT_ALL_NATIVE_FREE(MSt14), + first_pass(Rest5, MMod, MSt14, State0); % 121 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -1494,12 +1504,11 @@ first_pass(<>, MMod, MSt0, State0) -> {MSt4, BSOffsetReg} = MMod:get_array_element(MSt3, MatchStateRegPtr, 2), MSt5 = MMod:free_native_registers(MSt4, [MatchStateRegPtr]), MSt6 = MMod:add(MSt5, BSOffsetReg, Bits), - {MSt7, BSBinarySize} = term_binary_size({free, BSBinaryReg}, MMod, MSt6), - MSt8 = MMod:shift_left(MSt7, BSBinarySize, 3), - MSt9 = cond_jump_to_label({{free, BSBinarySize}, '!=', BSOffsetReg}, Fail, MMod, MSt8), - MSt10 = MMod:free_native_registers(MSt9, [BSOffsetReg]), - ?ASSERT_ALL_NATIVE_FREE(MSt10), - first_pass(Rest3, MMod, MSt10, State0); + {MSt7, BSBinarySize} = term_bit_size({free, BSBinaryReg}, MMod, MSt6), + MSt8 = cond_jump_to_label({{free, BSBinarySize}, '!=', BSOffsetReg}, Fail, MMod, MSt7), + MSt9 = MMod:free_native_registers(MSt8, [BSOffsetReg]), + ?ASSERT_ALL_NATIVE_FREE(MSt9), + first_pass(Rest3, MMod, MSt9, State0); % 124 first_pass( <>, MMod, MSt0, #state{import_resolver = ImportResolver} = State0 @@ -1568,16 +1577,15 @@ first_pass(<>, MMod, MSt0, State0) -> {MSt3, BSBinaryReg} = MMod:get_array_element(MSt2, MatchStateRegPtr, 1), {MSt4, BSOffsetReg} = MMod:get_array_element(MSt3, MatchStateRegPtr, 2), MSt5 = MMod:free_native_registers(MSt4, [MatchStateRegPtr]), - {MSt6, BSBinarySize} = term_binary_size({free, BSBinaryReg}, MMod, MSt5), - MSt7 = MMod:shift_left(MSt6, BSBinarySize, 3), - % BSBinarySize = binary_size * 8 - MSt8 = MMod:sub(MSt7, BSBinarySize, BSOffsetReg), + {MSt6, BSBinarySize} = term_bit_size({free, BSBinaryReg}, MMod, MSt5), + % BSBinarySize = source bit size + MSt7 = MMod:sub(MSt6, BSBinarySize, BSOffsetReg), % BSBinarySize = (binary_size * 8) - offset = remaining bits - MSt9 = MMod:free_native_registers(MSt8, [BSOffsetReg]), - {MSt10, BSBinarySize1} = MMod:and_(MSt9, {free, BSBinarySize}, Unit - 1), - MSt11 = cond_jump_to_label({{free, BSBinarySize1}, '!=', 0}, Fail, MMod, MSt10), - ?ASSERT_ALL_NATIVE_FREE(MSt11), - first_pass(Rest3, MMod, MSt11, State0); + MSt8 = MMod:free_native_registers(MSt7, [BSOffsetReg]), + MSt9 = bs_fail_or_badarg_unless_multiple_of_unit(BSBinarySize, Unit, Fail, MMod, MSt8), + MSt10 = MMod:free_native_registers(MSt9, [BSBinarySize]), + ?ASSERT_ALL_NATIVE_FREE(MSt10), + first_pass(Rest3, MMod, MSt10, State0); % 132 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -2356,83 +2364,86 @@ first_pass( {Rest6, MSt1, 0, undefined, State0, false}, lists:seq(1, NBSegments) ), - {MSt4, BinaryTotalSize} = + MaxBinarySize = max_binary_size_in_bytes(MMod), + {MSt6, BinaryTotalSize} = case {BinaryLitSize, BinaryRegSize} of + {_, _} when BinaryLitSize > MaxBinarySize * 8 -> + {raise_system_limit(MMod, MSt2), BinaryLitSize}; {_, undefined} -> {MSt2, BinaryLitSize}; {0, Reg} -> {MSt2, Reg}; {_, _} -> - MSt3 = MMod:add(MSt2, BinaryRegSize, BinaryLitSize), - {MSt3, BinaryRegSize} - end, - MSt5 = - if - is_integer(BinaryTotalSize) andalso BinaryTotalSize band 16#7 =/= 0 -> - MMod:call_primitive_last(MSt4, ?PRIM_RAISE_ERROR, [ - ctx, jit_state, offset, ?UNSUPPORTED_ATOM - ]); - is_integer(BinaryTotalSize) -> - MSt4; - true -> - MMod:if_block(MSt4, {BinaryTotalSize, '&', 16#7, '!=', 0}, fun(BlockSt) -> - MMod:call_primitive_last(BlockSt, ?PRIM_RAISE_ERROR, [ - ctx, jit_state, offset, ?UNSUPPORTED_ATOM - ]) - end) + {MSt3, LitReg} = MMod:move_to_native_register(MSt2, BinaryLitSize), + MSt4 = MMod:add(MSt3, BinaryRegSize, LitReg), + MSt5 = MMod:if_block(MSt4, {'(unsigned)', BinaryRegSize, '<', LitReg}, fun(BSt0) -> + raise_system_limit(MMod, BSt0) + end), + {MMod:free_native_registers(MSt5, [LitReg]), BinaryRegSize} end, - {MSt6, TrimResultReg} = MMod:call_primitive(MSt5, ?PRIM_TRIM_LIVE_REGS, [ctx, Live]), - MSt7 = MMod:free_native_registers(MSt6, [TrimResultReg]), - {MSt12, BinaryTotalSizeInBytes, AllocSize} = + {MSt7, TrimResultReg} = MMod:call_primitive(MSt6, ?PRIM_TRIM_LIVE_REGS, [ctx, Live]), + MSt8 = MMod:free_native_registers(MSt7, [TrimResultReg]), + {MSt19, BinaryTotalSizeInBytes, AllocSize, WrapInfo} = if is_integer(BinaryTotalSize) -> - {MSt7, (BinaryTotalSize div 8), - term_binary_heap_size((BinaryTotalSize div 8), MMod) + Alloc}; + Trailing = BinaryTotalSize rem 8, + ByteSize = (BinaryTotalSize + 7) div 8, + {Extra, Wrap} = + case Trailing of + 0 -> {0, no_wrap}; + _ -> {?TERM_BOXED_SUB_BINARY_SIZE, {wrap_lit, BinaryTotalSize}} + end, + MSt9 = + case ByteSize > MaxBinarySize of + true -> raise_system_limit(MMod, MSt8); + false -> MSt8 + end, + {MSt9, ByteSize, term_binary_heap_size(ByteSize, MMod) + Alloc + Extra, Wrap}; true -> - {MSt8, BinaryTotalSizeBytes} = MMod:shift_right(MSt7, {free, BinaryTotalSize}, 3), - {MSt9, BinaryTotalSizeBytes0} = MMod:copy_to_native_register( - MSt8, BinaryTotalSizeBytes - ), - {MSt10, AllocSizeReg} = term_binary_heap_size( - {free, BinaryTotalSizeBytes0}, MMod, MSt9 - ), - case Alloc of - 0 -> - {MSt10, BinaryTotalSizeBytes, AllocSizeReg}; - _ -> - MSt11 = MMod:add(MSt10, AllocSizeReg, Alloc), - {MSt11, BinaryTotalSizeBytes, AllocSizeReg} - end + {MSt9, SevenReg} = MMod:move_to_native_register(MSt8, 7), + MSt10 = MMod:add(MSt9, BinaryTotalSize, SevenReg), + MSt11 = MMod:if_block(MSt10, {'(unsigned)', BinaryTotalSize, '<', SevenReg}, fun( + BSt0 + ) -> + raise_system_limit(MMod, BSt0) + end), + MSt12 = MMod:free_native_registers(MSt11, [SevenReg]), + {MSt13, CeilBytes} = MMod:shift_right(MSt12, {free, BinaryTotalSize}, 3), + MSt14 = MMod:if_block(MSt13, {MaxBinarySize, '<', CeilBytes}, fun(BSt0) -> + raise_system_limit(MMod, BSt0) + end), + {MSt15, CeilCopy} = MMod:copy_to_native_register(MSt14, CeilBytes), + {MSt16, AllocSizeReg} = term_binary_heap_size({free, CeilCopy}, MMod, MSt15), + MSt17 = MMod:add(MSt16, AllocSizeReg, ?TERM_BOXED_SUB_BINARY_SIZE), + MSt18 = + case Alloc of + 0 -> MSt17; + _ -> MMod:add(MSt17, AllocSizeReg, Alloc) + end, + {MSt18, CeilBytes, AllocSizeReg, wrap_dyn} end, - {MSt13, MemoryEnsureFreeReg} = MMod:call_primitive( - MSt12, ?PRIM_MEMORY_ENSURE_FREE_WITH_ROOTS, [ + {MSt20, MemoryEnsureFreeReg} = MMod:call_primitive( + MSt19, ?PRIM_MEMORY_ENSURE_FREE_WITH_ROOTS, [ ctx, jit_state, {free, AllocSize}, Live, ?MEMORY_CAN_SHRINK ] ), - MSt14 = handle_error_if({'(bool)', {free, MemoryEnsureFreeReg}, '==', false}, MMod, MSt13), - {MSt17, InitialCreatedBin} = + MSt21 = handle_error_if({'(bool)', {free, MemoryEnsureFreeReg}, '==', false}, MMod, MSt20), + {MSt23, InitialCreatedBin} = case ReuseSourceBinary of false -> % No reuse - create the binary now - {MSt15, CreatedBinResult} = MMod:call_primitive( - MSt14, ?PRIM_TERM_CREATE_EMPTY_BINARY, [ + {MSt22, CreatedBinResult} = MMod:call_primitive( + MSt21, ?PRIM_TERM_CREATE_EMPTY_BINARY, [ ctx, {free, BinaryTotalSizeInBytes} ] ), - MSt16 = MMod:if_block(MSt15, {CreatedBinResult, '==', ?TERM_INVALID_TERM}, fun( - BSt0 - ) -> - MMod:call_primitive_last(BSt0, ?PRIM_RAISE_ERROR, [ - ctx, jit_state, offset, ?OUT_OF_MEMORY_ATOM - ]) - end), - {MSt16, CreatedBinResult}; + {raise_out_of_memory_if_invalid(CreatedBinResult, MMod, MSt22), CreatedBinResult}; true -> % Will reuse - defer creation until first segment - {MSt14, {private_append, BinaryTotalSizeInBytes}} + {MSt21, {private_append, BinaryTotalSizeInBytes}} end, % We redo the decoding. Rest7 should still be equal to previous value. - {Rest7, MSt18, FinalOffset, CreatedBin} = lists:foldl( + {Rest7, MSt24, FinalOffset, CreatedBin} = lists:foldl( fun(_Index, {AccRest0, AccMSt0, AccOffset0, AccCreatedBin}) -> {AtomTypeIndex, AccRest1} = decode_atom(AccRest0), AtomType = AtomResolver(AtomTypeIndex), @@ -2454,18 +2465,31 @@ first_pass( MMod, AccMSt3 ), - AccMSt5 = MMod:free_native_registers(AccMSt4, [Flags, Src, Size]), + AccMSt5 = MMod:free_native_registers(AccMSt4, [Flags, Size]), {AccRest6, AccMSt5, AccOffset1, AccCreatedBin1} end, - {Rest6, MSt17, 0, InitialCreatedBin}, + {Rest6, MSt23, 0, InitialCreatedBin}, lists:seq(1, NBSegments) ), ?TRACE("]\n", []), - MSt19 = MMod:free_native_registers(MSt18, [FinalOffset]), - MSt20 = MMod:move_to_vm_register(MSt19, CreatedBin, Dest), - MSt21 = MMod:free_native_registers(MSt20, [CreatedBin, Dest]), - ?ASSERT_ALL_NATIVE_FREE(MSt21), - first_pass(Rest7, MMod, MSt21, State1); + {MSt25, CreatedBin1} = + case WrapInfo of + no_wrap -> + {MSt24, CreatedBin}; + {wrap_lit, TotalBits} -> + MMod:call_primitive(MSt24, ?PRIM_BS_CREATE_BIN_WRAP, [ + ctx, {free, CreatedBin}, TotalBits + ]); + wrap_dyn -> + MMod:call_primitive(MSt24, ?PRIM_BS_CREATE_BIN_WRAP, [ + ctx, {free, CreatedBin}, FinalOffset + ]) + end, + MSt26 = MMod:free_native_registers(MSt25, [FinalOffset]), + MSt27 = MMod:move_to_vm_register(MSt26, CreatedBin1, Dest), + MSt28 = MMod:free_native_registers(MSt27, [CreatedBin1, Dest]), + ?ASSERT_ALL_NATIVE_FREE(MSt28), + first_pass(Rest7, MMod, MSt28, State1); % 178 first_pass(<>, MMod, MSt0, State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), @@ -2663,14 +2687,8 @@ first_pass_bs_create_bin_compute_size( {ResultReg, '==', 0}, Fail, MMod, MSt2 ), MSt4 = MMod:shift_left(MSt3, ResultReg, 3), - case AccSizeReg0 of - undefined -> - {MSt4, AccLiteralSize0, ResultReg, State0}; - _ -> - MSt5 = MMod:add(MSt4, AccSizeReg0, ResultReg), - MSt6 = MMod:free_native_registers(MSt5, [ResultReg]), - {MSt6, AccLiteralSize0, AccSizeReg0, State0} - end; + {MSt5, AccSizeReg1} = accumulate_segment_size(AccSizeReg0, ResultReg, MMod, MSt4), + {MSt5, AccLiteralSize0, AccSizeReg1, State0}; first_pass_bs_create_bin_compute_size( utf32, Src, _Size, _SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 ) -> @@ -2695,14 +2713,8 @@ first_pass_bs_create_bin_compute_size( MMod, MSt2 ), - case AccSizeReg0 of - undefined -> - {MSt3, AccLiteralSize0, SizeValue, State0}; - _ -> - MSt4 = MMod:add(MSt3, AccSizeReg0, SizeValue), - MSt5 = MMod:free_native_registers(MSt4, [SizeValue]), - {MSt5, AccLiteralSize0, AccSizeReg0, State0} - end + {MSt4, AccSizeReg1} = accumulate_segment_size(AccSizeReg0, SizeValue, MMod, MSt3), + {MSt4, AccLiteralSize0, AccSizeReg1, State0} end; first_pass_bs_create_bin_compute_size( integer, Src, Size, SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 @@ -2729,15 +2741,9 @@ first_pass_bs_create_bin_compute_size( is_integer(SizeValue) -> {MSt5, AccLiteralSize0 + (SizeValue * SegmentUnit), AccSizeReg0, State0}; true -> - MSt6 = MMod:mul(MSt5, SizeValue, SegmentUnit), - case AccSizeReg0 of - undefined -> - {MSt6, AccLiteralSize0, SizeValue, State0}; - _ -> - MSt7 = MMod:add(MSt6, AccSizeReg0, SizeValue), - MSt8 = MMod:free_native_registers(MSt7, [SizeValue]), - {MSt8, AccLiteralSize0, AccSizeReg0, State0} - end + MSt6 = scale_size_by_unit_raising(SizeValue, SegmentUnit, Fail, MMod, MSt5), + {MSt7, AccSizeReg1} = accumulate_segment_size(AccSizeReg0, SizeValue, MMod, MSt6), + {MSt7, AccLiteralSize0, AccSizeReg1, State0} end; first_pass_bs_create_bin_compute_size( string, _Src, Size, SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt1, State0 @@ -2757,59 +2763,47 @@ first_pass_bs_create_bin_compute_size( end, {MSt5, AccLiteralSize0 + (SizeValue * SegmentUnit), AccSizeReg0, State0}; first_pass_bs_create_bin_compute_size( - AtomType, Src, ?ALL_ATOM, _SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 + AtomType, Src, ?ALL_ATOM, SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 ) when AtomType =:= binary orelse AtomType =:= append orelse AtomType =:= private_append -> - MSt1 = verify_is_binary(Src, Fail, MMod, MSt0), - {MSt2, Reg} = MMod:copy_to_native_register(MSt1, Src), - {MSt3, Reg} = MMod:and_(MSt2, {free, Reg}, ?TERM_PRIMARY_CLEAR_MASK), - MSt4 = MMod:move_array_element(MSt3, Reg, 1, Reg), - MSt5 = MMod:shift_left(MSt4, Reg, 3), - case AccSizeReg0 of - undefined -> - {MSt5, AccLiteralSize0, Reg, State0}; - _ -> - MSt6 = MMod:add(MSt5, AccSizeReg0, Reg), - MSt7 = MMod:free_native_registers(MSt6, [Reg]), - {MSt7, AccLiteralSize0, AccSizeReg0, State0} - end; + MSt1 = verify_is_bitstring(Src, Fail, MMod, MSt0), + {MSt2, Reg0} = MMod:copy_to_native_register(MSt1, Src), + {MSt3, Reg1} = term_bit_size({free, Reg0}, MMod, MSt2), + MSt4 = bs_fail_or_badarg_unless_multiple_of_unit(Reg1, SegmentUnit, Fail, MMod, MSt3), + {MSt5, AccSizeReg1} = accumulate_segment_size(AccSizeReg0, Reg1, MMod, MSt4), + {MSt5, AccLiteralSize0, AccSizeReg1, State0}; first_pass_bs_create_bin_compute_size( AtomType, Src, Size, SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 ) when (AtomType =:= binary orelse AtomType =:= append orelse AtomType =:= private_append) andalso is_integer(Size) andalso Size > 0 -> - MSt1 = verify_is_binary(Src, Fail, MMod, MSt0), + MSt1 = verify_is_bitstring(Src, Fail, MMod, MSt0), {MSt2, SizeValue} = term_to_int(Size, 0, MMod, MSt1), - {MSt2, AccLiteralSize0 + (SizeValue * SegmentUnit), AccSizeReg0, State0}; + RequiredBits = SizeValue * SegmentUnit, + {MSt3, SrcBitsReg0} = MMod:copy_to_native_register(MSt2, Src), + {MSt4, SrcBitsReg} = term_bit_size({free, SrcBitsReg0}, MMod, MSt3), + MSt5 = cond_raise_badarg_or_jump_to_fail_label( + {{free, SrcBitsReg}, '<', RequiredBits}, Fail, MMod, MSt4 + ), + {MSt5, AccLiteralSize0 + RequiredBits, AccSizeReg0, State0}; first_pass_bs_create_bin_compute_size( AtomType, Src, Size, SegmentUnit, Fail, AccLiteralSize0, AccSizeReg0, MMod, MSt0, State0 ) when AtomType =:= binary orelse AtomType =:= append orelse AtomType =:= private_append -> - MSt1 = verify_is_binary(Src, Fail, MMod, MSt0), + MSt1 = verify_is_bitstring(Src, Fail, MMod, MSt0), {MSt2, Reg0} = MMod:copy_to_native_register(MSt1, Size), {MSt3, Reg1} = MMod:copy_to_native_register(MSt2, Src), - {MSt4, Reg1} = MMod:and_(MSt3, {free, Reg1}, ?TERM_PRIMARY_CLEAR_MASK), - MSt5 = MMod:move_array_element(MSt4, Reg1, 1, Reg1), - MSt6 = MMod:shift_left(MSt5, Reg1, 3), - MSt7 = MMod:if_block(MSt6, {{free, Reg0}, '!=', ?ALL_ATOM}, fun(BSt0) -> + {MSt4, Reg2} = term_bit_size({free, Reg1}, MMod, MSt3), + MSt5 = MMod:if_block(MSt4, {{free, Reg0}, '!=', ?ALL_ATOM}, fun(BSt0) -> {BSt1, SizeReg} = term_to_int(Size, Fail, MMod, BSt0), - BSt2 = cond_raise_badarg_or_jump_to_fail_label( - {SizeReg, '<', 0}, Fail, MMod, BSt1 - ), - BSt3 = MMod:mul(BSt2, SizeReg, SegmentUnit), + BSt3 = scale_size_by_unit_raising(SizeReg, SegmentUnit, Fail, MMod, BSt1), BSt4 = cond_raise_badarg_or_jump_to_fail_label( - {Reg1, '<', SizeReg}, Fail, MMod, BSt3 + {Reg2, '<', SizeReg}, Fail, MMod, BSt3 ), - BSt5 = MMod:move_to_native_register(BSt4, SizeReg, Reg1), + BSt5 = MMod:move_to_native_register(BSt4, SizeReg, Reg2), MMod:free_native_registers(BSt5, [SizeReg]) end), - case AccSizeReg0 of - undefined -> - {MSt7, AccLiteralSize0, Reg1, State0}; - _ -> - MSt8 = MMod:add(MSt7, AccSizeReg0, Reg1), - MSt9 = MMod:free_native_registers(MSt8, [Reg1]), - {MSt9, AccLiteralSize0, AccSizeReg0, State0} - end. + {MSt6, AccSizeReg1} = accumulate_segment_size(AccSizeReg0, Reg2, MMod, MSt5), + {MSt6, AccLiteralSize0, AccSizeReg1, State0}. first_pass_bs_create_bin_insert_value( utf8, _Flags, Src, _Size, _SegmentUnit, Fail, CreatedBin, Offset, MMod, MSt0 @@ -2855,8 +2849,6 @@ first_pass_bs_create_bin_insert_value( % term_to_int can raise a badarg and use a temp register for this, start % with it. {MSt1, SizeValue} = term_to_int(Size, Fail, MMod, MSt0), - % Because we're calling a function without ctx as an arg, we need to move - % the value now to a register {MSt2, SrcReg} = MMod:move_to_native_register(MSt1, Src), {MSt3, FlagsValue} = decode_flags_list(Flags, MMod, MSt2), MSt4 = MMod:mul(MSt3, SizeValue, SegmentUnit), @@ -2916,22 +2908,16 @@ first_pass_bs_create_bin_insert_value( ) -> % Special case: first segment is private_append with undefined CreatedBin % Get original size before reusing - {MSt1, OriginalSize} = term_binary_size(Src, MMod, MSt0), + {MSt1, OriginalBits} = term_bit_size(Src, MMod, MSt0), % Reuse the source binary (content is already there, no need to copy) {MSt2, CreatedBin} = MMod:call_primitive(MSt1, ?PRIM_TERM_REUSE_BINARY, [ ctx, {free, Src}, {free, BinaryTotalSizeInBytes} ]), - MSt3 = MMod:if_block(MSt2, {CreatedBin, '==', ?TERM_INVALID_TERM}, fun(BSt0) -> - MMod:call_primitive_last(BSt0, ?PRIM_RAISE_ERROR, [ - ctx, jit_state, offset, ?OUT_OF_MEMORY_ATOM - ]) - end), - % Convert original size to bits and update offset - MSt4 = MMod:shift_left(MSt3, OriginalSize, 3), - {MSt5, NewOffset} = first_pass_bs_create_bin_insert_value_increment_offset( - MMod, MSt4, Offset, OriginalSize, 1 + MSt3 = raise_out_of_memory_if_invalid(CreatedBin, MMod, MSt2), + {MSt4, NewOffset} = first_pass_bs_create_bin_insert_value_increment_offset( + MMod, MSt3, Offset, OriginalBits, 1 ), - {MSt5, NewOffset, CreatedBin}; + {MSt4, NewOffset, CreatedBin}; first_pass_bs_create_bin_insert_value( AtomType, _Flags, Src, Size, SegmentUnit, Fail, CreatedBin, Offset, MMod, MSt0 ) when AtomType =:= binary orelse AtomType =:= append orelse AtomType =:= private_append -> @@ -2953,22 +2939,24 @@ first_pass_bs_create_bin_insert_value( {MSt3, SizeReg} = term_from_int(SizeReg, MMod, MSt2), {MSt3, {free, SizeReg}} end, - {MSt5, SizeValue} = MMod:call_primitive(MSt4, ?PRIM_BITSTRING_COPY_BINARY, [ - ctx, jit_state, CreatedBin, Offset, Src, SizeInBits + {MSt5, SrcReg} = MMod:move_to_native_register(MSt4, Src), + {MSt6, SizeValue} = MMod:call_primitive(MSt5, ?PRIM_BITSTRING_COPY_BINARY, [ + CreatedBin, Offset, {free, SrcReg}, SizeInBits ]), - MSt6 = MMod:if_block(MSt5, {SizeValue, '<', 0}, fun(BlockSt) -> + MSt7 = MMod:if_block(MSt6, {SizeValue, '<', 0}, fun(BlockSt) -> MMod:call_primitive_last(BlockSt, ?PRIM_HANDLE_ERROR, [ ctx, jit_state, offset ]) end), - {MSt7, NewOffset} = first_pass_bs_create_bin_insert_value_increment_offset( - MMod, MSt6, Offset, SizeValue, 1 + {MSt8, NewOffset} = first_pass_bs_create_bin_insert_value_increment_offset( + MMod, MSt7, Offset, SizeValue, 1 ), - {MSt7, NewOffset, CreatedBin}; + {MSt8, NewOffset, CreatedBin}; first_pass_bs_create_bin_insert_value( - _OtherType, _Flag, _Src, _Size, _SegmentUnit, _Fail, CreatedBin, Offset, _MMod, MSt0 + _OtherType, _Flag, Src, _Size, _SegmentUnit, _Fail, CreatedBin, Offset, MMod, MSt0 ) -> - {MSt0, Offset, CreatedBin}. + MSt1 = MMod:free_native_registers(MSt0, [Src]), + {MSt1, Offset, CreatedBin}. first_pass_bs_create_bin_insert_value_increment_offset(_MMod, MSt0, Offset, Size, Unit) when is_integer(Offset) andalso is_integer(Size) andalso is_integer(Unit) @@ -3073,22 +3061,22 @@ first_pass_bs_match_ensure_at_least( true -> {Unit, Rest2} = decode_literal(Rest1), ?TRACE("{ensure_at_least,~p,~p},", [Stride, Unit]), - {MSt1, Reg} = MMod:get_array_element(MSt0, BSBinaryReg, 1), - MSt2 = MMod:shift_left(MSt1, Reg, 3), - % Reg is bs_bin_size * 8 - MSt3 = MMod:sub(MSt2, Reg, BSOffsetReg), + {MSt1, Reg} = term_bit_size({ptr, BSBinaryReg}, MMod, MSt0), + % Reg is the source bit size + MSt2 = MMod:sub(MSt1, Reg, BSOffsetReg), % Reg is (bs_bin_size * 8) - bs_offset = remaining bits - MSt4 = cond_jump_to_label({Reg, '<', Stride}, Fail, MMod, MSt3), + MSt3 = cond_jump_to_label({Reg, '<', Stride}, Fail, MMod, MSt2), % Also check unit alignment: (remaining - stride) % unit == 0 MSt7 = if Unit > 1 -> - MSt4b = MMod:sub(MSt4, Reg, Stride), - {MSt5, UnitReg} = MMod:and_(MSt4b, {free, Reg}, Unit - 1), - MSt6 = cond_jump_to_label({{free, UnitReg}, '!=', 0}, Fail, MMod, MSt5), - MSt6; + MSt4 = MMod:sub(MSt3, Reg, Stride), + MSt5 = bs_fail_or_badarg_unless_multiple_of_unit( + Reg, Unit, Fail, MMod, MSt4 + ), + MMod:free_native_registers(MSt5, [Reg]); true -> - MMod:free_native_registers(MSt4, [Reg]) + MMod:free_native_registers(MSt3, [Reg]) end, {J0 - 2, Rest2, MatchState, BSOffsetReg, MSt7} end. @@ -3105,14 +3093,13 @@ first_pass_bs_match_ensure_exactly( {J0, Rest0, MatchState, BSOffsetReg, MSt1}; true -> ?TRACE("{ensure_exactly,~p},", [Stride]), - {MSt1, Reg} = MMod:get_array_element(MSt0, BSBinaryReg, 1), - MSt2 = MMod:shift_left(MSt1, Reg, 3), - % Reg is bs_bin_size * 8 (use unit instead ??) - MSt3 = MMod:sub(MSt2, Reg, BSOffsetReg), + {MSt1, Reg} = term_bit_size({ptr, BSBinaryReg}, MMod, MSt0), + % Reg is the source bit size + MSt2 = MMod:sub(MSt1, Reg, BSOffsetReg), % Reg is (bs_bin_size * 8) - bs_offset - MSt4 = cond_jump_to_label({Reg, '!=', Stride}, Fail, MMod, MSt3), - MSt5 = MMod:free_native_registers(MSt4, [Reg]), - {J0 - 1, Rest1, MatchState, BSOffsetReg, MSt5} + MSt3 = cond_jump_to_label({Reg, '!=', Stride}, Fail, MMod, MSt2), + MSt4 = MMod:free_native_registers(MSt3, [Reg]), + {J0 - 1, Rest1, MatchState, BSOffsetReg, MSt4} end. first_pass_bs_match_integer( @@ -3121,46 +3108,38 @@ first_pass_bs_match_integer( {_Live, Rest1} = decode_literal(Rest0), {Flags, Rest2} = decode_compile_time_literal(Rest1, State0), {MSt1, FlagsValue} = decode_flags_list(Flags, MMod, MSt0), - {MSt2, Size, Rest3} = decode_typed_compact_term(Rest2, MMod, MSt0, State0), + {Size, Rest3} = decode_literal(Rest2), {Unit, Rest4} = decode_literal(Rest3), ?TRACE("{integer,~p,~p,~p, ", [Flags, Size, Unit]), - {MSt3, SizeReg} = term_to_int(Size, 0, MMod, MSt1), - {MSt6, NumBits} = - if - is_integer(SizeReg) -> - {MSt2, SizeReg * Unit}; - true -> - MSt3 = MMod:mul(SizeReg, Unit), - {MSt3, SizeReg} - end, - {MSt7, Result} = MMod:call_primitive(MSt6, ?PRIM_BITSTRING_EXTRACT_INTEGER, [ + NumBits = Size * Unit, + {MSt2, Result} = MMod:call_primitive(MSt1, ?PRIM_BITSTRING_EXTRACT_INTEGER, [ ctx, jit_state, BSBinaryReg, BSOffsetReg, NumBits, {free, FlagsValue} ]), - MSt8 = handle_error_if({Result, '==', 0}, MMod, MSt7), - MSt9 = cond_jump_to_label({Result, '==', ?FALSE_ATOM}, Fail, MMod, MSt8), - MSt10 = - case MMod:available_regs(MSt9) of + MSt3 = handle_error_if({Result, '==', 0}, MMod, MSt2), + MSt4 = cond_jump_to_label({Result, '==', ?FALSE_ATOM}, Fail, MMod, MSt3), + MSt5 = + case MMod:available_regs(MSt4) of [] -> - MMod:free_native_registers(MSt9, [BSOffsetReg]); + MMod:free_native_registers(MSt4, [BSOffsetReg]); _ -> - MSt9 + MSt4 end, - {MSt11, Dest, Rest5} = decode_dest(Rest4, MMod, MSt10), + {MSt6, Dest, Rest5} = decode_dest(Rest4, MMod, MSt5), ?TRACE("~p},", [Dest]), - MSt12 = MMod:move_to_vm_register(MSt11, Result, Dest), - MSt13 = MMod:free_native_registers(MSt12, [Result, Dest]), - case MMod:available_regs(MSt9) of + MSt7 = MMod:move_to_vm_register(MSt6, Result, Dest), + MSt8 = MMod:free_native_registers(MSt7, [Result, Dest]), + case MMod:available_regs(MSt4) of [] -> - {MSt14, MatchState} = MMod:and_(MSt13, {free, MatchState}, ?TERM_PRIMARY_CLEAR_MASK), - {MSt15, NewBSOffsetReg} = MMod:get_array_element(MSt14, MatchState, 2), - MSt16 = MMod:or_(MSt15, MatchState, ?TERM_PRIMARY_BOXED), - MSt17 = MMod:add(MSt16, NewBSOffsetReg, NumBits), - MSt18 = MMod:free_native_registers(MSt17, [NumBits]), - {J0 - 5, Rest5, MatchState, NewBSOffsetReg, MSt18}; + {MSt9, MatchState} = MMod:and_(MSt8, {free, MatchState}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt10, NewBSOffsetReg} = MMod:get_array_element(MSt9, MatchState, 2), + MSt11 = MMod:or_(MSt10, MatchState, ?TERM_PRIMARY_BOXED), + MSt12 = MMod:add(MSt11, NewBSOffsetReg, NumBits), + MSt13 = MMod:free_native_registers(MSt12, [NumBits]), + {J0 - 5, Rest5, MatchState, NewBSOffsetReg, MSt13}; _ -> - MSt14 = MMod:add(MSt13, BSOffsetReg, NumBits), - MSt15 = MMod:free_native_registers(MSt14, [NumBits]), - {J0 - 5, Rest5, MatchState, BSOffsetReg, MSt15} + MSt9 = MMod:add(MSt8, BSOffsetReg, NumBits), + MSt10 = MMod:free_native_registers(MSt9, [NumBits]), + {J0 - 5, Rest5, MatchState, BSOffsetReg, MSt10} end. first_pass_bs_match_binary( @@ -3181,42 +3160,31 @@ first_pass_bs_match_binary( {Unit, Rest4} = decode_literal(Rest3), ?TRACE("{binary,~p,~p,~p,~p", [Live, _Flags, Size, Unit]), MatchedBits = Size * Unit, - MSt1 = - if - MatchedBits rem 8 =:= 0 -> - cond_raise_badarg({BSOffsetReg, '&', 2#111, '!=', 0}, MMod, MSt0); - true -> - MMod:call_primitive_last(MSt0, ?PRIM_RAISE_ERROR, [ - ctx, jit_state, offset, ?BADARG_ATOM - ]) - end, - MatchedBytes = MatchedBits div 8, - {MSt2, BSOffseBytesReg} = MMod:shift_right(MSt1, BSOffsetReg, 3), - {MSt3, RemainingBytesReg} = MMod:get_array_element(MSt2, BSBinaryReg, 1), - MSt4 = MMod:sub(MSt3, RemainingBytesReg, BSOffseBytesReg), - MSt5 = cond_jump_to_label({RemainingBytesReg, '<', MatchedBytes}, Fail, MMod, MSt4), - MSt6 = MMod:free_native_registers(MSt5, [RemainingBytesReg]), - {MSt7, HeapSizeReg} = MMod:call_primitive(MSt6, ?PRIM_TERM_SUB_BINARY_HEAP_SIZE, [ - BSBinaryReg, MatchedBytes + {MSt1, RemainingBitsReg} = term_bit_size({ptr, BSBinaryReg}, MMod, MSt0), + MSt2 = MMod:sub(MSt1, RemainingBitsReg, BSOffsetReg), + MSt3 = cond_jump_to_label({{free, RemainingBitsReg}, '<', MatchedBits}, Fail, MMod, MSt2), + {MSt4, HeapSizeReg} = MMod:call_primitive(MSt3, ?PRIM_BITSTRING_SLICE_HEAP_SIZE, [ + BSBinaryReg, BSOffsetReg, MatchedBits ]), - {MSt8, NewMatchState} = memory_ensure_free_with_extra_root( - MatchState, Live, {free, HeapSizeReg}, MMod, MSt7 + {MSt5, NewMatchState} = memory_ensure_free_with_extra_root( + MatchState, Live, {free, HeapSizeReg}, MMod, MSt4 ), % Restore BSBinaryReg as it may have been gc'd as well - {MSt9, MatchStateReg0} = MMod:copy_to_native_register(MSt8, NewMatchState), - {MSt10, MatchStateReg0} = MMod:and_(MSt9, {free, MatchStateReg0}, ?TERM_PRIMARY_CLEAR_MASK), - MSt11 = MMod:move_array_element(MSt10, MatchStateReg0, 1, BSBinaryReg), - MSt12 = MMod:free_native_registers(MSt11, [MatchStateReg0]), - {MSt13, ResultTerm} = MMod:call_primitive(MSt12, ?PRIM_TERM_MAYBE_CREATE_SUB_BINARY, [ - ctx, BSBinaryReg, {free, BSOffseBytesReg}, MatchedBytes + {MSt6, MatchStateReg0} = MMod:copy_to_native_register(MSt5, NewMatchState), + {MSt7, MatchStateReg0} = MMod:and_(MSt6, {free, MatchStateReg0}, ?TERM_PRIMARY_CLEAR_MASK), + MSt8 = MMod:move_array_element(MSt7, MatchStateReg0, 1, BSBinaryReg), + MSt9 = MMod:free_native_registers(MSt8, [MatchStateReg0]), + {MSt10, ResultTerm} = MMod:call_primitive(MSt9, ?PRIM_BITSTRING_SLICE, [ + ctx, BSBinaryReg, BSOffsetReg, MatchedBits ]), - {MSt14, BSBinaryReg} = MMod:and_(MSt13, {free, BSBinaryReg}, ?TERM_PRIMARY_CLEAR_MASK), - {MSt15, Dest, Rest5} = decode_dest(Rest4, MMod, MSt14), + MSt11 = raise_out_of_memory_if_invalid(ResultTerm, MMod, MSt10), + {MSt12, BSBinaryReg} = MMod:and_(MSt11, {free, BSBinaryReg}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt13, Dest, Rest5} = decode_dest(Rest4, MMod, MSt12), ?TRACE("~p},", [Dest]), - MSt16 = MMod:move_to_vm_register(MSt15, ResultTerm, Dest), - MSt17 = MMod:free_native_registers(MSt16, [ResultTerm]), - MSt18 = MMod:add(MSt17, BSOffsetReg, MatchedBits), - {J0 - 5, Rest5, NewMatchState, BSOffsetReg, MSt18}. + MSt14 = MMod:move_to_vm_register(MSt13, ResultTerm, Dest), + MSt15 = MMod:free_native_registers(MSt14, [ResultTerm]), + MSt16 = MMod:add(MSt15, BSOffsetReg, MatchedBits), + {J0 - 5, Rest5, NewMatchState, BSOffsetReg, MSt16}. first_pass_bs_match_get_tail(MatchState, BSBinaryReg, BSOffsetReg, J0, Rest0, MMod, MSt0) -> {Live, Rest1} = decode_literal(Rest0), @@ -3235,29 +3203,22 @@ first_pass_bs_match_get_tail(MatchState, BSBinaryReg, BSOffsetReg, J0, Rest0, MM do_get_tail( MatchState, Live, BSOffsetReg, BSBinaryReg, MMod, MSt0 ) -> - MSt1 = cond_raise_badarg({BSOffsetReg, '&', 2#111, '!=', 0}, MMod, MSt0), - {MSt2, BSOffseBytesReg} = MMod:shift_right(MSt1, BSOffsetReg, 3), - {MSt3, TailBytesReg0} = MMod:get_array_element(MSt2, BSBinaryReg, 1), - MSt4 = MMod:sub(MSt3, TailBytesReg0, BSOffseBytesReg), - {MSt5, HeapSizeReg} = MMod:call_primitive(MSt4, ?PRIM_TERM_SUB_BINARY_HEAP_SIZE, [ - BSBinaryReg, {free, TailBytesReg0} + {MSt1, HeapSizeReg} = MMod:call_primitive(MSt0, ?PRIM_BITSTRING_GET_TAIL_HEAP_SIZE, [ + BSBinaryReg, BSOffsetReg ]), - {MSt6, NewMatchState} = memory_ensure_free_with_extra_root( - MatchState, Live, {free, HeapSizeReg}, MMod, MSt5 + {MSt2, NewMatchState} = memory_ensure_free_with_extra_root( + MatchState, Live, {free, HeapSizeReg}, MMod, MSt1 ), % Restore BSBinaryReg as it may have been gc'd as well - {MSt7, MatchStateReg0} = MMod:copy_to_native_register(MSt6, NewMatchState), - {MSt8, MatchStateReg0} = MMod:and_(MSt7, {free, MatchStateReg0}, ?TERM_PRIMARY_CLEAR_MASK), - MSt9 = MMod:move_array_element(MSt8, MatchStateReg0, 1, BSBinaryReg), - MSt10 = MMod:free_native_registers(MSt9, [MatchStateReg0]), - {MSt11, BSBinaryReg} = MMod:and_(MSt10, {free, BSBinaryReg}, ?TERM_PRIMARY_CLEAR_MASK), - {MSt12, TailBytesReg1} = MMod:get_array_element(MSt11, BSBinaryReg, 1), - MSt13 = MMod:sub(MSt12, TailBytesReg1, BSOffseBytesReg), - MSt14 = MMod:add(MSt13, BSBinaryReg, ?TERM_PRIMARY_BOXED), - {MSt15, ResultTerm} = MMod:call_primitive(MSt14, ?PRIM_TERM_MAYBE_CREATE_SUB_BINARY, [ - ctx, BSBinaryReg, {free, BSOffseBytesReg}, {free, TailBytesReg1} + {MSt3, MatchStateReg0} = MMod:copy_to_native_register(MSt2, NewMatchState), + {MSt4, MatchStateReg0} = MMod:and_(MSt3, {free, MatchStateReg0}, ?TERM_PRIMARY_CLEAR_MASK), + MSt5 = MMod:move_array_element(MSt4, MatchStateReg0, 1, BSBinaryReg), + MSt6 = MMod:free_native_registers(MSt5, [MatchStateReg0]), + {MSt7, ResultTerm} = MMod:call_primitive(MSt6, ?PRIM_BITSTRING_CREATE_TAIL, [ + ctx, BSBinaryReg, BSOffsetReg ]), - {MSt15, ResultTerm, NewMatchState}. + MSt8 = raise_out_of_memory_if_invalid(ResultTerm, MMod, MSt7), + {MSt8, ResultTerm, NewMatchState}. first_pass_bs_match_equal_colon_equal( Fail, MatchState, BSBinaryReg, BSOffsetReg, J0, Rest0, MMod, MSt0 @@ -4462,7 +4423,7 @@ verify_is_number(Arg1, Fail, MMod, MSt0) -> %% @param MSt0 backend state %% @return new backend state %%----------------------------------------------------------------------------- -verify_is_binary(Arg1, FailLabel, MMod, MSt0) -> +verify_is_bitstring(Arg1, FailLabel, MMod, MSt0) -> {MSt1, Reg} = MMod:copy_to_native_register(MSt0, Arg1), MSt2 = verify_is_boxed(MMod, MSt1, Reg, FailLabel), {MSt3, Reg} = MMod:and_(MSt2, {free, Reg}, ?TERM_PRIMARY_CLEAR_MASK), @@ -4508,6 +4469,23 @@ scale_size_by_unit(SizeReg, Unit, Fail, MMod, MSt0) -> end, MMod:mul(MSt2, SizeReg, Unit). +% As scale_size_by_unit, but raising for negative or overflow. +scale_size_by_unit_raising(SizeReg, Unit, Fail, MMod, MSt0) -> + MSt1 = cond_raise_badarg_or_jump_to_fail_label({SizeReg, '<', 0}, Fail, MMod, MSt0), + {_MinSmall, MaxSmall} = small_integer_bounds(MMod), + MaxSize = (1 bsl (MMod:word_size() * 8 - 2)) div max(Unit, 1), + MSt2 = + if + MaxSize >= MaxSmall -> + MSt1; + true -> + {MSt1a, MaxReg} = MMod:move_to_native_register(MSt1, MaxSize), + cond_raise_badarg_or_jump_to_fail_label( + {{free, MaxReg}, '<', SizeReg}, Fail, MMod, MSt1a + ) + end, + MMod:mul(MSt2, SizeReg, Unit). + term_to_int(Term, _FailLabel, _MMod, MSt0) when is_integer(Term) -> {MSt0, Term bsr 4}; term_to_int({literal, Val}, _FailLabel, _MMod, MSt0) when is_integer(Val) -> @@ -4998,6 +4976,45 @@ cond_jump_to_label(Cond, Label, MMod, MSt0) -> MMod:jump_to_label(BSt0, Label) end). +%% Largest binary size in bytes, matching TERM_MAX_BINARY_SIZE in term.h: a +%% sub-binary packs its byte offset shifted left by 3, so a larger offset would +%% be truncated. +max_binary_size_in_bytes(MMod) -> + (1 bsl (MMod:word_size() * 8 - 3)) - 1. + +%% Raise system_limit unconditionally, for a size known at compile time to be +%% unrepresentable. +raise_system_limit(MMod, MSt0) -> + MMod:call_primitive_last(MSt0, ?PRIM_RAISE_ERROR, [ + ctx, jit_state, offset, ?SYSTEM_LIMIT_ATOM + ]). + +%% Accumulate a segment bit size held in a register into the running total, +%% rejecting a total that no longer fits in a machine word. +%% +%% An unsigned addition wraps exactly when the result is below either operand, +%% which is the same test the interpreter performs as +%% `binary_size > SIZE_MAX - segment_bits`. Without it the total would silently +%% wrap, the binary would be allocated too small, and the segments would be +%% written past its end. +accumulate_segment_size(undefined, SizeReg, _MMod, MSt0) -> + {MSt0, SizeReg}; +accumulate_segment_size(AccSizeReg, SizeReg, MMod, MSt0) -> + MSt1 = MMod:add(MSt0, AccSizeReg, SizeReg), + MSt2 = MMod:if_block(MSt1, {'(unsigned)', AccSizeReg, '<', SizeReg}, fun(BSt0) -> + raise_system_limit(MMod, BSt0) + end), + {MMod:free_native_registers(MSt2, [SizeReg]), AccSizeReg}. + +%% Reserving heap words does not reserve the buffer of a refc binary, which is +%% allocated separately and may still fail. +raise_out_of_memory_if_invalid(Reg, MMod, MSt0) -> + MMod:if_block(MSt0, {Reg, '==', ?TERM_INVALID_TERM}, fun(BSt0) -> + MMod:call_primitive_last(BSt0, ?PRIM_RAISE_ERROR, [ + ctx, jit_state, offset, ?OUT_OF_MEMORY_ATOM + ]) + end). + term_binary_heap_size(Size, MMod) when is_integer(Size) -> case MMod:word_size() of 4 when Size < ?REFC_BINARY_MIN_32 -> @@ -5047,16 +5064,6 @@ term_binary_heap_size({free, Reg}, MMod, MSt0) -> end, {MSt1, Reg}. -term_binary_size({free, BinReg}, MMod, MSt0) -> - {MSt1, BinReg} = MMod:and_(MSt0, {free, BinReg}, ?TERM_PRIMARY_CLEAR_MASK), - MSt2 = MMod:move_array_element(MSt1, BinReg, 1, BinReg), - {MSt2, BinReg}; -term_binary_size(Src, MMod, MSt0) -> - {MSt1, SrcReg} = MMod:move_to_native_register(MSt0, Src), - {MSt2, SrcReg} = MMod:and_(MSt1, {free, SrcReg}, ?TERM_PRIMARY_CLEAR_MASK), - MSt3 = MMod:move_array_element(MSt2, SrcReg, 1, SrcReg), - {MSt3, SrcReg}. - term_set_map_assoc(MapPtrReg, {free, PosReg}, {free, Key}, {free, Value}, MMod, MSt0) -> {MSt1, MapKeysReg} = MMod:get_array_element(MSt0, MapPtrReg, 1), MSt2 = term_put_tuple_element({free, MapKeysReg}, PosReg, {free, Key}, MMod, MSt1), @@ -5068,6 +5075,47 @@ term_put_tuple_element({free, TupleReg}, PosReg, {free, Value}, MMod, MSt0) -> MSt2 = MMod:move_to_array_element(MSt1, Value, TupleReg, PosReg, 1), MMod:free_native_registers(MSt2, [TupleReg, Value]). +%% Total bit size = binary_size * 8 + sub-binary trailing bit count. +%% {free, BinReg} takes a tagged boxed term and consumes it; {ptr, PtrReg} +%% takes an already-tag-cleared pointer and leaves it allocated for the caller; +%% any other source is copied first and left untouched. +term_bit_size({free, BinReg}, MMod, MSt0) -> + {MSt1, PtrReg} = MMod:and_(MSt0, {free, BinReg}, ?TERM_PRIMARY_CLEAR_MASK), + {MSt2, BitsReg} = term_bit_size({ptr, PtrReg}, MMod, MSt1), + MSt3 = MMod:free_native_registers(MSt2, [PtrReg]), + {MSt3, BitsReg}; +term_bit_size({ptr, PtrReg}, MMod, MSt0) -> + {MSt1, BitsReg} = MMod:get_array_element(MSt0, PtrReg, 1), + MSt2 = MMod:shift_left(MSt1, BitsReg, 3), + {MSt3, TagReg} = MMod:get_array_element(MSt2, PtrReg, 0), + {MSt4, TagReg} = MMod:and_(MSt3, {free, TagReg}, ?TERM_BOXED_TAG_MASK), + MSt5 = MMod:if_block(MSt4, {{free, TagReg}, '==', ?TERM_BOXED_SUB_BINARY}, fun(BSt0) -> + {BSt1, TrailReg} = MMod:get_array_element(BSt0, PtrReg, 2), + {BSt2, TrailReg} = MMod:and_(BSt1, {free, TrailReg}, 16#7), + BSt3 = MMod:add(BSt2, BitsReg, TrailReg), + MMod:free_native_registers(BSt3, [TrailReg]) + end), + {MSt5, BitsReg}; +term_bit_size(Src, MMod, MSt0) -> + {MSt1, SrcReg} = MMod:copy_to_native_register(MSt0, Src), + term_bit_size({free, SrcReg}, MMod, MSt1). + +bs_fail_or_badarg_unless_multiple_of_unit(_BitsReg, 1, _Fail, _MMod, MSt0) -> + MSt0; +bs_fail_or_badarg_unless_multiple_of_unit(BitsReg, Unit, Fail, MMod, MSt0) when + Unit band (Unit - 1) =:= 0 +-> + cond_raise_badarg_or_jump_to_fail_label( + {BitsReg, '&', Unit - 1, '!=', 0}, Fail, MMod, MSt0 + ); +bs_fail_or_badarg_unless_multiple_of_unit(BitsReg, Unit, Fail, MMod, MSt0) -> + {MSt1, ResultReg} = MMod:call_primitive(MSt0, ?PRIM_BITSTRING_IS_MULTIPLE_OF, [ + BitsReg, Unit + ]), + cond_raise_badarg_or_jump_to_fail_label( + {'(bool)', {free, ResultReg}, '==', false}, Fail, MMod, MSt1 + ). + %% @doc Get the stream module %% @return The stream module for jit on this platform -spec stream_module() -> module(). diff --git a/libs/jit/src/jit_aarch64.erl b/libs/jit/src/jit_aarch64.erl index b8f96d746b..a92675e864 100644 --- a/libs/jit/src/jit_aarch64.erl +++ b/libs/jit/src/jit_aarch64.erl @@ -181,6 +181,7 @@ | {'(int)', maybe_free_aarch64_register(), '!=', aarch64_register() | integer()} | {'(bool)', maybe_free_aarch64_register(), '==', false} | {'(bool)', maybe_free_aarch64_register(), '!=', false} + | {'(unsigned)', maybe_free_aarch64_register(), '<', aarch64_register()} | {maybe_free_aarch64_register(), '&', non_neg_integer(), '!=', integer()} | {{free, aarch64_register()}, '==', {free, aarch64_register()}}. @@ -843,6 +844,26 @@ if_block_cond( State1 = if_block_free_reg(RegOrTuple, State0), State2 = State1#state{stream = Stream1}, {State2, ge, byte_size(I1)}; +if_block_cond( + #state{stream_module = StreamModule, stream = Stream0} = State0, + {'(unsigned)', RegOrTuple, '<', RegB} +) when is_atom(RegB) -> + Reg = + case RegOrTuple of + {free, Reg0} -> Reg0; + RegOrTuple -> RegOrTuple + end, + I1 = jit_aarch64_asm:cmp(Reg, RegB), + % cs (aka hs) = carry set = greater than or equal, unsigned + I2 = jit_aarch64_asm:bcc(cs, 0), + Code = << + I1/binary, + I2/binary + >>, + Stream1 = StreamModule:append(Stream0, Code), + State1 = if_block_free_reg(RegOrTuple, State0), + State2 = State1#state{stream = Stream1}, + {State2, cs, byte_size(I1)}; if_block_cond( #state{stream_module = StreamModule, stream = Stream0} = State0, {RegOrTuple, '<', RegB} diff --git a/libs/jit/src/jit_arm32.erl b/libs/jit/src/jit_arm32.erl index 324df7ea56..3258064ff3 100644 --- a/libs/jit/src/jit_arm32.erl +++ b/libs/jit/src/jit_arm32.erl @@ -170,6 +170,7 @@ | {'(int)', maybe_free_arm32_register(), '!=', arm32_register() | integer()} | {'(bool)', maybe_free_arm32_register(), '==', false} | {'(bool)', maybe_free_arm32_register(), '!=', false} + | {'(unsigned)', maybe_free_arm32_register(), '<', arm32_register()} | {maybe_free_arm32_register(), '&', non_neg_integer(), '!=', integer()} | {{free, arm32_register()}, '==', {free, arm32_register()}}. @@ -1054,6 +1055,23 @@ if_block_cond( Regs2 = jit_regs:invalidate_reg(State2#state.regs, Temp), State3 = State2#state{stream = Stream1, regs = Regs2}, {State3, CC, byte_size(I1)}; +if_block_cond( + #state{stream_module = StreamModule, stream = Stream0} = State0, + {'(unsigned)', RegOrTuple, '<', RegB} +) when is_atom(RegB) -> + Reg = + case RegOrTuple of + {free, Reg0} -> Reg0; + RegOrTuple -> RegOrTuple + end, + I1 = jit_arm32_asm:cmp(al, Reg, RegB), + % cs (aka hs) = greater than or equal, unsigned + CC = cs, + ?ASSERT(byte_size(jit_arm32_asm:b(CC, 0)) =:= 4), + Stream1 = StreamModule:append(Stream0, <>), + State1 = if_block_free_reg(RegOrTuple, State0), + State2 = State1#state{stream = Stream1}, + {State2, CC, byte_size(I1)}; if_block_cond( #state{stream_module = StreamModule, stream = Stream0} = State0, {RegOrTuple, '<', RegB} diff --git a/libs/jit/src/jit_armv6m.erl b/libs/jit/src/jit_armv6m.erl index 5b74cc9409..7f28423c9c 100644 --- a/libs/jit/src/jit_armv6m.erl +++ b/libs/jit/src/jit_armv6m.erl @@ -190,6 +190,7 @@ | {'(int)', maybe_free_armv6m_register(), '!=', armv6m_register() | integer()} | {'(bool)', maybe_free_armv6m_register(), '==', false} | {'(bool)', maybe_free_armv6m_register(), '!=', false} + | {'(unsigned)', maybe_free_armv6m_register(), '<', armv6m_register()} | {maybe_free_armv6m_register(), '&', non_neg_integer(), '!=', integer()} | {{free, armv6m_register()}, '==', {free, armv6m_register()}}. @@ -1289,6 +1290,23 @@ if_block_cond( Regs2 = jit_regs:set_contents(State2#state.regs, Temp, {imm, Val}), State3 = State2#state{stream = Stream1, regs = Regs2}, {State3, CC, byte_size(I1)}; +if_block_cond( + #state{stream_module = StreamModule, stream = Stream0} = State0, + {'(unsigned)', RegOrTuple, '<', RegB} +) when is_atom(RegB) -> + Reg = + case RegOrTuple of + {free, Reg0} -> Reg0; + RegOrTuple -> RegOrTuple + end, + I1 = jit_armv6m_asm:cmp(Reg, RegB), + % cs (aka hs) = greater than or equal, unsigned + CC = cs, + ?ASSERT(byte_size(jit_armv6m_asm:bcc(CC, 0)) =:= 2), + Stream1 = StreamModule:append(Stream0, <>), + State1 = if_block_free_reg(RegOrTuple, State0), + State2 = State1#state{stream = Stream1}, + {State2, CC, byte_size(I1)}; if_block_cond( #state{stream_module = StreamModule, stream = Stream0} = State0, {RegOrTuple, '<', RegB} diff --git a/libs/jit/src/jit_riscv32.erl b/libs/jit/src/jit_riscv32.erl index d301d7022b..b1f18b0521 100644 --- a/libs/jit/src/jit_riscv32.erl +++ b/libs/jit/src/jit_riscv32.erl @@ -199,6 +199,7 @@ | {'(int)', maybe_free_riscv32_register(), '!=', riscv32_register() | integer()} | {'(bool)', maybe_free_riscv32_register(), '==', false} | {'(bool)', maybe_free_riscv32_register(), '!=', false} + | {'(unsigned)', maybe_free_riscv32_register(), '<', riscv32_register()} | {maybe_free_riscv32_register(), '&', non_neg_integer(), '!=', integer()} | {{free, riscv32_register()}, '==', {free, riscv32_register()}}. diff --git a/libs/jit/src/jit_riscv64.erl b/libs/jit/src/jit_riscv64.erl index 4ed6c8eb0d..5d31a6f2d4 100644 --- a/libs/jit/src/jit_riscv64.erl +++ b/libs/jit/src/jit_riscv64.erl @@ -206,6 +206,7 @@ | {'(int)', maybe_free_riscv64_register(), '!=', riscv64_register() | integer()} | {'(bool)', maybe_free_riscv64_register(), '==', false} | {'(bool)', maybe_free_riscv64_register(), '!=', false} + | {'(unsigned)', maybe_free_riscv64_register(), '<', riscv64_register()} | {maybe_free_riscv64_register(), '&', non_neg_integer(), '!=', integer()} | {{free, riscv64_register()}, '==', {free, riscv64_register()}}. diff --git a/libs/jit/src/jit_riscv_impl.hrl b/libs/jit/src/jit_riscv_impl.hrl index 132127d498..f52c6ce137 100644 --- a/libs/jit/src/jit_riscv_impl.hrl +++ b/libs/jit/src/jit_riscv_impl.hrl @@ -765,6 +765,21 @@ if_block_cond( Regs2 = jit_regs:set_contents(State2#state.regs, Temp, {imm, Val}), State3 = State2#state{stream = Stream2, regs = Regs2}, {State3, {bge, Temp, Reg}, BranchDelta}; +if_block_cond( + #state{stream_module = StreamModule, stream = Stream0} = State0, + {'(unsigned)', RegOrTuple, '<', RegB} +) when is_atom(RegB) -> + Reg = + case RegOrTuple of + {free, Reg0} -> Reg0; + RegOrTuple -> RegOrTuple + end, + % RISC-V: bgeu Reg, RegB, offset (branch if Reg >= RegB unsigned, i.e., NOT below) + BranchInstr = <<16#FFFFFFFF:32/little>>, + Stream1 = StreamModule:append(Stream0, BranchInstr), + State1 = if_block_free_reg(RegOrTuple, State0), + State2 = State1#state{stream = Stream1}, + {State2, {bgeu, Reg, RegB}, 0}; if_block_cond( #state{stream_module = StreamModule, stream = Stream0} = State0, {RegOrTuple, '<', RegB} diff --git a/libs/jit/src/jit_wasm32.erl b/libs/jit/src/jit_wasm32.erl index 3e9d4f729f..587cced2f3 100644 --- a/libs/jit/src/jit_wasm32.erl +++ b/libs/jit/src/jit_wasm32.erl @@ -193,6 +193,7 @@ | {'(int)', maybe_free_local(), '!=', wasm_local() | integer()} | {'(bool)', maybe_free_local(), '==', false} | {'(bool)', maybe_free_local(), '!=', false} + | {'(unsigned)', maybe_free_local(), '<', wasm_local()} | {maybe_free_local(), '&', non_neg_integer(), '!=', integer()} | {{free, wasm_local()}, '==', {free, wasm_local()}}. @@ -1605,6 +1606,15 @@ emit_condition(State0, {Local, '<', Val}) when is_integer(Val) -> >>, State1 = maybe_free(State0, Local), emit(State1, Code); +emit_condition(State0, {'(unsigned)', Local, '<', OtherLocal}) when is_atom(OtherLocal) -> + L = unwrap_free(Local), + Code = << + (emit_unwrapped_to_stack(L))/binary, + (jit_wasm32_asm:local_get(OtherLocal))/binary, + (jit_wasm32_asm:i32_lt_u())/binary + >>, + State1 = maybe_free(State0, Local), + emit(State1, Code); emit_condition(State0, {Local, '<', OtherLocal}) when is_atom(OtherLocal) -> L = unwrap_free(Local), Code = << diff --git a/libs/jit/src/jit_x86_64.erl b/libs/jit/src/jit_x86_64.erl index 7bc907b974..f47075342f 100644 --- a/libs/jit/src/jit_x86_64.erl +++ b/libs/jit/src/jit_x86_64.erl @@ -164,6 +164,7 @@ | {'(int)', maybe_free_x86_64_register(), '!=', x86_64_register() | integer()} | {'(bool)', maybe_free_x86_64_register(), '==', false} | {'(bool)', maybe_free_x86_64_register(), '!=', false} + | {'(unsigned)', maybe_free_x86_64_register(), '<', x86_64_register()} | {maybe_free_x86_64_register(), '&', non_neg_integer(), '!=', integer()} | {{free, x86_64_register()}, '==', {free, x86_64_register()}}. @@ -850,6 +851,16 @@ if_block_cond0(State0, {RegOrTuple, '<', Value}) when ?IS_SINT32_T(Value) -> {RelocJGEOffset, I2} = jit_x86_64_asm:jge_rel8(1), State1 = if_block_free_reg(RegOrTuple, State0), {State1, <>, byte_size(I1) + RelocJGEOffset}; +if_block_cond0(State0, {'(unsigned)', RegOrTuple, '<', RegB}) when is_atom(RegB) -> + Reg = + case RegOrTuple of + {free, Reg0} -> Reg0; + RegOrTuple -> RegOrTuple + end, + I1 = jit_x86_64_asm:cmpq(RegB, Reg), + {RelocJAEOffset, I2} = jit_x86_64_asm:jae_rel8(1), + State1 = if_block_free_reg(RegOrTuple, State0), + {State1, <>, byte_size(I1) + RelocJAEOffset}; if_block_cond0(State0, {RegOrTuple, '<', RegB}) when is_atom(RegB) -> Reg = case RegOrTuple of diff --git a/libs/jit/src/jit_x86_64_asm.erl b/libs/jit/src/jit_x86_64_asm.erl index 39d16d0b48..389b33e17a 100644 --- a/libs/jit/src/jit_x86_64_asm.erl +++ b/libs/jit/src/jit_x86_64_asm.erl @@ -35,6 +35,10 @@ jnz_rel8/1, jge/1, jge_rel8/1, + jb/1, + jb_rel8/1, + jae/1, + jae_rel8/1, jle/1, jle_rel8/1, jmp/1, @@ -372,6 +376,22 @@ jge(Offset) when Offset >= -126 andalso Offset =< 129 -> jge_rel8(Offset) when Offset >= -126 andalso Offset =< 129 -> {1, jge(Offset)}. +jb(Offset) when Offset >= -126 andalso Offset =< 129 -> + % Use short jump (matches assembler behavior) + AdjustedOffset = Offset - 2, + <<16#72, AdjustedOffset>>. + +jb_rel8(Offset) when Offset >= -126 andalso Offset =< 129 -> + {1, jb(Offset)}. + +jae(Offset) when Offset >= -126 andalso Offset =< 129 -> + % Use short jump (matches assembler behavior) + AdjustedOffset = Offset - 2, + <<16#73, AdjustedOffset>>. + +jae_rel8(Offset) when Offset >= -126 andalso Offset =< 129 -> + {1, jae(Offset)}. + jle(Offset) when Offset >= -126 andalso Offset =< 129 -> % Use short jump (matches assembler behavior) AdjustedOffset = Offset - 2, diff --git a/libs/jit/src/jit_xtensa.erl b/libs/jit/src/jit_xtensa.erl index 8b808fcfb8..c89810ef12 100644 --- a/libs/jit/src/jit_xtensa.erl +++ b/libs/jit/src/jit_xtensa.erl @@ -191,6 +191,7 @@ | {'(int)', maybe_free_xtensa_register(), '!=', xtensa_register() | integer()} | {'(bool)', maybe_free_xtensa_register(), '==', false} | {'(bool)', maybe_free_xtensa_register(), '!=', false} + | {'(unsigned)', maybe_free_xtensa_register(), '<', xtensa_register()} | {maybe_free_xtensa_register(), '&', non_neg_integer(), '!=', integer()} | {{free, xtensa_register()}, '==', {free, xtensa_register()}}. @@ -1066,6 +1067,22 @@ if_block_cond( State3 = State2#state{stream = Stream2}, JumpDelta = MovSize + byte_size(I1), {State3, JumpDelta}; +if_block_cond( + #state{stream_module = StreamModule, stream = Stream0} = State0, + {'(unsigned)', RegOrTuple, '<', RegB} +) when is_atom(RegB) -> + Reg = + case RegOrTuple of + {free, Reg0} -> Reg0; + RegOrTuple -> RegOrTuple + end, + %% Xtensa: bltu Reg, RegB, +2; J placeholder + I1 = jit_xtensa_asm:bltu(Reg, RegB, 2), + JPlaceholder = <<16#FF, 16#FF, 16#FF>>, + Stream1 = StreamModule:append(Stream0, <>), + State1 = if_block_free_reg(RegOrTuple, State0), + State2 = State1#state{stream = Stream1}, + {State2, byte_size(I1)}; if_block_cond( #state{stream_module = StreamModule, stream = Stream0} = State0, {RegOrTuple, '<', RegB} diff --git a/libs/jit/src/jit_xtensa_asm.erl b/libs/jit/src/jit_xtensa_asm.erl index fb2a0c0c3d..055f95aa20 100644 --- a/libs/jit/src/jit_xtensa_asm.erl +++ b/libs/jit/src/jit_xtensa_asm.erl @@ -43,6 +43,8 @@ beq/3, bne/3, blt/3, + bltu/3, + bgeu/3, beqz/2, bnez/2, bltz/2, @@ -383,6 +385,18 @@ bne(As, At, Offset) when Offset >= -128, Offset =< 127 -> blt(As, At, Offset) when Offset >= -128, Offset =< 127 -> encode_bri8(16#7, reg_to_num(At), reg_to_num(As), 16#2, Offset). +%% BLTU: if AR[s] < AR[t] (unsigned) then PC += sign_extend(imm8) +%% op0=7, r=3 +-spec bltu(xtensa_register(), xtensa_register(), integer()) -> binary(). +bltu(As, At, Offset) when Offset >= -128, Offset =< 127 -> + encode_bri8(16#7, reg_to_num(At), reg_to_num(As), 16#3, Offset). + +%% BGEU: if AR[s] >= AR[t] (unsigned) then PC += sign_extend(imm8) +%% op0=7, r=11 +-spec bgeu(xtensa_register(), xtensa_register(), integer()) -> binary(). +bgeu(As, At, Offset) when Offset >= -128, Offset =< 127 -> + encode_bri8(16#7, reg_to_num(At), reg_to_num(As), 16#B, Offset). + %% BEQZ: if AR[s] == 0 then PC += sign_extend(imm12) %% Encoding: imm12[11:0] in bits[23:12], s[11:8], t=0001b[7:4], op0=0110b[3:0] -spec beqz(xtensa_register(), integer()) -> binary(). diff --git a/libs/jit/src/primitives.hrl b/libs/jit/src/primitives.hrl index edbff68864..b5ff7b0284 100644 --- a/libs/jit/src/primitives.hrl +++ b/libs/jit/src/primitives.hrl @@ -98,6 +98,12 @@ -define(PRIM_RAW_RAISE, 75). -define(PRIM_RAISE_ERROR_MFA, 76). -define(PRIM_TRY_CASE, 77). +-define(PRIM_BITSTRING_GET_TAIL_HEAP_SIZE, 78). +-define(PRIM_BITSTRING_CREATE_TAIL, 79). +-define(PRIM_BS_CREATE_BIN_WRAP, 80). +-define(PRIM_BITSTRING_SLICE_HEAP_SIZE, 81). +-define(PRIM_BITSTRING_SLICE, 82). +-define(PRIM_BITSTRING_IS_MULTIPLE_OF, 83). % Parameters to ?PRIM_MEMORY_ENSURE_FREE_WITH_ROOTS % -define(MEMORY_NO_SHRINK, 0). diff --git a/libs/jit/src/term.hrl b/libs/jit/src/term.hrl index 2a40a032c6..100b5485c4 100644 --- a/libs/jit/src/term.hrl +++ b/libs/jit/src/term.hrl @@ -41,6 +41,7 @@ -define(TERM_BOXED_REFC_BINARY, 16#20). -define(TERM_BOXED_HEAP_BINARY, 16#24). -define(TERM_BOXED_SUB_BINARY, 16#28). +-define(TERM_BOXED_SUB_BINARY_SIZE, 4). -define(TERM_BOXED_MAP, 16#2C). -define(TERM_BOXED_EXTERNAL_PID, 16#30). -define(TERM_BOXED_EXTERNAL_PORT, 16#34). diff --git a/src/libAtomVM/bif.c b/src/libAtomVM/bif.c index 42cae38ee8..d4f68b9e25 100644 --- a/src/libAtomVM/bif.c +++ b/src/libAtomVM/bif.c @@ -122,10 +122,10 @@ term bif_erlang_byte_size_1(Context *ctx, uint32_t fail_label, int live, term ar if (term_is_match_state(arg1)) { avm_int_t offset = term_get_match_state_offset(arg1); term src_bin = term_get_match_state_binary(arg1); - len = term_binary_size(src_bin) - offset / 8; + len = (term_bit_size(src_bin) - offset + 7) / 8; } else { - VALIDATE_VALUE_BIF(fail_label, arg1, term_is_binary); - len = term_binary_size(arg1); + VALIDATE_VALUE_BIF(fail_label, arg1, term_is_bitstring); + len = (term_bit_size(arg1) + 7) / 8; } return term_from_int(len); @@ -140,10 +140,10 @@ term bif_erlang_bit_size_1(Context *ctx, uint32_t fail_label, int live, term arg if (term_is_match_state(arg1)) { avm_int_t offset = term_get_match_state_offset(arg1); term src_bin = term_get_match_state_binary(arg1); - len = term_binary_size(src_bin) * 8 - offset; + len = term_bit_size(src_bin) - offset; } else { - VALIDATE_VALUE_BIF(fail_label, arg1, term_is_binary); - len = term_binary_size(arg1) * 8; + VALIDATE_VALUE_BIF(fail_label, arg1, term_is_bitstring); + len = term_bit_size(arg1); } return term_from_int(len); @@ -189,6 +189,14 @@ term bif_erlang_is_binary_1(Context *ctx, uint32_t fail_label, term arg1) return term_is_binary(arg1) ? TRUE_ATOM : FALSE_ATOM; } +term bif_erlang_is_bitstring_1(Context *ctx, uint32_t fail_label, term arg1) +{ + UNUSED(ctx); + UNUSED(fail_label); + + return term_is_bitstring(arg1) ? TRUE_ATOM : FALSE_ATOM; +} + term bif_erlang_is_boolean_1(Context *ctx, uint32_t fail_label, term arg1) { UNUSED(ctx); @@ -2100,9 +2108,12 @@ term bif_erlang_max_2(Context *ctx, uint32_t fail_label, term arg1, term arg2) term bif_erlang_size_1(Context *ctx, uint32_t fail_label, int live, term arg1) { - if (term_is_binary(arg1)) { - // For bitstrings, number of bytes is rounded down - return bif_erlang_byte_size_1(ctx, fail_label, live, arg1); + UNUSED(live); + + if (term_is_bitstring(arg1)) { + // For bitstrings, number of bytes is rounded down, + // unlike byte_size/1 which rounds up + return term_from_int(term_binary_size(arg1)); } else if (term_is_tuple(arg1)) { return bif_erlang_tuple_size_1(ctx, fail_label, arg1); } diff --git a/src/libAtomVM/bif.h b/src/libAtomVM/bif.h index 15e5971f1c..2992af53cd 100644 --- a/src/libAtomVM/bif.h +++ b/src/libAtomVM/bif.h @@ -51,6 +51,7 @@ term bif_erlang_length_1(Context *ctx, uint32_t fail_label, int live, term arg1) term bif_erlang_is_atom_1(Context *ctx, uint32_t fail_label, term arg1); term bif_erlang_is_binary_1(Context *ctx, uint32_t fail_label, term arg1); +term bif_erlang_is_bitstring_1(Context *ctx, uint32_t fail_label, term arg1); term bif_erlang_is_boolean_1(Context *ctx, uint32_t fail_label, term arg1); term bif_erlang_is_float_1(Context *ctx, uint32_t fail_label, term arg1); term bif_erlang_is_function_1(Context *ctx, uint32_t fail_label, term arg1); diff --git a/src/libAtomVM/bifs.gperf b/src/libAtomVM/bifs.gperf index 58ef245fdc..88c97aef90 100644 --- a/src/libAtomVM/bifs.gperf +++ b/src/libAtomVM/bifs.gperf @@ -49,7 +49,7 @@ erlang:bit_size/1, {.gcbif.base.type = GCBIFFunctionType, .gcbif.gcbif1_ptr = bi erlang:binary_part/3, {.gcbif.base.type = GCBIFFunctionType, .gcbif.gcbif3_ptr = bif_erlang_binary_part_3} erlang:get/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_get_1} erlang:is_atom/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_atom_1} -erlang:is_bitstring/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_binary_1} +erlang:is_bitstring/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_bitstring_1} erlang:is_binary/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_binary_1} erlang:is_boolean/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_boolean_1} erlang:is_float/1, {.bif.base.type = BIFFunctionType, .bif.bif1_ptr = bif_erlang_is_float_1} diff --git a/src/libAtomVM/bitstring.c b/src/libAtomVM/bitstring.c index 9c279c6d05..d22720ab73 100644 --- a/src/libAtomVM/bitstring.c +++ b/src/libAtomVM/bitstring.c @@ -24,11 +24,6 @@ #include #include -static inline uint64_t from_le64(uint64_t value) -{ - return ((((value) &0xFF) << 56) | (((value) &0xFF00) << 40) | (((value) &0xFF0000) << 24) | (((value) &0xFF000000) << 8) | (((value) &0xFF00000000) >> 8) | (((value) &0xFF0000000000) >> 24) | (((value) &0xFF000000000000) >> 40) | (((value) &0xFF00000000000000) >> 56)); -} - bool bitstring_extract_any_integer(const uint8_t *src, size_t offset, avm_int_t n, enum BitstringFlags bs_flags, union maybe_unsigned_int64 *dst) { @@ -46,11 +41,22 @@ bool bitstring_extract_any_integer(const uint8_t *src, size_t offset, avm_int_t } if (bs_flags & LittleEndianIntegerMask) { - out = from_le64(out) >> (64 - n); + // Inverse of the little-endian insertion layout: the field stores the + // complete low-order bytes first (LSB byte first), then the remaining + // n rem 8 high-order bits. + size_t rem = (size_t) n & 0x7; + uint64_t raw = out; + uint64_t value = raw & ((((uint64_t) 1) << rem) - 1); + raw >>= rem; + for (size_t j = 0; j < ((size_t) n >> 3); ++j) { + value = (value << 8) | (raw & 0xFF); + raw >>= 8; + } + out = value; } - if ((bs_flags & SignedInteger) && (out & ((uint64_t) 1) << (i - 1))) { - dst->u = ((uint64_t) 0xFFFFFFFFFFFFFFFF << i) | out; + if ((bs_flags & SignedInteger) && i > 0 && (out & ((uint64_t) 1) << (i - 1))) { + dst->u = (i < 64) ? (((uint64_t) 0xFFFFFFFFFFFFFFFF << i) | out) : out; } else { dst->u = out; } @@ -58,16 +64,45 @@ bool bitstring_extract_any_integer(const uint8_t *src, size_t offset, avm_int_t return true; } +static void insert_bits_msb_first(uint8_t *dst, size_t offset, uint64_t value, size_t n) +{ + for (size_t i = 0; i < n; ++i) { + size_t k = (n - 1) - i; + int bit_val = (value & (0x01ULL << k)) >> k; + if (bit_val) { + size_t bit_pos = offset + i; + size_t byte_pos = bit_pos >> 3; // div 8 + uint8_t *pos = dst + byte_pos; + int shift = 7 - (bit_pos & 7); // mod 8 + *pos ^= (0x01 << shift); + } + } +} + +static void insert_sign_bits(uint8_t *dst, size_t offset, size_t count, bool negative) +{ + if (!negative) { + return; + } + while (count > 0) { + size_t chunk = count < 64 ? count : 64; + uint64_t ones = (chunk == 64) ? ~UINT64_C(0) : ((UINT64_C(1) << chunk) - 1); + insert_bits_msb_first(dst, offset, ones, chunk); + offset += chunk; + count -= chunk; + } +} + bool bitstring_insert_any_integer(uint8_t *dst, avm_int_t offset, avm_int64_t value, size_t n, enum BitstringFlags bs_flags) { // SignedInteger flag does not affect insertion (caller handles sign extension) bool little_endian = bs_flags & LittleEndianIntegerMask; + uint8_t sign_fill = (value < 0) ? 0xFF : 0x00; if (little_endian && (offset & 0x7) == 0 && (n & 0x7) == 0) { // Byte-aligned little-endian: write bytes LSB first size_t byte_offset = offset >> 3; size_t num_bytes = n >> 3; - // value is truncated to 64 bits size_t val_bytes = sizeof(value); uint64_t uvalue = (uint64_t) value; for (size_t i = 0; i < num_bytes; ++i) { @@ -75,39 +110,33 @@ bool bitstring_insert_any_integer(uint8_t *dst, avm_int_t offset, avm_int64_t va dst[byte_offset + i] = (uint8_t) (uvalue & 0xFF); uvalue >>= 8; } else { - dst[byte_offset + i] = 0; + dst[byte_offset + i] = sign_fill; } } + } else if (little_endian) { + // low-order bytes first (LSB byte first), then remaining high-order bits + size_t whole_bytes = n >> 3; + size_t rem = n & 0x7; + uint64_t uvalue = (uint64_t) value; + for (size_t i = 0; i < whole_bytes; ++i) { + uint8_t byte = (i < 8) ? (uint8_t) ((uvalue >> (8 * i)) & 0xFF) : sign_fill; + insert_bits_msb_first(dst, offset + 8 * i, byte, 8); + } + if (rem != 0) { + uint8_t byte + = (whole_bytes < 8) ? (uint8_t) ((uvalue >> (8 * whole_bytes)) & 0xFF) : sign_fill; + insert_bits_msb_first(dst, offset + 8 * whole_bytes, byte, rem); + } } else { - // Big-endian (or unaligned): write bits MSB first + // Big-endian: write bits MSB first, high-order sign bits first uint64_t write_value = (uint64_t) value; - if (little_endian) { - // Byte-swap the value so MSB-first bit writing produces little-endian bytes - size_t num_bytes = (n + 7) >> 3; - uint64_t swapped = 0; - for (size_t i = 0; i < num_bytes; ++i) { - swapped = (swapped << 8) | (write_value & 0xFF); - write_value >>= 8; - } - write_value = swapped; - } - - // value is truncated to 64 bits if (n > 8 * sizeof(value)) { - offset += n - (8 * sizeof(value)); + size_t high_bits = n - (8 * sizeof(value)); + insert_sign_bits(dst, offset, high_bits, value < 0); + offset += high_bits; n = 8 * sizeof(value); } - for (size_t i = 0; i < n; ++i) { - size_t k = (n - 1) - i; - int bit_val = (write_value & (0x01ULL << k)) >> k; - if (bit_val) { - int bit_pos = offset + i; - int byte_pos = bit_pos >> 3; // div 8 - uint8_t *pos = (uint8_t *) (dst + byte_pos); - int shift = 7 - (bit_pos & 7); // mod 8 - *pos ^= (0x01 << shift); - } - } + insert_bits_msb_first(dst, offset, write_value, n); } return true; } @@ -324,39 +353,84 @@ void bitstring_copy_bits_incomplete_bytes(uint8_t *dst, size_t bits_offset, cons { size_t byte_offset = bits_offset / 8; size_t bit_offset = bits_offset - (8 * byte_offset); - if (bits_offset % 8 == 0 && bits_count >= 8) { + if (bit_offset == 0 && bits_count >= 8) { size_t bytes_count = bits_count / 8; memcpy(dst + byte_offset, src, bytes_count); src += bytes_count; byte_offset += bytes_count; bits_count -= bytes_count * 8; } + if (bits_count == 0) { + return; + } // Eventually copy bit by bit dst += byte_offset; uint8_t dest_byte = *dst; - uint8_t src_byte = *src++; - int dest_bit_ix = 7 - bit_offset; - int src_bit_ix = 7; - while (bits_count > 0) { - if (src_byte & (1 << src_bit_ix)) { + int dest_bit_ix = 7 - (int) bit_offset; + for (size_t i = 0; i < bits_count; i++) { + if (src[i / 8] & (1 << (7 - (i % 8)))) { dest_byte |= 1 << dest_bit_ix; } else { dest_byte &= ~(1 << dest_bit_ix); } if (dest_bit_ix == 0) { *dst++ = dest_byte; - dest_byte = *dst; - dest_bit_ix = 8; + if (i + 1 < bits_count) { + dest_byte = *dst; + } + dest_bit_ix = 7; + } else { + dest_bit_ix--; } - if (src_bit_ix == 0) { - src_byte = *src++; - src_bit_ix = 8; + } + // The last byte was already written if the final bit completed it + if (dest_bit_ix != 7) { + *dst = dest_byte; + } +} + +void bitstring_copy_bits_from(uint8_t *dst, const uint8_t *src, size_t src_offset, size_t bits_count) +{ + for (size_t i = 0; i < bits_count; i++) { + size_t s = src_offset + i; + if (src[s / 8] & (uint8_t) (1 << (7 - (s % 8)))) { + dst[i / 8] |= (uint8_t) (1 << (7 - (i % 8))); } - dest_bit_ix--; - src_bit_ix--; - bits_count--; } - *dst = dest_byte; +} + +size_t bitstring_slice_heap_size(term bs_bin, size_t offset, size_t len_bits) +{ + if (offset % 8 == 0 && len_bits % 8 == 0) { + return term_sub_binary_heap_size(bs_bin, len_bits / 8); + } + size_t words = term_binary_heap_size((len_bits + 7) / 8); + if (len_bits % 8 != 0) { + words += TERM_BOXED_SUB_BINARY_SIZE; + } + return words; +} + +term bitstring_slice(term bs_bin, size_t offset, size_t len_bits, Heap *heap, GlobalContext *glb) +{ + if (offset % 8 == 0 && len_bits % 8 == 0) { + return term_maybe_create_sub_binary(bs_bin, offset / 8, len_bits / 8, heap, glb); + } + size_t result_bytes = (len_bits + 7) / 8; + // term_create_empty_binary zero-fills, as bitstring_copy_bits_from requires + term bin = term_create_empty_binary(result_bytes, heap, glb); + if (UNLIKELY(term_is_invalid_term(bin))) { + // the caller reserved heap words, but a refc binary buffer is allocated + // separately and that allocation may still fail + return bin; + } + uint8_t *dst = (uint8_t *) term_binary_data(bin); + bitstring_copy_bits_from(dst, (const uint8_t *) term_binary_data(bs_bin), offset, len_bits); + size_t trailing = len_bits % 8; + if (trailing != 0) { + return term_alloc_sub_binary_bits(bin, 0, len_bits / 8, (uint8_t) trailing, heap); + } + return bin; } bool bitstring_extract_f16( diff --git a/src/libAtomVM/bitstring.h b/src/libAtomVM/bitstring.h index 2c38cc43f7..50ffab0544 100644 --- a/src/libAtomVM/bitstring.h +++ b/src/libAtomVM/bitstring.h @@ -116,8 +116,8 @@ bool bitstring_insert_any_integer(uint8_t *dst, avm_int_t offset, avm_int64_t va static inline bool bitstring_extract_integer(term src_bin, size_t offset, avm_int_t n, enum BitstringFlags bs_flags, union maybe_unsigned_int64 *dst) { - unsigned long capacity = term_binary_size(src_bin); - if (8 * capacity - offset < (unsigned long) n) { + unsigned long capacity_bits = term_bit_size(src_bin); + if (capacity_bits - offset < (unsigned long) n) { return false; } @@ -486,6 +486,20 @@ static inline bool bitstring_match_utf32(term src_bin, size_t offset, int32_t *c */ void bitstring_copy_bits_incomplete_bytes(uint8_t *dst, size_t bits_offset, const uint8_t *src, size_t bits_count); +/** + * @brief Copy bits_count bits starting at bit src_offset of src into dst[0..]. + * + * @details Used to materialize a non-byte-aligned bitstring slice (e.g. a + * matched tail whose start is not a byte boundary) into a fresh byte-aligned + * buffer. The destination must be pre-zeroed; only set bits are written. + * + * @param dst destination buffer (pre-zeroed), receiving bits at offset 0 + * @param src source buffer + * @param src_offset offset in bits in the source buffer + * @param bits_count number of bits to copy + */ +void bitstring_copy_bits_from(uint8_t *dst, const uint8_t *src, size_t src_offset, size_t bits_count); + /** * @brief Copy bits_count bits from src to dst[bits_offset..] * @@ -519,6 +533,48 @@ bool bitstring_insert_f32( bool bitstring_insert_f64( term dst_bin, size_t offset, avm_float_t value, enum BitstringFlags bs_flags); +/** + * @brief Heap words needed to extract a len_bits slice at bit offset offset. + * @details Worst case: a byte-aligned offset and length shares storage via a + * sub-binary; otherwise the bits are copied into a fresh (possibly bitstring) + * binary. Pair with bitstring_slice. + */ +size_t bitstring_slice_heap_size(term bs_bin, size_t offset, size_t len_bits); + +/** + * @brief Extract a len_bits slice of bs_bin starting at bit offset offset. + * @details Byte-aligned offset+length shares storage via a sub-binary; a + * non-byte-aligned offset or length copies the bits into a fresh bitstring. + * Heap must already be reserved (see bitstring_slice_heap_size). + * @return the extracted slice, or `term_invalid_term()` if a refc binary buffer + * could not be allocated for a copied slice. + */ +term bitstring_slice(term bs_bin, size_t offset, size_t len_bits, Heap *heap, GlobalContext *glb); + +/** + * @brief Heap words needed to extract the tail of a match at bit offset bs_offset. + * @details Worst case: a byte-aligned start and length shares storage via a + * sub-binary; otherwise the remaining bits are copied into a fresh (possibly + * bitstring) binary. Pair with bitstring_get_tail. + */ +static inline size_t bitstring_get_tail_heap_size(term bs_bin, size_t bs_offset) +{ + return bitstring_slice_heap_size(bs_bin, bs_offset, term_bit_size(bs_bin) - bs_offset); +} + +/** + * @brief Materialize the tail of a match at bit offset bs_offset. + * @details Byte-aligned start+length shares storage via a sub-binary; a + * non-byte-aligned start or length copies the remaining bits into a fresh + * bitstring. Heap must already be reserved (see bitstring_get_tail_heap_size). + * @return the tail, or `term_invalid_term()` if a refc binary buffer could not + * be allocated for a copied tail. + */ +static inline term bitstring_get_tail(term bs_bin, size_t bs_offset, Heap *heap, GlobalContext *glb) +{ + return bitstring_slice(bs_bin, bs_offset, term_bit_size(bs_bin) - bs_offset, heap, glb); +} + #ifdef __cplusplus } #endif diff --git a/src/libAtomVM/defaultatoms.def b/src/libAtomVM/defaultatoms.def index 8911bc90a0..92c567aa6d 100644 --- a/src/libAtomVM/defaultatoms.def +++ b/src/libAtomVM/defaultatoms.def @@ -39,12 +39,12 @@ X(UNSUPPORTED_ATOM, "\xB", "unsupported") X(ALL_ATOM, "\x3", "all") X(LOWERCASE_EXIT_ATOM, "\x4", "exit") X(BADRECORD_ATOM, "\x9", "badrecord") +X(SYSTEM_LIMIT_ATOM, "\xC", "system_limit") // End of atoms used in jit compiler X(UNDEFINED_ATOM, "\x9", "undefined") X(BADARITY_ATOM, "\x08", "badarity") -X(SYSTEM_LIMIT_ATOM, "\xC", "system_limit") X(OVERFLOW_ATOM, "\x8", "overflow") X(FLUSH_ATOM, "\x5", "flush") diff --git a/src/libAtomVM/ets_multimap.c b/src/libAtomVM/ets_multimap.c index a16cac8b32..17408f9872 100644 --- a/src/libAtomVM/ets_multimap.c +++ b/src/libAtomVM/ets_multimap.c @@ -708,7 +708,7 @@ static uint32_t hash_term_incr(term t, uint32_t h, GlobalContext *global) return hash_local_reference(t, h, global); } else if (term_is_external_reference(t)) { return hash_external_reference(t, h, global); - } else if (term_is_binary(t)) { + } else if (term_is_bitstring(t)) { return hash_binary(t, h, global); } else if (term_is_tuple(t)) { size_t arity = term_get_tuple_arity(t); diff --git a/src/libAtomVM/external_term.c b/src/libAtomVM/external_term.c index f306e2e810..8d5dce1bef 100644 --- a/src/libAtomVM/external_term.c +++ b/src/libAtomVM/external_term.c @@ -53,6 +53,7 @@ #define STRING_EXT 107 #define LIST_EXT 108 #define BINARY_EXT 109 +#define BIT_BINARY_EXT 77 #define SMALL_BIG_EXT 110 #define NEW_FUN_EXT 112 #define EXPORT_EXT 113 @@ -70,6 +71,7 @@ #define STRING_EXT_BASE_SIZE 3 #define LIST_EXT_BASE_SIZE 5 #define BINARY_EXT_BASE_SIZE 5 +#define BIT_BINARY_EXT_BASE_SIZE 6 #define MAP_EXT_BASE_SIZE 5 #define SMALL_ATOM_EXT_BASE_SIZE 2 @@ -83,7 +85,7 @@ static term parse_external_terms(const uint8_t *external_term_buf, size_t *eterm static int calculate_heap_usage(const uint8_t *external_term_buf, size_t remaining, size_t *eterm_size, bool copy, external_term_read_opts_t opts, GlobalContext *glb); static size_t compute_external_size(term t, GlobalContext *glb); static int external_term_from_term(uint8_t **buf, size_t *len, term t, GlobalContext *glb); -static int serialize_term(uint8_t *buf, term t, GlobalContext *glb); +static size_t serialize_term(uint8_t *buf, term t, GlobalContext *glb); external_term_read_result_t external_term_validate_buf_raw(const void *buf, size_t buf_size, external_term_read_opts_t opts, size_t *required_heap, size_t *bytes_read, GlobalContext *glb) @@ -233,7 +235,7 @@ static void write_bytes(uint8_t *buf, avm_uint64_t val) } } -static int serialize_term(uint8_t *buf, term t, GlobalContext *glb) +static size_t serialize_term(uint8_t *buf, term t, GlobalContext *glb) { if (term_is_uint8(t)) { if (!IS_NULL_PTR(buf)) { @@ -380,17 +382,31 @@ static int serialize_term(uint8_t *buf, term t, GlobalContext *glb) } return k; - } else if (term_is_binary(t)) { - if (!IS_NULL_PTR(buf)) { - buf[0] = BINARY_EXT; - } - size_t len = term_binary_size(t); - if (!IS_NULL_PTR(buf)) { - const uint8_t *data = (const uint8_t *) term_binary_data(t); - WRITE_32_UNALIGNED(buf + 1, len); - memcpy(buf + 5, data, len); + } else if (term_is_bitstring(t)) { + if (term_is_binary(t)) { + size_t len = term_binary_size(t); + if (!IS_NULL_PTR(buf)) { + buf[0] = BINARY_EXT; + const uint8_t *data = (const uint8_t *) term_binary_data(t); + WRITE_32_UNALIGNED(buf + 1, len); + memcpy(buf + 5, data, len); + } + return 5 + len; + } else { + size_t total_bits = term_bit_size(t); + // ceil(total_bits / 8) without a `total_bits + 7` that could wrap + size_t nbytes = total_bits / 8 + (total_bits % 8 != 0); + uint8_t last_byte_bits = (uint8_t) (total_bits % 8); + if (!IS_NULL_PTR(buf)) { + buf[0] = BIT_BINARY_EXT; + const uint8_t *data = (const uint8_t *) term_binary_data(t); + WRITE_32_UNALIGNED(buf + 1, nbytes); + buf[5] = last_byte_bits; + memcpy(buf + 6, data, nbytes); + buf[6 + nbytes - 1] &= (uint8_t) (0xFF << (8 - last_byte_bits)); + } + return 6 + nbytes; } - return 5 + len; } else if (term_is_map(t)) { size_t size = term_get_map_size(t); @@ -659,6 +675,28 @@ static term insert_latin1_atom_ext( return globalcontext_insert_atom_maybe_copy(glb, utf8_buf, required_buf_size, true); } +static term parse_bigint( + const uint8_t *int_bytes, uint8_t int_len, bool is_negative, Heap *heap) +{ + intn_digit_t bigint[INTN_MAX_RES_LEN]; + int count = intn_from_integer_bytes(int_bytes, int_len, IntnLittleEndian, bigint, NULL); + if (UNLIKELY(count < 0)) { + // this means a bug, `calculate_heap_usage` already checks this + AVM_ABORT(); + } + + size_t intn_data_size; + size_t rounded_res_len; + term_bigint_size_requirements(count, &intn_data_size, &rounded_res_len); + + intn_integer_sign_t sign = is_negative ? IntNNegativeInteger : IntNPositiveInteger; + term bigint_term + = term_create_uninitialized_bigint(intn_data_size, (term_integer_sign_t) sign, heap); + term_initialize_bigint(bigint_term, bigint, count, rounded_res_len); + + return bigint_term; +} + static term parse_external_terms(const uint8_t *external_term_buf, size_t *eterm_size, bool copy, Heap *heap, GlobalContext *glb, external_term_read_opts_t opts) { // The safe flag is enforced in calculate_heap_usage (which must run first); @@ -705,23 +743,7 @@ static term parse_external_terms(const uint8_t *external_term_buf, size_t *eterm } // int_len > 8 || uint64_does_overflow_int64 - intn_digit_t bigint[INTN_MAX_RES_LEN]; - int count = intn_from_integer_bytes(int_bytes, int_len, IntnLittleEndian, bigint, NULL); - if (UNLIKELY(count < 0)) { - // this means a bug, `calculate_heap_usage` already checks this - AVM_ABORT(); - } - - size_t intn_data_size; - size_t rounded_res_len; - term_bigint_size_requirements(count, &intn_data_size, &rounded_res_len); - - intn_integer_sign_t sign = is_negative ? IntNNegativeInteger : IntNPositiveInteger; - term bigint_term - = term_create_uninitialized_bigint(intn_data_size, (term_integer_sign_t) sign, heap); - term_initialize_bigint(bigint_term, bigint, count, rounded_res_len); - - return bigint_term; + return parse_bigint(int_bytes, int_len, is_negative, heap); } case ATOM_EXT: { @@ -839,6 +861,33 @@ static term parse_external_terms(const uint8_t *external_term_buf, size_t *eterm } } + case BIT_BINARY_EXT: { + uint32_t binary_size = READ_32_UNALIGNED(external_term_buf + 1); + uint8_t last_byte_bits = external_term_buf[5]; + // last_byte_bits is the number of significant bits in the trailing + // byte and must be in 1..8 (8 meaning byte-aligned), over at least + // one byte. A zero length is what underflowed the size computation + // in OTP (CVE-2026-54890); the fixed decoders reject it, as do we. + if (UNLIKELY(last_byte_bits < 1 || last_byte_bits > 8 || binary_size == 0)) { + return term_invalid_term(); + } + *eterm_size = (size_t) BIT_BINARY_EXT_BASE_SIZE + (size_t) binary_size; + term bin; + if (copy) { + bin = term_from_literal_binary((uint8_t *) external_term_buf + 6, binary_size, heap, glb); + } else { + bin = term_from_const_binary((uint8_t *) external_term_buf + 6, binary_size, heap, glb); + } + if (UNLIKELY(term_is_invalid_term(bin))) { + return bin; + } + // 8 significant bits means byte-aligned. + if (last_byte_bits == 8) { + return bin; + } + return term_alloc_sub_binary_bits(bin, 0, binary_size - 1, last_byte_bits, heap); + } + case EXPORT_EXT: { size_t buf_pos = 1; size_t element_size; @@ -1314,6 +1363,43 @@ static int calculate_heap_usage(const uint8_t *external_term_buf, size_t remaini } } + case BIT_BINARY_EXT: { + if (UNLIKELY(remaining < BIT_BINARY_EXT_BASE_SIZE)) { + return INVALID_TERM_SIZE; + } + uint32_t binary_size = READ_32_UNALIGNED(external_term_buf + 1); + uint8_t last_byte_bits = external_term_buf[5]; + // See BIT_BINARY_EXT in parse_external_terms: 1..8 significant bits + // over at least one byte. + if (UNLIKELY(last_byte_bits < 1 || last_byte_bits > 8 || binary_size == 0)) { + return INVALID_TERM_SIZE; + } + remaining -= BIT_BINARY_EXT_BASE_SIZE; + if (UNLIKELY(remaining < binary_size)) { + return INVALID_TERM_SIZE; + } + *eterm_size = (size_t) BIT_BINARY_EXT_BASE_SIZE + (size_t) binary_size; + +#if TERM_BYTES == 4 + int size_in_terms = ((binary_size + 4 - 1) >> 2); +#elif TERM_BYTES == 8 + int size_in_terms = ((binary_size + 8 - 1) >> 3); +#else +#error +#endif + + size_t words; + if (copy && term_binary_size_is_heap_binary(binary_size)) { + words = 2 + size_in_terms; + } else { + words = TERM_BOXED_REFC_BINARY_SIZE; + } + if (last_byte_bits < 8) { + words += TERM_BOXED_SUB_BINARY_SIZE; + } + return words; + } + case EXPORT_EXT: { if (UNLIKELY(remaining < 1)) { return INVALID_TERM_SIZE; diff --git a/src/libAtomVM/interop.c b/src/libAtomVM/interop.c index c98077080b..9b249dc214 100644 --- a/src/libAtomVM/interop.c +++ b/src/libAtomVM/interop.c @@ -267,8 +267,10 @@ InteropFunctionResult interop_chardata_fold(term t, interop_chardata_fold_fun fo return InteropMemoryAllocFail; } + bool tail_position = true; + while (!temp_stack_is_empty(&temp_stack)) { - if (term_is_integer(t) || term_is_binary(t)) { + if ((term_is_integer(t) && !tail_position) || term_is_binary(t)) { InteropFunctionResult result = fold_fun(t, accum); if (UNLIKELY(result != InteropOk)) { if (rest_fun) { @@ -284,10 +286,12 @@ InteropFunctionResult interop_chardata_fold(term t, interop_chardata_fold_fun fo return result; } else { t = temp_stack_pop(&temp_stack); + tail_position = true; } } else if (term_is_nil(t)) { t = temp_stack_pop(&temp_stack); + tail_position = true; } else if (term_is_nonempty_list(t)) { if (UNLIKELY(temp_stack_push(&temp_stack, term_get_list_tail(t)) != TempStackOk)) { @@ -295,6 +299,7 @@ InteropFunctionResult interop_chardata_fold(term t, interop_chardata_fold_fun fo return InteropMemoryAllocFail; } t = term_get_list_head(t); + tail_position = false; } else { if (rest_fun) { @@ -499,7 +504,7 @@ static InteropFunctionResult chardata_to_bytes_size_fold_fun(term t, void *accum static void chardata_to_bytes_size_rest_fun(term t, void *accum) { struct CharDataToBytesSizeAcc *acc = (struct CharDataToBytesSizeAcc *) accum; - if (!term_is_binary(t) && !term_is_integer(t) && !term_is_list(t)) { + if (!term_is_binary(t) && !term_is_list(t)) { acc->badarg = true; } if (!acc->badarg) { @@ -612,7 +617,7 @@ static InteropFunctionResult chardata_to_bytes_fold_fun(term t, void *accum) static void chardata_to_bytes_rest_fun(term t, void *accum) { struct CharDataToBytesAcc *acc = (struct CharDataToBytesAcc *) accum; - if (!term_is_binary(t) && !term_is_integer(t) && !term_is_list(t)) { + if (!term_is_binary(t) && !term_is_list(t)) { acc->badarg = true; } if (!acc->badarg) { diff --git a/src/libAtomVM/jit.c b/src/libAtomVM/jit.c index 30798f5cf1..d95a9b6722 100644 --- a/src/libAtomVM/jit.c +++ b/src/libAtomVM/jit.c @@ -139,6 +139,7 @@ _Static_assert(UNSUPPORTED_ATOM_INDEX == 14, "UNSUPPORTED_ATOM_INDEX is 14 in li _Static_assert(ALL_ATOM_INDEX == 15, "ALL_ATOM_INDEX is 15 in libs/jit/src/default_atoms.hrl "); _Static_assert(LOWERCASE_EXIT_ATOM_INDEX == 16, "LOWERCASE_EXIT_ATOM_INDEX is 16 in libs/jit/src/default_atoms.hrl "); _Static_assert(BADRECORD_ATOM_INDEX == 17, "BADRECORD_ATOM_INDEX is 17 in libs/jit/src/default_atoms.hrl "); +_Static_assert(SYSTEM_LIMIT_ATOM_INDEX == 18, "SYSTEM_LIMIT_ATOM_INDEX is 18 in libs/jit/src/default_atoms.hrl "); // Verify n_words constants in primitives.hrl _Static_assert( @@ -1504,19 +1505,20 @@ static term jit_bitstring_extract_integer( } return t; - } else if ((offset % 8 == 0) && (n % 8 == 0) && (n <= INTN_MAX_UNSIGNED_BITS_SIZE)) { + } else { term bs_bin = (term) (((uintptr_t) bin_ptr) | TERM_PRIMARY_BOXED); - unsigned long capacity = term_binary_size(bs_bin); - if (8 * capacity - offset < (unsigned long) n) { + size_t capacity_bits = term_bit_size(bs_bin); + if (offset > capacity_bits || capacity_bits - offset < (size_t) n) { return FALSE_ATOM; } - size_t byte_offset = offset / 8; - const uint8_t *int_bytes = (const uint8_t *) term_binary_data(bs_bin); - - return extract_bigint( - ctx, jit_state, int_bytes + byte_offset, n / 8, bitstring_flags_to_intn_opts(bs_flags)); - } else { - return FALSE_ATOM; + if ((offset % 8 == 0) && (n % 8 == 0) && (n <= INTN_MAX_UNSIGNED_BITS_SIZE)) { + size_t byte_offset = offset / 8; + const uint8_t *int_bytes = (const uint8_t *) term_binary_data(bs_bin); + return extract_bigint(ctx, jit_state, int_bytes + byte_offset, n / 8, + bitstring_flags_to_intn_opts(bs_flags)); + } + set_error(ctx, jit_state, 0, UNSUPPORTED_ATOM); + return term_invalid_term(); } } @@ -1564,6 +1566,42 @@ static term jit_term_maybe_create_sub_binary(Context *ctx, term binary, size_t o return term_maybe_create_sub_binary(binary, offset, len, &ctx->heap, ctx->global); } +static size_t jit_bitstring_get_tail_heap_size(term *bs_bin_ptr, size_t bs_offset) +{ + term bs_bin = (term) (((uintptr_t) bs_bin_ptr) | TERM_PRIMARY_BOXED); + return bitstring_get_tail_heap_size(bs_bin, bs_offset); +} + +static term jit_bs_create_bin_wrap(Context *ctx, term byte_binary, size_t total_bits) +{ + if (total_bits % 8 == 0) { + return byte_binary; + } + return term_alloc_sub_binary_bits(byte_binary, 0, total_bits / 8, (uint8_t) (total_bits % 8), &ctx->heap); +} + +static term jit_bitstring_create_tail(Context *ctx, term bs_bin, size_t bs_offset) +{ + return bitstring_get_tail(bs_bin, bs_offset, &ctx->heap, ctx->global); +} + +static size_t jit_bitstring_slice_heap_size(term *bs_bin_ptr, size_t offset, size_t len_bits) +{ + term bs_bin = (term) (((uintptr_t) bs_bin_ptr) | TERM_PRIMARY_BOXED); + return bitstring_slice_heap_size(bs_bin, offset, len_bits); +} + +// Heap is assumed already reserved (see jit_bitstring_slice_heap_size); does not GC. +static term jit_bitstring_slice(Context *ctx, term bs_bin, size_t offset, size_t len_bits) +{ + return bitstring_slice(bs_bin, offset, len_bits, &ctx->heap, ctx->global); +} + +static bool jit_bitstring_is_multiple_of(size_t bits, size_t unit) +{ + return unit != 0 && bits % unit == 0; +} + static int jit_term_find_map_pos(Context *ctx, term map, term key) { return term_find_map_pos(map, key, ctx->global); @@ -1654,6 +1692,10 @@ static bool jit_bitstring_insert_integer(term bin, size_t offset, term value, si avm_uint64_t int_value = term_maybe_unbox_int64(value); return bitstring_insert_integer(bin, offset, int_value, n, flags); + } else if ((offset % 8 != 0) || (n % 8 != 0)) { + // TODO: intn_to_integer_bytes writes whole bytes only, so a bignum + // segment at a bit offset needs a bit-level writer. + return false; } else { const intn_digit_t *big_src_value = NULL; size_t big_len = 0; @@ -1699,26 +1741,18 @@ static void jit_bitstring_copy_module_str(Context *ctx, JITState *jit_state, ter bitstring_copy_bits(dst, offset, str, len); } -static int jit_bitstring_copy_binary(Context *ctx, JITState *jit_state, term t, size_t offset, term src, term size) +static int jit_bitstring_copy_binary(term t, size_t offset, term src, term size) { TRACE("jit_bitstring_copy_binary: t=%p offset=%d src=%p size=%p\n", (void *) t, (int) offset, (void *) src, (void *) size); - if (offset % 8) { - set_error(ctx, jit_state, 0, UNSUPPORTED_ATOM); - return -1; - } - uint8_t *dst = (uint8_t *) term_binary_data(t) + (offset / 8); + uint8_t *dst = (uint8_t *) term_binary_data(t); const uint8_t *bin = (const uint8_t *) term_binary_data(src); - size_t binary_size = term_binary_size(src); + size_t src_bits = term_bit_size(src); + size_t copy_bits = src_bits; if (size != ALL_ATOM) { - binary_size = (size_t) term_to_int(size); - if (binary_size % 8) { - set_error(ctx, jit_state, 0, UNSUPPORTED_ATOM); - return -1; - } - binary_size = binary_size / 8; + copy_bits = (size_t) term_to_int(size); } - memcpy(dst, bin, binary_size); - return binary_size * 8; + bitstring_copy_bits(dst, offset, bin, copy_bits); + return (int) copy_bits; } static Context *jit_apply(Context *ctx, JITState *jit_state, int offset, term module, term function, unsigned int arity) @@ -1893,7 +1927,8 @@ static bool jit_bitstring_match_module_str(Context *ctx, JITState *jit_state, te size_t remaining = 0; const uint8_t *str = module_get_str(jit_state->module, str_id, &remaining); - if (term_binary_size(bs_bin) * 8 - bs_offset < MIN(remaining * 8, bits)) { + size_t capacity_bits = term_bit_size(bs_bin); + if (bs_offset > capacity_bits || capacity_bits - bs_offset < MIN(remaining * 8, bits)) { return false; } @@ -2080,7 +2115,13 @@ const ModuleNativeInterface module_native_interface = { jit_bitstring_insert_float, jit_raw_raise, jit_raise_error_mfa, - jit_try_case + jit_try_case, + jit_bitstring_get_tail_heap_size, + jit_bitstring_create_tail, + jit_bs_create_bin_wrap, + jit_bitstring_slice_heap_size, + jit_bitstring_slice, + jit_bitstring_is_multiple_of }; #endif diff --git a/src/libAtomVM/jit.h b/src/libAtomVM/jit.h index e136597986..cc98ed818d 100644 --- a/src/libAtomVM/jit.h +++ b/src/libAtomVM/jit.h @@ -229,7 +229,7 @@ struct ModuleNativeInterface bool (*bitstring_insert_utf32)(term bin, size_t offset, uint32_t c, enum BitstringFlags flags); bool (*bitstring_insert_integer)(term bin, size_t offset, term value, size_t n, enum BitstringFlags flags); void (*bitstring_copy_module_str)(Context *ctx, JITState *jit_state, term bin, size_t offset, int str_id, size_t len); - int (*bitstring_copy_binary)(Context *ctx, JITState *jit_state, term t, size_t offset, term src, term size); + int (*bitstring_copy_binary)(term t, size_t offset, term src, term size); Context *(*apply)(Context *ctx, JITState *jit_state, int offset, term module, term function, unsigned int arity); void *(*malloc)(Context *ctx, JITState *jit_state, size_t sz); void (*free)(void *ptr); @@ -249,6 +249,12 @@ struct ModuleNativeInterface Context *(*raise_error_mfa)( Context *ctx, JITState *jit_state, int offset, int function_atom_index, int arity); void (*try_case)(Context *ctx); + size_t (*bitstring_get_tail_heap_size)(term *bs_bin_ptr, size_t bs_offset); + term (*bitstring_create_tail)(Context *ctx, term bs_bin, size_t bs_offset); + term (*bs_create_bin_wrap)(Context *ctx, term byte_binary, size_t total_bits); + size_t (*bitstring_slice_heap_size)(term *bs_bin_ptr, size_t offset, size_t len_bits); + term (*bitstring_slice)(Context *ctx, term bs_bin, size_t offset, size_t len_bits); + bool (*bitstring_is_multiple_of)(size_t bits, size_t unit); }; extern const ModuleNativeInterface module_native_interface; diff --git a/src/libAtomVM/nifs.c b/src/libAtomVM/nifs.c index 1e425d6b17..86ba6fa38b 100644 --- a/src/libAtomVM/nifs.c +++ b/src/libAtomVM/nifs.c @@ -172,6 +172,7 @@ static term nif_erlang_binary_to_atom_1(Context *ctx, int argc, term argv[]); static term nif_erlang_binary_to_float_1(Context *ctx, int argc, term argv[]); static term nif_erlang_binary_to_integer(Context *ctx, int argc, term argv[]); static term nif_erlang_binary_to_list_1(Context *ctx, int argc, term argv[]); +static term nif_erlang_bitstring_to_list_1(Context *ctx, int argc, term argv[]); static term nif_erlang_binary_to_existing_atom_1(Context *ctx, int argc, term argv[]); static term nif_erlang_concat_2(Context *ctx, int argc, term argv[]); static term nif_erlang_display_1(Context *ctx, int argc, term argv[]); @@ -408,6 +409,11 @@ static const struct Nif binary_to_list_nif = { .nif_ptr = nif_erlang_binary_to_list_1 }; +static const struct Nif bitstring_to_list_nif = { + .base.type = NIFFunctionType, + .nif_ptr = nif_erlang_bitstring_to_list_1 +}; + static const struct Nif binary_to_existing_atom_1_nif = { .base.type = NIFFunctionType, .nif_ptr = nif_erlang_binary_to_existing_atom_1 @@ -2608,20 +2614,38 @@ static term nif_erlang_list_to_float_1(Context *ctx, int argc, term argv[]) } static term nif_erlang_binary_to_list_1(Context *ctx, int argc, term argv[]) +{ + VALIDATE_VALUE(argv[0], term_is_binary); + return nif_erlang_bitstring_to_list_1(ctx, argc, argv); +} + +static term nif_erlang_bitstring_to_list_1(Context *ctx, int argc, term argv[]) { UNUSED(argc); term value = argv[0]; - VALIDATE_VALUE(value, term_is_binary); + VALIDATE_VALUE(value, term_is_bitstring); int bin_size = term_binary_size(value); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, bin_size * 2, 1, &value, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + uint8_t trailing_bits = (uint8_t) (term_bit_size(value) % 8); + + size_t heap_needed = (size_t) bin_size * 2; + if (trailing_bits != 0) { + heap_needed += term_binary_heap_size(1) + TERM_BOXED_SUB_BINARY_SIZE + CONS_SIZE; + } + if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_needed, 1, &value, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } const uint8_t *bin_data = (const uint8_t *) term_binary_data(value); term prev = term_nil(); + if (trailing_bits != 0) { + term tbin = term_create_empty_binary(1, &ctx->heap, ctx->global); + ((uint8_t *) term_binary_data(tbin))[0] = bin_data[bin_size]; + term tsub = term_alloc_sub_binary_bits(tbin, 0, 0, trailing_bits, &ctx->heap); + prev = term_list_prepend(tsub, prev, &ctx->heap); + } for (int i = bin_size - 1; i >= 0; i--) { prev = term_list_prepend(term_from_int11(bin_data[i]), prev, &ctx->heap); } @@ -3720,7 +3744,7 @@ static term nif_erlang_split_binary(Context *ctx, int argc, term argv[]) term bin_term = argv[0]; term pos_term = argv[1]; - VALIDATE_VALUE(bin_term, term_is_binary); + VALIDATE_VALUE(bin_term, term_is_bitstring); VALIDATE_VALUE(pos_term, term_is_integer); int32_t size = term_binary_size(bin_term); @@ -3730,12 +3754,24 @@ static term nif_erlang_split_binary(Context *ctx, int argc, term argv[]) RAISE_ERROR(BADARG_ATOM); } - size_t alloc_heap_size = term_sub_binary_heap_size(bin_term, pos) + term_sub_binary_heap_size(bin_term, size - pos) + TUPLE_SIZE(2); + uint8_t trailing_bits = (uint8_t) (term_bit_size(bin_term) % 8); + + size_t second_part_heap_size = trailing_bits == 0 + ? term_sub_binary_heap_size(bin_term, size - pos) + : TERM_BOXED_SUB_BINARY_SIZE; + size_t alloc_heap_size = term_sub_binary_heap_size(bin_term, pos) + second_part_heap_size + TUPLE_SIZE(2); if (UNLIKELY(memory_ensure_free_with_roots(ctx, alloc_heap_size, 1, &bin_term, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } term sub_binary_a = term_maybe_create_sub_binary(bin_term, 0, pos, &ctx->heap, ctx->global); - term sub_binary_b = term_maybe_create_sub_binary(bin_term, pos, size - pos, &ctx->heap, ctx->global); + term sub_binary_b; + if (trailing_bits == 0) { + sub_binary_b = term_maybe_create_sub_binary(bin_term, pos, size - pos, &ctx->heap, ctx->global); + } else { + // only a sub-binary can carry trailing bits + size_t offset = term_get_sub_binary_offset(bin_term); + sub_binary_b = term_alloc_sub_binary_bits(bin_term, offset + pos, size - pos, trailing_bits, &ctx->heap); + } term tuple = term_alloc_tuple(2, &ctx->heap); term_put_tuple_element(tuple, 0, sub_binary_a); term_put_tuple_element(tuple, 1, sub_binary_b); @@ -7614,10 +7650,139 @@ static term nif_lists_keyfind(Context *ctx, int argc, term argv[]) return FALSE_ATOM; } +enum BitstringFoldResult +{ + BitstringFoldOk, + BitstringFoldMemoryAllocFail, + BitstringFoldBadArg, + BitstringFoldSystemLimit +}; + +static enum BitstringFoldResult fold_bitstring_append(term t, uint8_t *dst, size_t *bits) +{ + size_t n = term_bit_size(t); + // The list may reference the same large bitstring several times, so the + // total can overflow even for a short list. + if (UNLIKELY(n > SIZE_MAX - *bits)) { + return BitstringFoldSystemLimit; + } + if (dst && n != 0) { + bitstring_copy_bits(dst, *bits, (const uint8_t *) term_binary_data(t), n); + } + *bits += n; + return BitstringFoldOk; +} + +// Walks a list of bytes and bitstrings, measuring it in bits and, when dst is +// not NULL, copying it there. Bytes are only accepted as list elements, while +// an improper tail may be a bitstring: `[1 | <<2>>]' is valid but `[1 | 2]' is +// not. +static enum BitstringFoldResult fold_bitstring_list(term t, uint8_t *dst, size_t *total_bits) +{ + size_t bits = 0; + struct TempStack temp_stack; + if (UNLIKELY(temp_stack_init(&temp_stack) != TempStackOk)) { + return BitstringFoldMemoryAllocFail; + } + // t is always in tail position here: the stack holds the tails of the + // lists we descended into, and they are resumed in tail position too. + for (;;) { + if (term_is_nonempty_list(t)) { + term head = term_get_list_head(t); + if (term_is_integer(head)) { + avm_int_t value = term_to_int(head); + if (UNLIKELY(value < 0 || value > UCHAR_MAX)) { + temp_stack_destroy(&temp_stack); + return BitstringFoldBadArg; + } + if (dst) { + uint8_t byte = (uint8_t) value; + bitstring_copy_bits(dst, bits, &byte, 8); + } + // A list of integers long enough to overflow `bits` cannot be + // built: its cons cells alone would not fit in memory. Only + // the bitstring case needs an overflow check. + bits += 8; + } else if (term_is_bitstring(head)) { + enum BitstringFoldResult result = fold_bitstring_append(head, dst, &bits); + if (UNLIKELY(result != BitstringFoldOk)) { + temp_stack_destroy(&temp_stack); + return result; + } + } else if (term_is_list(head)) { + if (UNLIKELY(temp_stack_push(&temp_stack, term_get_list_tail(t)) != TempStackOk)) { + temp_stack_destroy(&temp_stack); + return BitstringFoldMemoryAllocFail; + } + t = head; + continue; + } else { + temp_stack_destroy(&temp_stack); + return BitstringFoldBadArg; + } + t = term_get_list_tail(t); + } else if (term_is_bitstring(t)) { + enum BitstringFoldResult result = fold_bitstring_append(t, dst, &bits); + if (UNLIKELY(result != BitstringFoldOk)) { + temp_stack_destroy(&temp_stack); + return result; + } + if (temp_stack_is_empty(&temp_stack)) { + break; + } + t = temp_stack_pop(&temp_stack); + } else if (term_is_nil(t)) { + if (temp_stack_is_empty(&temp_stack)) { + break; + } + t = temp_stack_pop(&temp_stack); + } else { + temp_stack_destroy(&temp_stack); + return BitstringFoldBadArg; + } + } + temp_stack_destroy(&temp_stack); + *total_bits = bits; + return BitstringFoldOk; +} + static term nif_erlang_list_to_bitstring_1(Context *ctx, int argc, term argv[]) { - // TODO: implement proper list_to_bitstring function when the bitstrings are supported - return nif_erlang_list_to_binary_1(ctx, argc, argv); + VALIDATE_VALUE(argv[0], term_is_list); + + size_t total_bits; + enum BitstringFoldResult result = fold_bitstring_list(argv[0], NULL, &total_bits); + if (UNLIKELY(result != BitstringFoldOk)) { + RAISE_ERROR(result == BitstringFoldBadArg + ? BADARG_ATOM + : (result == BitstringFoldSystemLimit ? SYSTEM_LIMIT_ATOM : OUT_OF_MEMORY_ATOM)); + } + + size_t whole_bytes = total_bits / 8; + uint8_t trailing_bits = (uint8_t) (total_bits % 8); + size_t alloc_bytes = whole_bytes + (trailing_bits != 0 ? 1 : 0); + if (UNLIKELY(alloc_bytes > TERM_MAX_BINARY_SIZE)) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + size_t heap_size = term_binary_heap_size(alloc_bytes) + + (trailing_bits != 0 ? TERM_BOXED_SUB_BINARY_SIZE : 0); + if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, argc, argv, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } + term bin = term_create_empty_binary(alloc_bytes, &ctx->heap, ctx->global); + if (UNLIKELY(term_is_invalid_term(bin))) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } + if (total_bits != 0) { + result = fold_bitstring_list(argv[0], (uint8_t *) term_binary_data(bin), &total_bits); + if (UNLIKELY(result != BitstringFoldOk)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } + } + if (trailing_bits != 0) { + return term_alloc_sub_binary_bits(bin, 0, whole_bytes, trailing_bits, &ctx->heap); + } + return bin; } #ifndef WITH_ZLIB diff --git a/src/libAtomVM/nifs.gperf b/src/libAtomVM/nifs.gperf index 8457186429..cf7b102825 100644 --- a/src/libAtomVM/nifs.gperf +++ b/src/libAtomVM/nifs.gperf @@ -65,7 +65,7 @@ erlang:binary_to_float/1, &binary_to_float_nif erlang:binary_to_integer/1, &binary_to_integer_nif erlang:binary_to_integer/2, &binary_to_integer_nif erlang:binary_to_list/1, &binary_to_list_nif -erlang:bitstring_to_list/1, &binary_to_list_nif +erlang:bitstring_to_list/1, &bitstring_to_list_nif erlang:binary_to_existing_atom/1, &binary_to_existing_atom_1_nif erlang:delete_element/2, &delete_element_nif erlang:erase/0, &erase_0_nif diff --git a/src/libAtomVM/opcodesswitch.h b/src/libAtomVM/opcodesswitch.h index 413311fba5..fe697758db 100644 --- a/src/libAtomVM/opcodesswitch.h +++ b/src/libAtomVM/opcodesswitch.h @@ -1063,14 +1063,14 @@ static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) } \ } -#define VERIFY_IS_BINARY(t, opcode_name, label) \ - if (UNLIKELY(!term_is_binary(t))) { \ - TRACE(opcode_name ": " #t " is not a binary\n"); \ - if (label == 0) { \ - RAISE_ERROR(BADARG_ATOM); \ - } else { \ - JUMP_TO_LABEL(mod, label); \ - } \ +#define VERIFY_IS_BITSTRING(t, opcode_name, label) \ + if (UNLIKELY(!term_is_bitstring(t))) { \ + TRACE(opcode_name ": " #t " is not a bitstring\n"); \ + if (label == 0) { \ + RAISE_ERROR(BADARG_ATOM); \ + } else { \ + JUMP_TO_LABEL(mod, label); \ + } \ } #define VERIFY_IS_MATCH_STATE(t, opcode_name, label) \ @@ -1083,10 +1083,10 @@ static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) } \ } -#define VERIFY_IS_MATCH_OR_BINARY(t, opcode_name) \ - if (UNLIKELY(!(term_is_binary(t) || term_is_match_state(t)))) { \ - TRACE(opcode_name ": " #t " is not a binary or match context.\n"); \ - RAISE_ERROR(BADARG_ATOM); \ +#define VERIFY_IS_MATCH_OR_BINARY(t, opcode_name) \ + if (UNLIKELY(!(term_is_bitstring(t) || term_is_match_state(t)))) { \ + TRACE(opcode_name ": " #t " is not a bitstring or match context.\n"); \ + RAISE_ERROR(BADARG_ATOM); \ } #define CALL_FUN(fun, args_count) \ @@ -1239,36 +1239,35 @@ static bool sort_kv_pairs(struct kv_pair *kv, int size, GlobalContext *global) #endif /** - * @brief Scale a dynamic segment size by its unit. + * @brief Scale a dynamic segment size by its unit, in bits. * - * @details Matching opcodes take a segment size from a register and scale it by - * the segment unit. The size can be negative or large enough for the product to - * overflow, so it cannot be multiplied blindly: the scaled size is compared - * against the remaining capacity and added to the match offset, and scaling - * first lets a negative size wrap to a small one that passes the capacity check, - * or move the match offset before the start of the binary. + * @details Matching opcodes take a segment size that is scaled by the segment + * unit. The size comes from a register and can be negative or large enough for + * the product to overflow, so it cannot be multiplied blindly: converting a + * negative value to `size_t` and multiplying can wrap back to a small size that + * then passes the capacity check and matches, where BEAM fails the match. * * @param size the segment size, as a term (must be any integer) - * @param unit the segment unit - * @param max_value the largest acceptable scaled size - * @param scaled_size on success, the scaled size - * @returns \c true if the scaled size is representable and at most - * \c max_value, \c false if the match should fail + * @param unit the segment unit, in bits + * @param max_bits the number of bits available for this segment + * @param size_bits on success, the scaled size, in bits + * @returns \c true if the scaled size is representable and fits within + * \c max_bits, \c false if the match should fail */ -static inline bool bs_scaled_size(term size, uint32_t unit, size_t max_value, size_t *scaled_size) +static inline bool bs_scaled_size_bits(term size, uint32_t unit, size_t max_bits, size_t *size_bits) { if (!term_is_integer(size)) { - // a size that doesn't fit in a small integer cannot fit in max_value + // a size that doesn't fit in a small integer cannot fit in max_bits return false; } avm_int_t size_val = term_to_int(size); if (size_val < 0) { return false; } - if (unit != 0 && (size_t) size_val > max_value / unit) { + if (unit != 0 && (size_t) size_val > max_bits / unit) { return false; } - *scaled_size = (size_t) size_val * unit; + *size_bits = (size_t) size_val * unit; return true; } @@ -3438,9 +3437,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_INTEGER(size, "bs_init_bits", 0); avm_int_t size_val = term_to_int(size); - if (size_val % 8 != 0) { - TRACE("bs_init_bits: size_val (" AVM_INT_FMT ") is not evenly divisible by 8\n", size_val); - RAISE_ERROR(UNSUPPORTED_ATOM); + if (UNLIKELY(size_val < 0)) { + TRACE("bs_init_bits: size_val (" AVM_INT_FMT ") is negative\n", size_val); + RAISE_ERROR(BADARG_ATOM); } if (flags_value != 0) { TRACE("bs_init_bits: neither signed nor native or little endian encoding supported.\n"); @@ -3449,11 +3448,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_init_bits/6, fail=%i size=" AVM_INT_FMT " words=%u live=%u dreg=%c%i\n", fail, size_val, (unsigned) words, (unsigned) live, T_DEST_REG(dreg)); + // A non-byte-aligned total size yields a bitstring + size_t bs_total_bits = (size_t) size_val; + size_t bs_total_bytes = (bs_total_bits + 7) / 8; + size_t bs_trailing_bits = bs_total_bits % 8; + size_t bs_heap_size = words + term_binary_heap_size(bs_total_bytes); + if (bs_trailing_bits != 0) { + bs_heap_size += TERM_BOXED_SUB_BINARY_SIZE; + } TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, words + term_binary_heap_size(size_val / 8), live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, bs_heap_size, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - term t = term_create_empty_binary(size_val / 8, &ctx->heap, ctx->global); + term t = term_create_empty_binary(bs_total_bytes, &ctx->heap, ctx->global); if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } @@ -3461,6 +3468,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) ctx->bs = t; ctx->bs_offset = 0; + if (bs_trailing_bits != 0) { + t = term_alloc_sub_binary_bits(t, 0, bs_total_bits / 8, (uint8_t) bs_trailing_bits, &ctx->heap); + } WRITE_REGISTER(dreg, t); break; } @@ -3485,37 +3495,58 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); - VERIFY_IS_BINARY(src, "bs_append", 0); + VERIFY_IS_BITSTRING(src, "bs_append", 0); VERIFY_IS_INTEGER(size, "bs_append", 0); avm_int_t size_val = term_to_int(size); - if (size_val % 8 != 0) { - TRACE("bs_append: size_val (" AVM_INT_FMT ") is not evenly divisible by 8\n", size_val); - RAISE_ERROR(UNSUPPORTED_ATOM); + if (UNLIKELY(size_val < 0)) { + TRACE("bs_append: size_val (" AVM_INT_FMT ") is negative\n", size_val); + RAISE_ERROR(BADARG_ATOM); } - if (unit != 8) { - TRACE("bs_append: unit is not equal to 8; unit=%u\n", (unsigned) unit); - RAISE_ERROR(UNSUPPORTED_ATOM); + + size_t src_bits = term_bit_size(src); + if (UNLIKELY(unit == 0 || (src_bits % unit) != 0)) { + TRACE("bs_append: source bit size (%zu) is not a multiple of unit %u\n", src_bits, (unsigned) unit); + RAISE_ERROR(BADARG_ATOM); } - size_t src_size = term_binary_size(src); + // As in bs_create_bin, an unrepresentable total is system_limit + if (UNLIKELY(src_bits > SIZE_MAX - (size_t) size_val)) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + size_t bs_total_bits = src_bits + (size_t) size_val; + if (UNLIKELY(bs_total_bits > SIZE_MAX - 7)) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + size_t bs_total_bytes = (bs_total_bits + 7) / 8; + if (UNLIKELY(bs_total_bytes > TERM_MAX_BINARY_SIZE)) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + size_t bs_trailing_bits = bs_total_bits % 8; + size_t bs_heap_size = term_binary_heap_size(bs_total_bytes); + if (bs_trailing_bits != 0) { + bs_heap_size += TERM_BOXED_SUB_BINARY_SIZE; + } TRIM_LIVE_REGS(live); x_regs[live] = src; - if (UNLIKELY(memory_ensure_free_with_roots(ctx, term_binary_heap_size(src_size + size_val / 8), live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, bs_heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } TRACE("bs_append/8, fail=%u size=" AVM_INT_FMT " unit=%u src=0x%" TERM_X_FMT " dreg=%c%i\n", (unsigned) fail, size_val, (unsigned) unit, src, T_DEST_REG(dreg)); src = x_regs[live]; - term t = term_create_empty_binary(src_size + size_val / 8, &ctx->heap, ctx->global); + term t = term_create_empty_binary(bs_total_bytes, &ctx->heap, ctx->global); if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - memcpy((void *) term_binary_data(t), (void *) term_binary_data(src), src_size); + bitstring_copy_bits((uint8_t *) term_binary_data(t), 0, (const uint8_t *) term_binary_data(src), src_bits); ctx->bs = t; - ctx->bs_offset = src_size * 8; + ctx->bs_offset = src_bits; + if (bs_trailing_bits != 0) { + t = term_alloc_sub_binary_bits(t, 0, bs_total_bits / 8, (uint8_t) bs_trailing_bits, &ctx->heap); + } WRITE_REGISTER(dreg, t); break; } @@ -3536,29 +3567,64 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); - VERIFY_IS_BINARY(src, "bs_private_append", 0); + VERIFY_IS_BITSTRING(src, "bs_private_append", 0); VERIFY_IS_INTEGER(size, "bs_private_append", 0); avm_int_t size_val = term_to_int(size); - if (size_val % 8 != 0) { - TRACE("bs_private_append: size_val (%li) is not evenly divisible by 8\n", (long int) size_val); - RAISE_ERROR(UNSUPPORTED_ATOM); + if (UNLIKELY(size_val < 0)) { + TRACE("bs_private_append: size_val (%li) is negative\n", (long int) size_val); + RAISE_ERROR(BADARG_ATOM); } - size_t src_size = term_binary_size(src); - if (UNLIKELY(memory_ensure_free_opt(ctx, term_binary_heap_size(src_size + size_val / 8), MEMORY_NO_GC) != MEMORY_GC_OK)) { + size_t src_bits = term_bit_size(src); + if (UNLIKELY(unit == 0 || (src_bits % unit) != 0)) { + TRACE("bs_private_append: source bit size (%zu) is not a multiple of unit %u\n", src_bits, (unsigned) unit); + RAISE_ERROR(BADARG_ATOM); + } + + // As in bs_create_bin, an unrepresentable total is system_limit + if (UNLIKELY(src_bits > SIZE_MAX - (size_t) size_val)) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + size_t bs_total_bits = src_bits + (size_t) size_val; + if (UNLIKELY(bs_total_bits > SIZE_MAX - 7)) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + size_t bs_total_bytes = (bs_total_bits + 7) / 8; + if (UNLIKELY(bs_total_bytes > TERM_MAX_BINARY_SIZE)) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + size_t bs_trailing_bits = bs_total_bits % 8; + // The binary can only be reused in place when it owns whole bytes + bool bs_can_reuse = (src_bits % 8) == 0; + size_t bs_heap_size = term_binary_heap_size(bs_total_bytes); + if (bs_trailing_bits != 0) { + bs_heap_size += TERM_BOXED_SUB_BINARY_SIZE; + } + if (UNLIKELY(memory_ensure_free_opt(ctx, bs_heap_size, MEMORY_NO_GC) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } DECODE_COMPACT_TERM(src, src_pc) - term t = term_reuse_binary(src, src_size + size_val / 8, &ctx->heap, ctx->global); + term t; + if (bs_can_reuse) { + t = term_reuse_binary(src, bs_total_bytes, &ctx->heap, ctx->global); + } else { + t = term_create_empty_binary(bs_total_bytes, &ctx->heap, ctx->global); + if (LIKELY(!term_is_invalid_term(t))) { + bitstring_copy_bits((uint8_t *) term_binary_data(t), 0, (const uint8_t *) term_binary_data(src), src_bits); + } + } if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } ctx->bs = t; - ctx->bs_offset = src_size * 8; + ctx->bs_offset = src_bits; TRACE("bs_private_append/6, fail=%u size=" AVM_INT_FMT " unit=%u src=0x%" TERM_X_FMT " dreg=%c%i\n", (unsigned) fail, size_val, (unsigned) unit, src, T_DEST_REG(dreg)); + if (bs_trailing_bits != 0) { + t = term_alloc_sub_binary_bits(t, 0, bs_total_bits / 8, (uint8_t) bs_trailing_bits, &ctx->heap); + } WRITE_REGISTER(dreg, t); break; } @@ -3596,6 +3662,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(BADARG_ATOM); } } else { + if (size_value < 0 || (unit != 0 && (size_t) size_value > SIZE_MAX / unit)) { + RAISE_ERROR(BADARG_ATOM); + } + size_t field_bits = (size_t) size_value * (size_t) unit; + // TODO: intn_to_integer_bytes writes whole bytes only, so a + // bignum segment at a bit offset needs a bit-level writer. + if ((ctx->bs_offset % 8) != 0 || (field_bits % 8) != 0) { + TRACE("bs_put_integer: non-byte-aligned big integer segment unsupported\n"); + RAISE_ERROR(UNSUPPORTED_ATOM); + } + const intn_digit_t *big_src_value = NULL; size_t big_len = 0; intn_integer_sign_t big_sign; @@ -3633,43 +3710,44 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term src; DECODE_COMPACT_TERM(src, pc); - VERIFY_IS_BINARY(src, "bs_put_binary", 0); - unsigned long size_val = 0; + VERIFY_IS_BITSTRING(src, "bs_put_binary", 0); + size_t src_bits = term_bit_size(src); + size_t size_bits = 0; if (term_is_integer(size)) { - avm_int_t bit_size = term_to_int(size) * unit; - if (bit_size % 8 != 0) { - TRACE("bs_put_binary: Bit size must be evenly divisible by 8.\n"); - RAISE_ERROR(UNSUPPORTED_ATOM); + avm_int_t signed_size_value = term_to_int(size); + if (UNLIKELY(signed_size_value < 0)) { + TRACE("bs_put_binary: size is negative\n"); + RAISE_ERROR(BADARG_ATOM); } - size_val = bit_size / 8; + if (UNLIKELY(unit != 0 && (size_t) signed_size_value > src_bits / unit)) { + TRACE("bs_put_binary: size larger than the source bitstring\n"); + RAISE_ERROR(BADARG_ATOM); + } + size_bits = (size_t) signed_size_value * unit; } else if (size == ALL_ATOM) { - size_val = term_binary_size(src); + size_bits = src_bits; } else { TRACE("bs_put_binary: Unsupported size term type in put binary: %p\n", (void *) size); RAISE_ERROR(BADARG_ATOM); } - if (size_val > term_binary_size(src)) { - TRACE("bs_put_binary: binary data size (%li) larger than source binary size (%li)\n", (long) size_val, (long) term_binary_size(src)); + if (UNLIKELY(size_bits > src_bits)) { + TRACE("bs_put_binary: binary data size (%zu bits) larger than source binary size (%zu bits)\n", size_bits, src_bits); RAISE_ERROR(BADARG_ATOM); } if (flags_value != 0) { TRACE("bs_put_binary: neither signed nor native or little endian encoding supported.\n"); RAISE_ERROR(UNSUPPORTED_ATOM); } - - if (ctx->bs_offset % 8 != 0) { - TRACE("bs_put_binary: Unsupported bit syntax operation. Writing binaries must be byte-aligend.\n"); - RAISE_ERROR(UNSUPPORTED_ATOM); + size_t bs_capacity_bits = term_binary_size(ctx->bs) * 8; + if (UNLIKELY(ctx->bs_offset > bs_capacity_bits || bs_capacity_bits - ctx->bs_offset < size_bits)) { + TRACE("bs_put_binary: insufficient capacity to write binary\n"); + RAISE_ERROR(BADARG_ATOM); } - TRACE("bs_put_binary/5, fail=%u size=%li unit=%u flags=%x src=0x%x\n", (unsigned) fail, size_val, (unsigned) unit, (int) flags_value, (unsigned int) src); + TRACE("bs_put_binary/5, fail=%u size=%zu bits unit=%u flags=%x src=0x%x\n", (unsigned) fail, size_bits, (unsigned) unit, (int) flags_value, (unsigned int) src); - int result = term_bs_insert_binary(ctx->bs, ctx->bs_offset, src, size_val); - if (UNLIKELY(result)) { - TRACE("bs_put_binary: Failed to insert binary into binary: %i\n", result); - RAISE_ERROR(BADARG_ATOM); - } - ctx->bs_offset += 8 * size_val; + bitstring_copy_bits((uint8_t *) term_binary_data(ctx->bs), ctx->bs_offset, (const uint8_t *) term_binary_data(src), size_bits); + ctx->bs_offset += size_bits; break; } @@ -3740,7 +3818,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) uint32_t offset; DECODE_LITERAL(offset, pc); - if (UNLIKELY(!term_is_binary(ctx->bs))) { + if (UNLIKELY(!term_is_bitstring(ctx->bs))) { TRACE("bs_put_string: Bad state. ctx->bs is not a binary.\n"); RAISE_ERROR(BADARG_ATOM); } @@ -3791,7 +3869,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_INTEGER(src, "bs_put_utf8/3", 0); avm_int_t src_value = term_to_int(src); TRACE("bs_put_utf8/3 flags=%x, src=0x%lx\n", (int) flags, (long) src_value); - if (UNLIKELY(!term_is_binary(ctx->bs))) { + if (UNLIKELY(!term_is_bitstring(ctx->bs))) { TRACE("bs_put_utf8/3: Bad state. ctx->bs is not a binary.\n"); RAISE_ERROR(BADARG_ATOM); } @@ -3839,7 +3917,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_INTEGER(src, "bs_put_utf16/3", 0); avm_int_t src_value = term_to_int(src); TRACE("bs_put_utf16/3 flags=%x, src=" AVM_INT_FMT "\n", (int) flags, src_value); - if (UNLIKELY(!term_is_binary(ctx->bs))) { + if (UNLIKELY(!term_is_bitstring(ctx->bs))) { TRACE("bs_put_utf16: Bad state. ctx->bs is not a binary.\n"); RAISE_ERROR(BADARG_ATOM); } @@ -3868,7 +3946,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_INTEGER(src, "bs_put_utf32/3", 0); avm_int_t src_value = term_to_int(src); TRACE("bs_put_utf32/3 flags=%x, src=" AVM_INT_FMT "\n", (int) flags, src_value); - if (UNLIKELY(!term_is_binary(ctx->bs))) { + if (UNLIKELY(!term_is_bitstring(ctx->bs))) { TRACE("bs_put_utf32/3: Bad state. ctx->bs is not a binary.\n"); RAISE_ERROR(BADARG_ATOM); } @@ -4102,7 +4180,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_DEST_REGISTER_GC_SAFE(dreg, pc); TRACE("bs_start_match3/4, fail=%i src=0x%" TERM_X_FMT " live=%u dreg=%c%i\n", fail, src, live, T_DEST_REG_GC_SAFE(dreg)); - if (!(term_is_binary(src) || term_is_match_state(src))) { + if (!(term_is_bitstring(src) || term_is_match_state(src))) { pc = mod->labels[fail]; } else { // MEMORY_CAN_SHRINK because bs_start_match is classified as gc in beam_ssa_codegen.erl @@ -4155,29 +4233,21 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_get_tail/3 src=0x%" TERM_X_FMT " dreg=%c%i live=%u\n", src, T_DEST_REG_GC_SAFE(dreg), live); if (bs_offset == 0) { - WRITE_REGISTER_GC_SAFE(dreg, bs_bin); - } else { - if (bs_offset % 8 != 0) { - TRACE("bs_get_tail: Unsupported alignment.\n"); - RAISE_ERROR(UNSUPPORTED_ATOM); - } else { - size_t start_pos = bs_offset / 8; - size_t src_size = term_binary_size(bs_bin); - size_t new_bin_size = src_size - start_pos; - size_t heap_size = term_sub_binary_heap_size(bs_bin, src_size - start_pos); - - TRIM_LIVE_REGS(live); - x_regs[live] = src; - if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { - RAISE_ERROR(OUT_OF_MEMORY_ATOM); - } - src = x_regs[live]; - bs_bin = term_get_match_state_binary(src); - term t = term_maybe_create_sub_binary(bs_bin, start_pos, new_bin_size, &ctx->heap, ctx->global); - WRITE_REGISTER_GC_SAFE(dreg, t); + size_t heap_size = bitstring_get_tail_heap_size(bs_bin, bs_offset); + TRIM_LIVE_REGS(live); + x_regs[live] = src; + if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } + src = x_regs[live]; + bs_bin = term_get_match_state_binary(src); + term t = bitstring_get_tail(bs_bin, bs_offset, &ctx->heap, ctx->global); + if (UNLIKELY(term_is_invalid_term(t))) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); } + WRITE_REGISTER_GC_SAFE(dreg, t); } break; } @@ -4221,7 +4291,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(BADARG_ATOM); } - if (term_binary_size(bs_bin) * 8 - bs_offset < MINI(remaining * 8, bits)) { + if (term_bit_size(bs_bin) - bs_offset < MINI(remaining * 8, bits)) { TRACE("bs_match_string: failed to match (binary is shorter)\n"); JUMP_TO_ADDRESS(mod->labels[fail]); } else { @@ -4288,14 +4358,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_MATCH_STATE(src, "bs_skip_bits2", 0); VERIFY_IS_ANY_INTEGER(size, "bs_skip_bits2", 0); // Ignore flags value as skipping bits is the same whatever the endianness - TRACE("bs_skip_bits2/5, fail=%u src=%p unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned) unit, (int) flags_value); + TRACE("bs_skip_bits2/5, fail=%u src=%p size=0x%lx unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned long) term_to_int(size), (unsigned) unit, (int) flags_value); avm_int_t bs_offset = term_get_match_state_offset(src); term bs_bin = term_get_match_state_binary(src); - size_t bs_capacity = term_binary_size(bs_bin) * 8; + size_t bs_capacity = term_bit_size(bs_bin); size_t increment; if ((size_t) bs_offset > bs_capacity - || !bs_scaled_size(size, unit, bs_capacity - bs_offset, &increment)) { + || !bs_scaled_size_bits(size, unit, bs_capacity - bs_offset, &increment)) { TRACE("bs_skip_bits2: Insufficient capacity to skip bits: %lu\n", (unsigned long) bs_offset); JUMP_TO_ADDRESS(mod->labels[fail]); } else { @@ -4320,7 +4390,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); - if ((term_binary_size(bs_bin) * 8 - bs_offset) % unit != 0) { + if ((term_bit_size(bs_bin) - bs_offset) % unit != 0) { TRACE("bs_test_unit: Available bits in source not evenly divisible by unit\n"); JUMP_TO_ADDRESS(mod->labels[fail]); } @@ -4344,8 +4414,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); - if ((term_binary_size(bs_bin) * 8 - bs_offset) != (unsigned int) bits) { - TRACE("bs_test_tail2: Expected exactly %u bits remaining, but remaining=%u\n", (unsigned) bits, (unsigned) (term_binary_size(bs_bin) * 8 - bs_offset)); + if ((term_bit_size(bs_bin) - bs_offset) != (unsigned int) bits) { + TRACE("bs_test_tail2: Expected exactly %u bits remaining, but remaining=%u\n", (unsigned) bits, (unsigned) (term_bit_size(bs_bin) - bs_offset)); JUMP_TO_ADDRESS(mod->labels[fail]); } break; @@ -4370,15 +4440,15 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) VERIFY_IS_MATCH_STATE(src, "bs_get_integer", 0); VERIFY_IS_ANY_INTEGER(size, "bs_get_integer", 0); - TRACE("bs_get_integer2/7, fail=%u src=%p live=%u unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned) live, (unsigned) unit, (int) flags_value); + TRACE("bs_get_integer2/7, fail=%u src=%p live=%u size=%u unit=%u flags=%x\n", (unsigned) fail, (void *) src, (unsigned) term_to_int(size), (unsigned) live, (unsigned) unit, (int) flags_value); union maybe_unsigned_int64 value; term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); - size_t bs_capacity = term_binary_size(bs_bin) * 8; + size_t bs_capacity = term_bit_size(bs_bin); size_t increment_bits; if ((size_t) bs_offset > bs_capacity - || !bs_scaled_size(size, unit, bs_capacity - bs_offset, &increment_bits)) { + || !bs_scaled_size_bits(size, unit, bs_capacity - bs_offset, &increment_bits)) { TRACE("bs_get_integer2: size is negative or exceeds the remaining capacity\n"); JUMP_TO_ADDRESS(mod->labels[fail]); } @@ -4407,8 +4477,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } } } else if ((bs_offset % 8 == 0) && (increment % 8 == 0) && (increment <= INTN_MAX_UNSIGNED_BITS_SIZE)) { - unsigned long capacity = term_binary_size(bs_bin); - if (8 * capacity - bs_offset < (unsigned long) increment) { + unsigned long capacity_bits = term_bit_size(bs_bin); + if (capacity_bits - bs_offset < (unsigned long) increment) { JUMP_TO_ADDRESS(mod->labels[fail]); } size_t byte_offset = bs_offset / 8; @@ -4421,7 +4491,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) HANDLE_ERROR(); } } else { - JUMP_TO_ADDRESS(mod->labels[fail]); + // As the bs_match integer command and the JIT do: a >64-bit + // field that is not byte-aligned is unsupported, not a nomatch. + RAISE_ERROR(UNSUPPORTED_ATOM); } DEST_REGISTER(dreg); @@ -4453,10 +4525,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) avm_float_t value; term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); - size_t bs_capacity = term_binary_size(bs_bin) * 8; + size_t bs_capacity = term_bit_size(bs_bin); size_t increment_bits; if ((size_t) bs_offset > bs_capacity - || !bs_scaled_size(size, unit, bs_capacity - bs_offset, &increment_bits)) { + || !bs_scaled_size_bits(size, unit, bs_capacity - bs_offset, &increment_bits)) { TRACE("bs_get_float2: size is negative or exceeds the remaining capacity\n"); JUMP_TO_ADDRESS(mod->labels[fail]); } @@ -4515,54 +4587,53 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(flags_value, pc) VERIFY_IS_MATCH_STATE(src, "bs_get_binary2", 0); + // a segment unit is 1..256 in the language, so it never divides by zero + assert(unit != 0); term bs_bin = term_get_match_state_binary(src); avm_int_t bs_offset = term_get_match_state_offset(src); - if (unit != 8) { - TRACE("bs_get_binary2: Unsupported: unit must be 8.\n"); + if (flags_value != 0) { + TRACE("bs_get_binary2: neither signed nor native or little endian encoding supported.\n"); RAISE_ERROR(UNSUPPORTED_ATOM); } - size_t bs_capacity = term_binary_size(bs_bin); - if ((size_t) bs_offset / 8 > bs_capacity) { - TRACE("bs_get_binary2: match state offset is past the end of the binary\n"); + + size_t bs_capacity = term_bit_size(bs_bin); + if ((size_t) bs_offset > bs_capacity) { + TRACE("bs_get_binary2: match state offset is past the end of the bitstring\n"); JUMP_TO_ADDRESS(mod->labels[fail]); } - size_t remaining_bytes = bs_capacity - bs_offset / 8; - size_t size_val = 0; + size_t remaining_bits = bs_capacity - bs_offset; + size_t size_bits; if (term_is_any_integer(size)) { - // A negative or oversized size fails the match, as on BEAM - if (!bs_scaled_size(size, 1, remaining_bytes, &size_val)) { + // A negative or overflowing size fails the match + if (!bs_scaled_size_bits(size, unit, remaining_bits, &size_bits)) { TRACE("bs_get_binary2: size is negative or exceeds the remaining capacity\n"); JUMP_TO_ADDRESS(mod->labels[fail]); } } else if (size == ALL_ATOM) { - size_val = remaining_bytes; + if (remaining_bits % unit != 0) { + TRACE("bs_get_binary2: remainder is not a multiple of unit\n"); + JUMP_TO_ADDRESS(mod->labels[fail]); + } + size_bits = remaining_bits; } else { TRACE("bs_get_binary2: size is neither an integer nor the atom `all`\n"); RAISE_ERROR(BADARG_ATOM); } - if (bs_offset % unit != 0) { - TRACE("bs_get_binary2: Unsupported. Offset on binary read must be aligned on byte boundaries.\n"); - RAISE_ERROR(BADARG_ATOM); - } - if (flags_value != 0) { - TRACE("bs_get_binary2: neither signed nor native or little endian encoding supported.\n"); - RAISE_ERROR(UNSUPPORTED_ATOM); - } TRACE("bs_get_binary2/7, fail=%u src=%p live=%u unit=%u\n", (unsigned) fail, (void *) bs_bin, (unsigned) live, (unsigned) unit); - if (size_val > remaining_bytes) { - TRACE("bs_get_binary2: insufficient capacity -- bs_offset = %d, size_val = %zu\n", (int) bs_offset, size_val); + if (size_bits > remaining_bits) { + TRACE("bs_get_binary2: insufficient capacity -- bs_offset = %d, size_bits = %d\n", (int) bs_offset, (int) size_bits); JUMP_TO_ADDRESS(mod->labels[fail]); } else { - term_set_match_state_offset(src, bs_offset + size_val * unit); + term_set_match_state_offset(src, bs_offset + size_bits); TRIM_LIVE_REGS(live); // there is always room for a MAX_REG + 1 register, used as working register x_regs[live] = bs_bin; - size_t heap_size = term_sub_binary_heap_size(bs_bin, size_val); + size_t heap_size = bitstring_slice_heap_size(bs_bin, bs_offset, size_bits); if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } @@ -4572,7 +4643,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) bs_bin = x_regs[live]; - term t = term_maybe_create_sub_binary(bs_bin, bs_offset / unit, size_val, &ctx->heap, ctx->global); + term t = bitstring_slice(bs_bin, bs_offset, size_bits, &ctx->heap, ctx->global); + if (UNLIKELY(term_is_invalid_term(t))) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } WRITE_REGISTER(dreg, t); } break; @@ -4813,7 +4887,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_bitstr/2, label=%i, arg1=%" TERM_X_FMT "\n", label, arg1); - if (!term_is_binary(arg1)) { + if (!term_is_bitstring(arg1)) { pc = mod->labels[label]; } @@ -5487,10 +5561,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // no_fail: we know it's a binary or a match_state // resume: we know it's a match_state - if (term_is_invalid_term(fail_atom) && !(term_is_binary(src) || term_is_match_state(src))) { + if (term_is_invalid_term(fail_atom) && !(term_is_bitstring(src) || term_is_match_state(src))) { pc = mod->labels[fail_label]; } else { - assert(term_is_binary(src) || term_is_match_state(src)); + assert(term_is_bitstring(src) || term_is_match_state(src)); TRIM_LIVE_REGS(live); x_regs[live] = src; @@ -5700,14 +5774,25 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case APPEND_ATOM: case BINARY_ATOM: case PRIVATE_APPEND_ATOM: { - VERIFY_IS_BINARY(src, "bs_create_bin/6", fail); + VERIFY_IS_BITSTRING(src, "bs_create_bin/6", fail); if (size == ALL_ATOM) { - // We only support src as a binary of bytes here. - segment_size = term_binary_size(src); - segment_unit = 8; + size_t src_bits = term_bit_size(src); + if (UNLIKELY(segment_unit == 0 || (src_bits % segment_unit) != 0)) { + if (fail == 0) { + RAISE_ERROR(BADARG_ATOM); + } else { + JUMP_TO_LABEL(mod, fail); + } + } if (atom_type == PRIVATE_APPEND_ATOM && j == 0) { + // The accumulator may be a bitstring: its + // storage is reused as a whole, so its + // trailing bits are kept (see + // term_reuse_binary). reuse_binary = true; } + segment_size = src_bits; + segment_unit = 1; } else { VERIFY_IS_INTEGER(size, "bs_create_bin/6", fail); avm_int_t signed_size_value = term_to_int(size); @@ -5718,14 +5803,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) JUMP_TO_LABEL(mod, fail); } } - avm_int_t size_in_bits = signed_size_value * segment_unit; - if (size_in_bits % 8) { - TRACE("bs_create_bin/6: size in bits (%d) is not evenly divisible by 8\n", (int) size_in_bits); - RAISE_ERROR(UNSUPPORTED_ATOM); - } - avm_int_t size_in_bytes = size_in_bits / 8; - size_t binary_size = term_binary_size(src); - if ((size_t) size_in_bytes > binary_size) { + size_t src_bits = term_bit_size(src); + if (segment_unit != 0 + && (size_t) signed_size_value > src_bits / (size_t) segment_unit) { if (fail == 0) { RAISE_ERROR(BADARG_ATOM); } else { @@ -5741,20 +5821,38 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(UNSUPPORTED_ATOM); } } - binary_size += segment_unit * segment_size; + if (segment_unit != 0 + && (size_t) segment_size > SIZE_MAX / (size_t) segment_unit) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + size_t segment_bits = (size_t) segment_unit * (size_t) segment_size; + if (binary_size > SIZE_MAX - segment_bits) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + binary_size += segment_bits; } - // Allocate and build binary in second iteration - if (binary_size % 8) { - TRACE("bs_create_bin/6: total binary size (%d) is not evenly divisible by 8\n", (int) binary_size); - RAISE_ERROR(UNSUPPORTED_ATOM); + // Allocate and build binary in second iteration. A non-byte-aligned + // total size yields a bitstring: the whole bytes are stored in a heap + // binary and wrapped in a sub-binary carrying the trailing bit count. + size_t trailing_bits = binary_size % 8; + if (binary_size > SIZE_MAX - 7) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); + } + size_t binary_bytes = (binary_size + 7) / 8; + if (UNLIKELY(binary_bytes > TERM_MAX_BINARY_SIZE)) { + RAISE_ERROR(SYSTEM_LIMIT_ATOM); } TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, alloc + term_binary_heap_size(binary_size / 8), live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + size_t bs_heap_size = alloc + term_binary_heap_size(binary_bytes); + if (trailing_bits != 0) { + bs_heap_size += TERM_BOXED_SUB_BINARY_SIZE; + } + if (UNLIKELY(memory_ensure_free_with_roots(ctx, bs_heap_size, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } term t; if (!reuse_binary) { - t = term_create_empty_binary(binary_size / 8, &ctx->heap, ctx->global); + t = term_create_empty_binary(binary_bytes, &ctx->heap, ctx->global); if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } @@ -5862,6 +5960,11 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) "binary\n"); RAISE_ERROR(BADARG_ATOM); } + // TODO: intn_to_integer_bytes writes whole bytes only, so a + // bignum segment at a bit offset needs a bit-level writer. + } else if ((offset % 8 != 0) || ((size_value * segment_unit) % 8 != 0)) { + TRACE("bs_create_bin/6: non-byte-aligned big integer segment unsupported\n"); + RAISE_ERROR(UNSUPPORTED_ATOM); } else { // when building a binary, `signed` flag is implicit intn_from_integer_options_t intn_flags @@ -5931,21 +6034,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case APPEND_ATOM: case BINARY_ATOM: case PRIVATE_APPEND_ATOM: { - if (offset % 8) { - TRACE("bs_create_bin/6: current offset (%d) is not evenly divisible by 8\n", (int) offset); - RAISE_ERROR(UNSUPPORTED_ATOM); - } - size_t src_size = term_binary_size(src); + size_t src_bits = term_bit_size(src); if (reuse_binary && j == 0) { - t = term_reuse_binary(src, binary_size / 8, &ctx->heap, ctx->global); + // src_bits is the exact bit count: a bitstring + // accumulator contributes its trailing bits too + t = term_reuse_binary(src, binary_bytes, &ctx->heap, ctx->global); if (UNLIKELY(term_is_invalid_term(t))) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - segment_size = src_size * 8; + segment_size = src_bits; break; } - uint8_t *dst = (uint8_t *) term_binary_data(t) + (offset / 8); + uint8_t *dst = (uint8_t *) term_binary_data(t); const uint8_t *bin = (const uint8_t *) term_binary_data(src); + size_t copy_bits = src_bits; if (size != ALL_ATOM) { VERIFY_IS_INTEGER(size, "bs_create_bin/6", fail); avm_int_t signed_size_value = term_to_int(size); @@ -5953,19 +6055,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_create_bin/6: size value less than 0: %i\n", (int) signed_size_value); RAISE_ERROR(BADARG_ATOM); } - // We checked earlier it's a multiple of 8 - size_value = ((size_t) signed_size_value) * segment_unit / 8; - if (size_value > src_size) { + copy_bits = ((size_t) signed_size_value) * segment_unit; + if (copy_bits > src_bits) { if (fail == 0) { RAISE_ERROR(BADARG_ATOM); } else { JUMP_TO_LABEL(mod, fail); } } - src_size = size_value; } - memcpy(dst, bin, src_size); - segment_size = src_size * 8; + bitstring_copy_bits(dst, offset, bin, copy_bits); + segment_size = copy_bits; break; } default: @@ -5973,6 +6073,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } offset += segment_size; } + if (trailing_bits != 0) { + t = term_alloc_sub_binary_bits(t, 0, binary_size / 8, (uint8_t) trailing_bits, &ctx->heap); + } WRITE_REGISTER_GC_SAFE(dreg, t); break; } @@ -6099,14 +6202,13 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) int unit; DECODE_LITERAL(unit, pc); j++; - size_t bs_bin_size = term_binary_size(bs_bin); if (UNLIKELY(stride < 0)) { RAISE_ERROR(BADARG_ATOM); } size_t unsigned_stride = (size_t) stride; - size_t remaining = (bs_bin_size * 8) - bs_offset; + size_t remaining = term_bit_size(bs_bin) - bs_offset; if (remaining < unsigned_stride || (remaining - unsigned_stride) % unit != 0) { - TRACE("bs_match/3: ensure_at_least failed -- bs_bin_size = %d, bs_offset = %d, stride = %d, unit = %d\n", (int) bs_bin_size, (int) bs_offset, (int) stride, (int) unit); + TRACE("bs_match/3: ensure_at_least failed -- bs_offset = %d, stride = %d, unit = %d\n", (int) bs_offset, (int) stride, (int) unit); goto bs_match_jump_to_fail; } break; @@ -6120,9 +6222,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(BADARG_ATOM); } size_t unsigned_stride = (size_t) stride; - size_t bs_bin_size = term_binary_size(bs_bin); - if ((bs_bin_size * 8) - bs_offset != unsigned_stride) { - TRACE("bs_match/3: ensure_exactly failed -- bs_bin_size = %lu, bs_offset = %lu, stride = %lu\n", (unsigned long) bs_bin_size, (unsigned long) bs_offset, (unsigned long) stride); + if (term_bit_size(bs_bin) - bs_offset != unsigned_stride) { + TRACE("bs_match/3: ensure_exactly failed -- bs_offset = %lu, stride = %lu\n", (unsigned long) bs_offset, (unsigned long) stride); goto bs_match_jump_to_fail; } break; @@ -6137,16 +6238,15 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) j++; avm_int_t flags_value; DECODE_FLAGS_LIST(flags_value, flags, opcode) - term size; - DECODE_COMPACT_TERM(size, pc); + // The size of a bs_match integer command is always a literal + int size; + DECODE_LITERAL(size, pc); j++; int unit; DECODE_LITERAL(unit, pc); j++; // context_clean_registers(ctx, live); // TODO: check if needed - VERIFY_IS_INTEGER(size, "bs_match/3", fail); - avm_int_t size_val = term_to_int(size); - avm_int_t increment = size_val * unit; + avm_int_t increment = (avm_int_t) size * unit; union maybe_unsigned_int64 value; term t; if (increment <= 64) { @@ -6181,7 +6281,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) HANDLE_ERROR(); } } else { - goto bs_match_jump_to_fail; + RAISE_ERROR(UNSUPPORTED_ATOM); } DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); @@ -6208,15 +6308,11 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(unit, pc); j++; int matched_bits = size * unit; - if (bs_offset % 8 != 0 || matched_bits % 8 != 0) { - TRACE("bs_match/3: Unsupported. Offset on binary read must be aligned on byte boundaries.\n"); - RAISE_ERROR(BADARG_ATOM); - } - if ((bs_offset + matched_bits) > term_binary_size(bs_bin) * 8) { + if ((bs_offset + matched_bits) > term_bit_size(bs_bin)) { TRACE("bs_match/3: insufficient capacity\n"); goto bs_match_jump_to_fail; } - size_t heap_size = term_sub_binary_heap_size(bs_bin, matched_bits / 8); + size_t heap_size = bitstring_slice_heap_size(bs_bin, bs_offset, matched_bits); TRIM_LIVE_REGS(live); x_regs[live] = match_state; if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { @@ -6224,7 +6320,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } match_state = x_regs[live]; bs_bin = term_get_match_state_binary(match_state); - term t = term_maybe_create_sub_binary(bs_bin, bs_offset / 8, matched_bits / 8, &ctx->heap, ctx->global); + term t = bitstring_slice(bs_bin, bs_offset, matched_bits, &ctx->heap, ctx->global); + if (UNLIKELY(term_is_invalid_term(t))) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); j++; @@ -6240,15 +6339,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) int unit; DECODE_LITERAL(unit, pc); j++; - // TODO: rewrite this bit once bitstrings are supported - if (bs_offset % 8 != 0) { - TRACE("bs_match/3: Unsupported. Offset on binary read must be aligned on byte boundaries.\n"); - RAISE_ERROR(BADARG_ATOM); - } - size_t total_bytes = term_binary_size(bs_bin); - size_t bs_offset_bytes = bs_offset / 8; - size_t tail_bytes = total_bytes - bs_offset_bytes; - size_t heap_size = term_sub_binary_heap_size(bs_bin, tail_bytes); + size_t heap_size = bitstring_get_tail_heap_size(bs_bin, bs_offset); TRIM_LIVE_REGS(live); x_regs[live] = match_state; if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_size, live + 1, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { @@ -6256,7 +6347,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } match_state = x_regs[live]; bs_bin = term_get_match_state_binary(match_state); - term t = term_maybe_create_sub_binary(bs_bin, bs_offset_bytes, tail_bytes, &ctx->heap, ctx->global); + term t = bitstring_get_tail(bs_bin, bs_offset, &ctx->heap, ctx->global); + if (UNLIKELY(term_is_invalid_term(t))) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } DEST_REGISTER(dreg); DECODE_DEST_REGISTER(dreg, pc); j++; diff --git a/src/libAtomVM/term.c b/src/libAtomVM/term.c index 03a1cadf47..601a09aca5 100644 --- a/src/libAtomVM/term.c +++ b/src/libAtomVM/term.c @@ -30,6 +30,7 @@ #include "tempstack.h" #include "utils.h" +#include #include #include #include @@ -346,11 +347,12 @@ int term_funprint(PrinterFun *fun, term t, const GlobalContext *global) ret += printed; return ret; - } else if (term_is_binary(t)) { + } else if (term_is_bitstring(t)) { int len = term_binary_size(t); const unsigned char *binary_data = (const unsigned char *) term_binary_data(t); + uint8_t trailing_bits = (uint8_t) (term_bit_size(t) % 8); - int is_printable = 1; + int is_printable = (trailing_bits == 0); for (int i = 0; i < len; i++) { if (!isprint(binary_data[i])) { is_printable = 0; @@ -391,6 +393,15 @@ int term_funprint(PrinterFun *fun, term t, const GlobalContext *global) ret += printed; } } + if (trailing_bits != 0) { + uint8_t trailing_value = (uint8_t) (binary_data[len] >> (8 - trailing_bits)); + int printed = fun->print(fun, "%s%u:%u", len != 0 ? "," : "", + (unsigned int) trailing_value, (unsigned int) trailing_bits); + if (UNLIKELY(printed < 0)) { + return printed; + } + ret += printed; + } int printed = fun->print(fun, ">>"); if (UNLIKELY(printed < 0)) { return printed; @@ -998,25 +1009,36 @@ TermCompareResult term_compare(term t, term other, TermCompareOpts opts, GlobalC break; } case TERM_TYPE_INDEX_BINARY: { - int t_size = term_binary_size(t); - int other_size = term_binary_size(other); + size_t t_bits = term_bit_size(t); + size_t other_bits = term_bit_size(other); - const char *t_data = term_binary_data(t); - const char *other_data = term_binary_data(other); + const unsigned char *t_data = (const unsigned char *) term_binary_data(t); + const unsigned char *other_data = (const unsigned char *) term_binary_data(other); - int cmp_size = (t_size > other_size) ? other_size : t_size; + size_t cmp_bits = (t_bits > other_bits) ? other_bits : t_bits; + size_t cmp_bytes = cmp_bits / 8; - int memcmp_result = memcmp(t_data, other_data, cmp_size); - if (memcmp_result == 0) { - if (t_size == other_size) { - CMP_POP_AND_CONTINUE(); - break; - } else { - result = (t_size > other_size) ? TermGreaterThan : TermLessThan; + int memcmp_result = memcmp(t_data, other_data, cmp_bytes); + if (memcmp_result != 0) { + result = (memcmp_result > 0) ? TermGreaterThan : TermLessThan; + goto unequal; + } + // Compare the trailing partial byte over its valid bits only. + size_t rem_bits = cmp_bits % 8; + if (rem_bits != 0) { + unsigned char mask = (unsigned char) (0xFF << (8 - rem_bits)); + unsigned char tb = t_data[cmp_bytes] & mask; + unsigned char ob = other_data[cmp_bytes] & mask; + if (tb != ob) { + result = (tb > ob) ? TermGreaterThan : TermLessThan; goto unequal; } + } + if (t_bits == other_bits) { + CMP_POP_AND_CONTINUE(); + break; } else { - result = (memcmp_result > 0) ? TermGreaterThan : TermLessThan; + result = (t_bits > other_bits) ? TermGreaterThan : TermLessThan; goto unequal; } } @@ -1226,6 +1248,9 @@ void term_get_function_mfa(term fun, term *m, term *f, term *a) term term_alloc_refc_binary(size_t size, bool is_const, Heap *heap, GlobalContext *glb) { + if (UNLIKELY(size > TERM_MAX_BINARY_SIZE)) { + return term_invalid_term(); + } term *boxed_value = memory_heap_alloc(heap, TERM_BOXED_REFC_BINARY_SIZE); boxed_value[0] = ((TERM_BOXED_REFC_BINARY_SIZE - 1) << 6) | TERM_BOXED_REFC_BINARY; boxed_value[1] = (term) size; @@ -1253,6 +1278,11 @@ term term_alloc_refc_binary(size_t size, bool is_const, Heap *heap, GlobalContex term term_reuse_binary(term src, size_t size, Heap *heap, GlobalContext *glb) { + if (term_is_sub_binary(src)) { + // OTP compiler only emits private_append for bitstrings with offset at 0 + assert(term_get_sub_binary_offset(src) == 0); + src = term_get_sub_binary_ref(src); + } if (term_is_refc_binary(src) && !term_refc_binary_is_const(src)) { term *boxed_value = term_to_term_ptr(src); struct RefcBinary *old_refc = (struct RefcBinary *) boxed_value[3]; @@ -1310,6 +1340,9 @@ term term_reuse_binary(term src, size_t size, Heap *heap, GlobalContext *glb) // Not a refc binary or it's a const refc binary - create a new one size_t src_size = term_binary_size(src); term t = term_create_empty_binary(size, heap, glb); + if (UNLIKELY(term_is_invalid_term(t))) { + return t; + } // Copy the source data (up to the smaller of src_size and size) size_t copy_size = src_size < size ? src_size : size; memcpy((void *) term_binary_data(t), (void *) term_binary_data(src), copy_size); @@ -1330,13 +1363,26 @@ static term find_binary(term binary_or_state) } term term_alloc_sub_binary(term binary_or_state, size_t offset, size_t len, Heap *heap) +{ + return term_alloc_sub_binary_bits(binary_or_state, offset, len, 0, heap); +} + +// boxed[2] packs the byte offset and the trailing bit count in a single word. +// Binaries are capped at TERM_MAX_BINARY_SIZE bytes so that no valid offset can +// be truncated by the shift, and so that term_bit_size cannot wrap. +_Static_assert(((TERM_MAX_BINARY_SIZE << 3) | 0x7) >> 3 == TERM_MAX_BINARY_SIZE, + "the largest binary offset must round-trip through the packed encoding"); +_Static_assert(TERM_MAX_BINARY_SIZE <= (SIZE_MAX - 7) / 8, + "term_bit_size must not overflow for the largest binary"); + +term term_alloc_sub_binary_bits(term binary_or_state, size_t offset, size_t len, uint8_t trailing_bits, Heap *heap) { term *boxed = memory_heap_alloc(heap, TERM_BOXED_SUB_BINARY_SIZE); term binary = find_binary(binary_or_state); boxed[0] = ((TERM_BOXED_SUB_BINARY_SIZE - 1) << 6) | TERM_BOXED_SUB_BINARY; boxed[1] = (term) len; - boxed[2] = (term) offset; + boxed[2] = (term) ((offset << 3) | trailing_bits); boxed[3] = binary; return ((term) boxed) | TERM_PRIMARY_BOXED; diff --git a/src/libAtomVM/term.h b/src/libAtomVM/term.h index 46038f6a1e..67dd5de2d8 100644 --- a/src/libAtomVM/term.h +++ b/src/libAtomVM/term.h @@ -143,6 +143,18 @@ extern "C" { #error #endif +/** + * @brief Largest representable binary size, in bytes. + * + * @details A sub-binary packs its byte offset and its trailing bit count into a + * single word as `(offset << 3) | trailing_bits`, and `term_bit_size` computes + * `byte_size * 8 + trailing_bits`. Both require byte sizes and offsets to fit in + * a word once multiplied by 8. Binaries are therefore capped at `SIZE_MAX >> 3` + * bytes: unreachable on 64-bit, slightly under 512 MiB on 32-bit. Larger sizes + * are rejected with `system_limit`, as OTP does for unrepresentable binaries. + */ +#define TERM_MAX_BINARY_SIZE (SIZE_MAX >> 3) + #define TERM_MAX_LOCAL_PROCESS_ID ((1 << 28) - 1) #define BINARY_HEADER_SIZE 2 @@ -377,6 +389,22 @@ term term_alloc_refc_binary(size_t size, bool is_const, Heap *heap, GlobalContex */ term term_alloc_sub_binary(term binary, size_t offset, size_t len, Heap *heap); +/** + * @brief Allocate a sub-binary that may be non-byte-aligned (a bitstring). + * + * @details Same as term_alloc_sub_binary but carries a trailing partial-byte + * bit count (0..7). When trailing_bits is 0 the result is an ordinary + * byte-aligned binary; when non-zero the total bit size is len*8 + trailing_bits + * and the last referenced byte is only partially valid. + * @param binary the referenced binary + * @param offset the offset (in bytes) into the referenced binary + * @param len the number of whole bytes of the sub-binary + * @param trailing_bits number of valid bits (0..7) in the byte following the whole bytes + * @param heap the heap to allocate the binary in + * @return a term (reference) pointing to the newly allocated sub-binary. + */ +term term_alloc_sub_binary_bits(term binary, size_t offset, size_t len, uint8_t trailing_bits, Heap *heap); + /** * @brief Gets a pointer to a term stored on the heap * @@ -507,10 +535,41 @@ static inline size_t term_boxed_size(term t) return term_get_size_from_boxed_header(*boxed_value); } +/** + * @brief Checks if a term is a bitstring + * + * @details Returns \c true if a term is a bitstring (a binary or a + * sub-binary carrying trailing bits), otherwise \c false. This is the + * predicate behind the is_bitstring/1 BIF. + * @param t the term that will be checked. + * @return \c true if check succeeds, \c false otherwise. + */ +static inline bool term_is_bitstring(term t) +{ + /* boxed: 10 */ + if ((t & TERM_PRIMARY_MASK) == TERM_PRIMARY_BOXED) { + const term *boxed_value = term_to_const_term_ptr(t); + int masked_value = boxed_value[0] & TERM_BOXED_TAG_MASK; + switch (masked_value) { + case TERM_BOXED_REFC_BINARY: + case TERM_BOXED_HEAP_BINARY: + case TERM_BOXED_SUB_BINARY: + return true; + default: + return false; + } + } + + return false; +} + /** * @brief Checks if a term is a binary * - * @details Returns \c true if a term is a binary stored on the heap, otherwise \c false. + * @details Returns \c true if a term is a byte-aligned binary, otherwise + * \c false. A sub-binary carrying trailing bits (a non-byte-aligned + * bitstring) is not a binary; use \c term_is_bitstring to accept it. This + * is the predicate behind the is_binary/1 BIF. * @param t the term that will be checked. * @return \c true if check succeeds, \c false otherwise. */ @@ -523,8 +582,10 @@ static inline bool term_is_binary(term t) switch (masked_value) { case TERM_BOXED_REFC_BINARY: case TERM_BOXED_HEAP_BINARY: - case TERM_BOXED_SUB_BINARY: return true; + case TERM_BOXED_SUB_BINARY: + // boxed_value[2] packs (byte_offset << 3) | trailing_bits + return (boxed_value[2] & 0x7) == 0; default: return false; } @@ -1949,7 +2010,7 @@ static inline size_t term_binary_heap_size(size_t size) */ static inline unsigned long term_binary_size(term t) { - TERM_DEBUG_ASSERT(term_is_binary(t)); + TERM_DEBUG_ASSERT(term_is_bitstring(t)); const term *boxed_value = term_to_const_term_ptr(t); return boxed_value[1]; @@ -1992,7 +2053,7 @@ static inline struct RefcBinary *term_resource_refc_binary_ptr(term resource) */ static inline const char *term_binary_data(term t) { - TERM_DEBUG_ASSERT(term_is_binary(t)); + TERM_DEBUG_ASSERT(term_is_bitstring(t)); const term *boxed_value = term_to_const_term_ptr(t); if (term_is_refc_binary(t)) { @@ -2006,7 +2067,8 @@ static inline const char *term_binary_data(term t) } } if (term_is_sub_binary(t)) { - return term_binary_data(boxed_value[3]) + boxed_value[2]; // offset + // boxed_value[2] packs (byte_offset << 3) | trailing_bits + return term_binary_data(boxed_value[3]) + (boxed_value[2] >> 3); } return (const char *) (boxed_value + 2); } @@ -2093,7 +2155,7 @@ static inline term term_maybe_create_sub_binary(term binary, size_t offset, size return term_alloc_sub_binary(binary, offset, len, heap); } else if (term_is_sub_binary(binary) && len >= SUB_BINARY_MIN) { const term *boxed_value = term_to_const_term_ptr(binary); - return term_alloc_sub_binary(boxed_value[3], boxed_value[2] + offset, len, heap); + return term_alloc_sub_binary(boxed_value[3], (boxed_value[2] >> 3) + offset, len, heap); } else { const char *data = term_binary_data(binary); return term_from_literal_binary(data + offset, len, heap, glb); @@ -2138,12 +2200,16 @@ static inline term term_create_empty_binary(size_t size, Heap *heap, GlobalConte * @brief Reuse a binary. If the binary is a refc binary with a ref count of * 1, try to reuse it. Otherwise, create a new binary and copy the data. * - * @details Try to reuse a binary and return a term pointing to it. - * @param src binary to reuse. - * @param size size of binary data buffer. + * @details Try to reuse a binary and return a term pointing to it. If src is a + * bitstring, its storage is reused as a whole, trailing bits included: src must + * then be a sub-binary at offset 0 over its own storage, which is what a + * private_append accumulator always is. + * @param src binary or bitstring to reuse. + * @param size size of binary data buffer, in bytes. * @param heap the heap to allocate memory in * @param glb the global context as refc binaries are global - * @return a term pointing to the boxed binary pointer. + * @return a term pointing to the boxed binary pointer, or `term_invalid_term()` + * if the buffer could not be allocated. */ term term_reuse_binary(term src, size_t size, Heap *heap, GlobalContext *glb); @@ -2174,47 +2240,6 @@ static inline BinaryPosLen term_nomatch_binary_pos_len(void) return (BinaryPosLen){ .pos = -1, .len = -1 }; } -/** - * @brief Insert an binary into a binary (using bit syntax). - * - * @details Insert the data from the input binary, starting - * at the bit position starting in offset. - * @param t a term pointing to binary data. Fails if t is not a binary term. - * @param offset the bitwise offset in t at which to start writing the integer value - * @param src binary source to insert binary data into. - * @param n the number of low-order bits from value to write. - * @return 0 on success; non-zero value if: - * t is not a binary term - * n is greater than the number of bits in an integer - * there is insufficient capacity in the binary to write these bits - * In general, none of these conditions should apply, if this function is being - * called in the context of generated bit syntax instructions. - */ -static inline int term_bs_insert_binary(term t, int offset, term src, int n) -{ - if (!term_is_binary(t)) { - fprintf(stderr, "Target is not a binary\n"); - return -1; - } - if (!term_is_binary(src)) { - fprintf(stderr, "Source is not a binary\n"); - return -2; - } - if (offset % 8 != 0) { - fprintf(stderr, "Offset not aligned on a byte boundary\n"); - return -3; - } - unsigned long capacity = term_binary_size(t); - if (capacity < (unsigned long) (offset / 8 + n)) { - fprintf(stderr, "Insufficient capacity to write binary\n"); - return -4; - } - uint8_t *dst_pos = (uint8_t *) term_binary_data(t) + offset / 8; - uint8_t *src_pos = (uint8_t *) term_binary_data(src); - memcpy(dst_pos, src_pos, n); - return 0; -} - /** * @brief Get a ref term from ref ticks * @@ -3062,6 +3087,40 @@ static inline term term_get_sub_binary_ref(term t) return boxed_value[3]; } +/** + * @brief Number of trailing (partial-byte) bits of a sub-binary (0..7). + * + * @details A byte-aligned sub-binary returns 0. A non-byte-aligned bitstring + * returns 1..7, meaning its total bit size is term_binary_size(t)*8 + this. + */ +static inline uint8_t term_get_sub_binary_num_trailing_bits(term t) +{ + const term *boxed_value = term_to_const_term_ptr(t); + return (uint8_t) (boxed_value[2] & 0x7); +} + +/** + * @brief Offset (in bytes) of a sub-binary into its parent binary. + */ +static inline size_t term_get_sub_binary_offset(term t) +{ + const term *boxed_value = term_to_const_term_ptr(t); + return (size_t) (boxed_value[2] >> 3); +} + +/** + * @brief Total size in bits of any bitstring (binary or sub-binary). + */ +static inline size_t term_bit_size(term t) +{ + size_t bits = term_binary_size(t) * 8; + const term *boxed_value = term_to_const_term_ptr(t); + if ((boxed_value[0] & TERM_BOXED_TAG_MASK) == TERM_BOXED_SUB_BINARY) { + bits += boxed_value[2] & 0x7; + } + return bits; +} + /** * @brief Create a resource on the heap. * @details This function creates a resource (obtained from `enif_alloc_resource`) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2ba0944bab..8859ac5e10 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,6 +22,7 @@ cmake_minimum_required (VERSION 3.13) project (tests) add_executable(test-erlang test.c) +add_executable(test-bitstring test-bitstring.c) add_executable(test-enif test-enif.c) add_executable(test-heap test-heap.c) add_executable(test-jit_stream_flash test-jit_stream_flash.c ../src/libAtomVM/jit_stream_flash.c) @@ -30,6 +31,7 @@ add_executable(test-structs test-structs.c) add_executable(test-term test-term.c) target_compile_features(test-erlang PUBLIC c_std_11) +target_compile_features(test-bitstring PUBLIC c_std_11) target_compile_features(test-enif PUBLIC c_std_11) target_compile_features(test-heap PUBLIC c_std_11) target_compile_features(test-jit_stream_flash PUBLIC c_std_11) @@ -39,6 +41,7 @@ target_compile_features(test-term PUBLIC c_std_11) if(CMAKE_COMPILER_IS_GNUCC) target_compile_options(test-erlang PUBLIC -Wall -pedantic -Wextra -ggdb) + target_compile_options(test-bitstring PUBLIC -Wall -pedantic -Wextra -ggdb) target_compile_options(test-enif PUBLIC -Wall -pedantic -Wextra -ggdb) target_compile_options(test-heap PUBLIC -Wall -pedantic -Wextra -ggdb) target_compile_options(test-jit_stream_flash PUBLIC -Wall -pedantic -Wextra -ggdb) @@ -54,6 +57,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") if (HAVE_CLOCK_GETTIME) find_library(LIBRT rt REQUIRED) target_link_libraries(test-erlang PRIVATE ${LIBRT}) + target_link_libraries(test-bitstring PRIVATE ${LIBRT}) target_link_libraries(test-enif PRIVATE ${LIBRT}) target_link_libraries(test-heap PRIVATE ${LIBRT}) target_link_libraries(test-jit_stream_flash PRIVATE ${LIBRT}) @@ -69,6 +73,7 @@ endif() include(MbedTLS) if (MbedTLS_FOUND) target_link_libraries(test-erlang PRIVATE MbedTLS::mbedtls) + target_link_libraries(test-bitstring PRIVATE MbedTLS::mbedtls) target_link_libraries(test-enif PRIVATE MbedTLS::mbedtls) target_link_libraries(test-heap PRIVATE MbedTLS::mbedtls) target_link_libraries(test-jit_stream_flash PRIVATE MbedTLS::mbedtls) @@ -87,6 +92,7 @@ if((${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") OR (${CMAKE_SYSTEM_NAME} STREQUAL "FreeBSD") OR (${CMAKE_SYSTEM_NAME} STREQUAL "DragonFly")) target_include_directories(test-erlang PRIVATE ../src/platforms/generic_unix/lib) + target_include_directories(test-bitstring PRIVATE ../src/platforms/generic_unix/lib) target_include_directories(test-enif PRIVATE ../src/platforms/generic_unix/lib) target_include_directories(test-heap PRIVATE ../src/platforms/generic_unix/lib) target_include_directories(test-jit_stream_flash PRIVATE ../src/platforms/generic_unix/lib) @@ -98,6 +104,7 @@ else() endif() target_include_directories(test-erlang PRIVATE ../src/libAtomVM) +target_include_directories(test-bitstring PRIVATE ../src/libAtomVM) target_include_directories(test-enif PRIVATE ../src/libAtomVM) target_include_directories(test-heap PRIVATE ../src/libAtomVM) target_include_directories(test-jit_stream_flash PRIVATE ../src/libAtomVM ${CMAKE_CURRENT_SOURCE_DIR}) @@ -105,6 +112,7 @@ target_include_directories(test-mailbox PRIVATE ../src/libAtomVM) target_include_directories(test-structs PRIVATE ../src/libAtomVM) target_include_directories(test-term PRIVATE ../src/libAtomVM) target_link_libraries(test-erlang PRIVATE libAtomVM libAtomVM${PLATFORM_LIB_SUFFIX}) +target_link_libraries(test-bitstring PRIVATE libAtomVM libAtomVM${PLATFORM_LIB_SUFFIX}) target_link_libraries(test-enif PRIVATE libAtomVM libAtomVM${PLATFORM_LIB_SUFFIX}) target_link_libraries(test-heap PRIVATE libAtomVM libAtomVM${PLATFORM_LIB_SUFFIX}) # test-jit_stream_flash includes jit_stream_flash.c and provides its own mock platform implementation @@ -136,6 +144,7 @@ endif() if (COVERAGE) include(CodeCoverage) append_coverage_compiler_flags_to_target(test-erlang) + append_coverage_compiler_flags_to_target(test-bitstring) append_coverage_compiler_flags_to_target(test-enif) append_coverage_compiler_flags_to_target(test-heap) append_coverage_compiler_flags_to_target(test-jit_stream_flash) @@ -143,6 +152,7 @@ if (COVERAGE) append_coverage_compiler_flags_to_target(test-structs) append_coverage_compiler_flags_to_target(test-term) append_coverage_linker_flags_to_target(test-erlang) + append_coverage_linker_flags_to_target(test-bitstring) append_coverage_linker_flags_to_target(test-enif) append_coverage_linker_flags_to_target(test-heap) append_coverage_linker_flags_to_target(test-jit_stream_flash) diff --git a/tests/erlang_tests/CMakeLists.txt b/tests/erlang_tests/CMakeLists.txt index 96fc78d8fb..50cd4105cd 100644 --- a/tests/erlang_tests/CMakeLists.txt +++ b/tests/erlang_tests/CMakeLists.txt @@ -454,9 +454,11 @@ compile_erlang(test_match) compile_erlang(test_ordering_0) compile_erlang(test_ordering_1) compile_erlang(test_bs) +compile_erlang(test_bs_gettail) compile_erlang(test_bs_int) compile_erlang(test_bs_int_any_flags) compile_erlang(test_bs_int_unaligned) +compile_erlang(test_bs_size_limits) compile_erlang(test_bs_start_match_live) compile_erlang(test_bs_utf) compile_erlang(test_catch) @@ -1019,9 +1021,11 @@ set(erlang_test_beams test_ordering_0.beam test_ordering_1.beam test_bs.beam + test_bs_gettail.beam test_bs_int.beam test_bs_int_any_flags.beam test_bs_int_unaligned.beam + test_bs_size_limits.beam test_bs_start_match_live.beam test_bs_utf.beam test_catch.beam diff --git a/tests/erlang_tests/test_binary_to_term.erl b/tests/erlang_tests/test_binary_to_term.erl index 2a73e0aada..f91160551b 100644 --- a/tests/erlang_tests/test_binary_to_term.erl +++ b/tests/erlang_tests/test_binary_to_term.erl @@ -82,6 +82,14 @@ start() -> 115, 116>> ), test_reverse(<<"foobar">>, <<131, 109, 0, 0, 0, 6, 102, 111, 111, 98, 97, 114>>), + test_reverse(<<1:1>>, <<131, 77, 0, 0, 0, 1, 1, 128>>), + test_reverse(<<5:31>>, <<131, 77, 0, 0, 0, 4, 7, 0, 0, 0, 10>>), + test_reverse(<<255, 5:7>>, <<131, 77, 0, 0, 0, 2, 7, 255, 10>>), + % BIT_BINARY_EXT: insignificant padding bits in the last byte are accepted + % on decode (as OTP does) and canonicalized (zeroed) on re-encode. + DirtyBitstring = erlang:binary_to_term(id(<<131, 77, 0, 0, 0, 1, 1, 255>>)), + true = DirtyBitstring =:= <<1:1>>, + <<131, 77, 0, 0, 0, 1, 1, 128>> = erlang:term_to_binary(DirtyBitstring), test_reverse(<<":アトムＶＭ">>, <<131, 109, 0, 0, 0, 6, 58, 162, 200, 224, 54, 45>>), test_reverse("", <<131, 106>>), test_reverse("foobar", <<131, 107, 0, 6, 102, 111, 111, 98, 97, 114>>), @@ -159,8 +167,21 @@ start() -> ok = test_atom_utf8_ext_node(), ok = test_encode_process_ref(), ok = test_term_to_binary_options(), + ok = test_invalid_bit_binary(), + ok = test_bitstring_used_offset(), 0. +% A bitstring embedded in a compound term: the decoder must advance past the +% BIT_BINARY_EXT by exactly base + nbytes so the following elements decode at +% the right offset (regression for a wrap in the reported consumed size). +test_bitstring_used_offset() -> + T = ?MODULE:id({<<5:3>>, <<255, 3:2>>, [<<1:1>>, done], <<1, 2, 3>>}), + Bin = erlang:term_to_binary(T), + T = erlang:binary_to_term(Bin), + {T, Used} = erlang:binary_to_term(Bin, [used]), + Used = byte_size(Bin), + ok. + test_term_to_binary_options() -> T = ?MODULE:id({foo, [1, 2, 3], #{a => <<"b">>}, 3.14}), Plain = erlang:term_to_binary(T), @@ -1334,6 +1355,61 @@ test_atom_utf8_ext_node() -> true = is_reference(Ref), ok. +test_invalid_bit_binary() -> + %% BIT_BINARY_EXT (tag 77) layout: <<77, Len:32, Bits:8, Data:Len/bytes>>. + %% Bits is the number of significant bits in the trailing byte: it must be + %% in 1..8 (8 meaning a byte-aligned last byte) and Len must be at least 1. + %% Anything else is a malformed encoding and must be rejected, matching OTP. + + %% Bits = 0 with a non-empty binary + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 1, 0, 128>>) end), + %% Bits = 9 (> 8) + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 1, 9, 128>>) end), + %% Bits = 255 (> 8) + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 1, 255, 128>>) end), + %% Len = 0 with non-zero trailing bits + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 0, 5>>) end), + + ok = test_cve_2026_54890(), + + %% Valid encodings still decode. Bits = 8 is a byte-aligned last byte. + <<255>> = binary_to_term(<<131, 77, 0, 0, 0, 1, 8, 255>>), + <<1:1>> = binary_to_term(<<131, 77, 0, 0, 0, 1, 1, 128>>), + <<255, 5:7>> = binary_to_term(<<131, 77, 0, 0, 0, 2, 7, 255, 10>>), + ok. + +test_cve_2026_54890() -> + case rejects_zero_length_bit_binary() of + false -> + ok; + true -> + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 0, 0>>) end), + ok = expect_badarg(fun() -> binary_to_term(<<131, 77, 0, 0, 0, 0, 0>>, [safe]) end) + end. + +rejects_zero_length_bit_binary() -> + case erlang:system_info(machine) of + "BEAM" -> erts_has_cve_2026_54890_fix(erts_version()); + _ -> true + end. + +%% Fixed in erts 15.2.7.11 (OTP 27.3.4.15), 16.4.0.4 (OTP 28.5.0.4) and +%% 17.0.4 (OTP 29.0.4). Affected: 15.0-15.2.7.10, 16.0-16.4.0.3, 17.0-17.0.3. +erts_has_cve_2026_54890_fix(Version) -> + (Version >= [15, 2, 7, 11] andalso Version < [16]) orelse + (Version >= [16, 4, 0, 4] andalso Version < [17]) orelse + Version >= [17, 0, 4]. + +erts_version() -> + erts_version(erlang:system_info(version), 0, []). + +erts_version([], Current, Acc) -> + lists:reverse([Current | Acc]); +erts_version([$. | Rest], Current, Acc) -> + erts_version(Rest, 0, [Current | Acc]); +erts_version([C | Rest], Current, Acc) when C >= $0, C =< $9 -> + erts_version(Rest, Current * 10 + (C - $0), Acc). + make_binterm_fun(Id) -> fun() -> Bin = ?MODULE:get_binary(Id), diff --git a/tests/erlang_tests/test_bitstring_to_list.erl b/tests/erlang_tests/test_bitstring_to_list.erl index c499994199..d9fad6641c 100644 --- a/tests/erlang_tests/test_bitstring_to_list.erl +++ b/tests/erlang_tests/test_bitstring_to_list.erl @@ -24,10 +24,6 @@ -define(ID(X), ?MODULE:id(X)). -%% AtomVM does not support non-byte-aligned bitstrings, so only byte-aligned -%% binaries are exercised. This matches the compiler's use of -%% erlang:bitstring_to_list/1 (beam_core_to_ssa:pattern_bin/3), which only ever -%% passes byte-aligned binaries. start() -> [1, 2, 3] = erlang:bitstring_to_list(?ID(<<1, 2, 3>>)), [] = erlang:bitstring_to_list(?ID(<<>>)), @@ -36,8 +32,27 @@ start() -> Bytes = erlang:bitstring_to_list(?ID(<<0, 1, 2, 127, 128, 200, 254, 255>>)), ok = raises_badarg(fun() -> erlang:bitstring_to_list(?ID([1, 2, 3])) end), ok = raises_badarg(fun() -> erlang:bitstring_to_list(?ID(not_a_bitstring)) end), + %% Non-byte-aligned bitstrings: the trailing partial byte is returned as a + %% final bitstring element. beam_core_to_ssa relies on this when compiling a + %% match against a non-byte-aligned literal segment (e.g. <<1:1,_:63>>). + [B1] = erlang:bitstring_to_list(?ID(<<1:1>>)), + true = is_bitstring(B1), + false = is_binary(B1), + 1 = bit_size(B1), + 1 = extract(B1), + [B3] = erlang:bitstring_to_list(?ID(<<3:3>>)), + 3 = bit_size(B3), + 3 = extract(B3), + [255, B7] = erlang:bitstring_to_list(?ID(<<255, 5:7>>)), + 7 = bit_size(B7), + 5 = extract(B7), 0. +extract(Bitstring) -> + Size = bit_size(Bitstring), + <> = Bitstring, + Value. + raises_badarg(Fun) -> try Fun() of Ret -> {unexpected, Ret} diff --git a/tests/erlang_tests/test_bs.erl b/tests/erlang_tests/test_bs.erl index 367a79410c..2b92c2f233 100644 --- a/tests/erlang_tests/test_bs.erl +++ b/tests/erlang_tests/test_bs.erl @@ -37,7 +37,17 @@ start() -> ok = test_create_with_invalid_int_value(), ok = test_create_with_invalid_int_size(), ok = test_create_with_int_unit(), - ok = test_create_with_unsupported_unaligned_int_size(), + ok = test_create_with_unaligned_int_size(), + ok = test_bitstring_compare(), + ok = test_bitstring_bif_guards(), + ok = test_bitstring_segments(), + ok = test_little_endian_unaligned(), + ok = test_bs_match_string_trailing(), + ok = test_dynamic_size_extraction(), + ok = test_signed_int_unaligned(), + ok = test_signed_int_zero_width(), + ok = test_big_int_unaligned_unsupported(), + ok = test_non_pow2_unit(), ok = test_create_with_int_little_endian(), ok = test_create_with_int_signed(), ok = test_create_with_invalid_binary_value(), @@ -53,6 +63,8 @@ start() -> ok = test_get_with_int_little_endian(), ok = test_get_with_int_signed(), ok = test_get_with_unaligned_binary(), + ok = test_large_unaligned_slice(), + ok = test_private_append_bitstring(), <<"">> = test_match_first_integer(<<16#FF>>), <<1, 2, 3>> = test_match_first_integer(<<16#FF, 1, 2, 3>>), @@ -106,6 +118,8 @@ start() -> ok = test_bs_variable_size_bitstring(), ok = test_negative_dynamic_size(), ok = test_oversized_dynamic_size(), + ok = test_construction_size_overflow(), + ok = test_wide_integer_construction(), ok = test_float(), 0. @@ -147,8 +161,322 @@ test_create_with_int_unit() -> ), ok. -test_create_with_unsupported_unaligned_int_size() -> - atom_unsupported(fun() -> create_int_binary(16#FFFF, id(28)) end). +test_create_with_unaligned_int_size() -> + B = create_int_binary(16#FFFF, id(28)), + 28 = bit_size(B), + 4 = byte_size(B), + false = is_binary(B), + true = is_bitstring(B), + ok. + +test_bitstring_compare() -> + false = id(<<5:31>>) =:= id(<<5:24>>), + false = id(<<5:31>>) == id(<<5:24>>), + true = id(<<5:3>>) =:= id(<<5:3>>), + true = id(<<5:3>>) == id(<<5:3>>), + true = id(<<(id(5)):31>>) =:= id(<<5:31>>), + false = id(<<5:3>>) < id(<<4>>), + true = id(<<4>>) < id(<<5:3>>), + true = id(<<255>>) < id(<<255, 0:1>>), + true = id(<<255, 0:1>>) < id(<<255, 1:1>>), + true = id(<<1:1>>) > id(<<0:7>>), + true = id(<<>>) < id(<<0:1>>), + [<<>>, <<254>>, <<255>>, <<255, 0:1>>, <<255, 1:1>>] = sort_bitstrings( + id([<<255, 1:1>>, <<255>>, <<255, 0:1>>, <<254>>, <<>>]) + ), + ok. + +% BIFs and NIFs that operate on binaries must badarg on a non-byte-aligned +% bitstring instead of silently truncating it to whole bytes; a few +% (byte_size/1, bit_size/1, size/1, split_binary/2) accept bitstrings. +test_bitstring_bif_guards() -> + Bits9 = id(<<1:9>>), + Bits1 = id(<<1:1>>), + 2 = byte_size(Bits9), + 9 = bit_size(Bits9), + % size/1 rounds down to whole bytes, unlike byte_size/1 which rounds up + 1 = size(Bits9), + 0 = size(Bits1), + % is_bitstring accepts a non-byte-aligned bitstring, is_binary does not + true = erlang:is_bitstring(Bits9), + false = erlang:is_binary(Bits9), + true = erlang:is_bitstring(id(<<1, 2>>)), + true = erlang:is_binary(id(<<1, 2>>)), + false = erlang:is_bitstring(id({})), + % binary_part/3 and binary:part/3 badarg on a non-byte-aligned bitstring + % only since OTP 27; OTP 26 truncated to whole bytes. AtomVM follows the + % modern behavior. + HasBitstringPartGuard = + erlang:system_info(machine) =:= "ATOM" orelse + list_to_integer(erlang:system_info(otp_release)) >= 27, + case HasBitstringPartGuard of + true -> + expect_error(fun() -> binary_part(Bits9, 0, 1) end, badarg), + expect_error(fun() -> binary:part(Bits9, 0, 1) end, badarg); + false -> + ok + end, + % split_binary/2 keeps the trailing bits in the second part + {<<>>, <<0, 1:1>>} = split_binary(Bits9, 0), + {<<0>>, <<1:1>>} = split_binary(Bits9, 1), + expect_error(fun() -> split_binary(Bits9, 2) end, badarg), + expect_error(fun() -> iolist_to_binary(Bits1) end, badarg), + expect_error(fun() -> list_to_binary([Bits1]) end, badarg), + expect_error(fun() -> iolist_size(Bits1) end, badarg), + expect_error(fun() -> binary_to_atom(Bits1, utf8) end, badarg), + expect_error(fun() -> binary_to_list(Bits1) end, badarg), + expect_error(fun() -> binary_to_term(Bits1) end, badarg), + expect_error(fun() -> erlang:crc32(Bits1) end, badarg), + expect_error(fun() -> binary:at(Bits9, 0) end, badarg), + expect_error(fun() -> binary:copy(Bits1) end, badarg), + expect_error(fun() -> binary:split(Bits9, id(<<0>>)) end, badarg), + expect_error(fun() -> binary:first(Bits9) end, badarg), + expect_error(fun() -> binary:last(Bits9) end, badarg), + ok. + +% A bitstring used as a segment source must be copied bit-granularly; these +% used to silently truncate the source to whole bytes. Expected values are +% written in byte layout so they build through the byte-aligned path even if +% the compiler does not constant-fold them. +test_bitstring_segments() -> + Bits1 = id(<<1:1>>), + <<1:1>> = copy_bitstring(Bits1), + <<213, 1:1>> = append_to_bitstring(Bits1), + <<5:3>> = copy_bitstring(id(<<5:3>>)), + % explicit bit-sized bitstring segments + <<2:3>> = take_bits(id(<<2:3>>)), + <<5:4>> = sized_then_bit(id(<<2:3>>)), + <> = id(<<5:3>>), + <<2:2>> = T2, + % mixed bitstring segments and a byte segment at an unaligned offset + <<13:4>> = mix_bitstrings(Bits1, id(<<5:3>>)), + <<255, 255, 1:1>> = sandwich(Bits1), + ok. + +copy_bitstring(B) -> <>. +append_to_bitstring(B) -> <>. +take_bits(B) -> <>. +sized_then_bit(B) -> <>. +mix_bitstrings(P, Q) -> <

>. +sandwich(P) -> <<255, P/bits, 255>>. + +% Little-endian integers whose width is not a multiple of 8 lay out complete +% low-order bytes first, then the remaining high-order bits (OTP layout). +% Expected values are written in byte layout (verified on OTP) so they do not +% depend on the little-endian runtime path under test. +test_little_endian_unaligned() -> + ok = check_le_cases([ + {1, 1, <<1:1>>, <<11:4>>}, + {4, 16#A, <<10:4>>, <<90:7>>}, + {7, 16#55, <<85:7>>, <<181, 1:2>>}, + {9, 16#155, <<85, 1:1>>, <<170, 11:4>>}, + {12, 16#ABC, <<188, 10:4>>, <<183, 74:7>>}, + {15, 16#5A5A, <<90, 90:7>>, <<171, 86, 2:2>>}, + {63, 16#123456789ABCDEF, <<239, 205, 171, 137, 103, 69, 35, 1:7>>, + <<189, 249, 181, 113, 44, 232, 164, 96, 1:2>>} + ]), + % signed little-endian round trip of a negative value + <<251, 15:4>> = make_le(id(-5), id(12)), + <> = id(<<251, 15:4>>), + -5 = S, + ok. + +check_le_cases([]) -> + ok; +check_le_cases([{W, V, Plain, Prefixed} | T]) -> + % construction, at bit offset 0 and after a 3-bit prefix + Plain = make_le(id(V), id(W)), + Prefixed = make_le_prefixed(id(V), id(W)), + % extraction (the inverse mapping), from byte-layout literals + <> = id(Plain), + V = X, + <<_:3, Y:W/little>> = id(Prefixed), + V = Y, + check_le_cases(T). + +make_le(V, W) -> <>. +make_le_prefixed(V, W) -> <<5:3, V:W/little>>. + +% Literal matches that consume trailing bits must measure the source capacity +% in bits, not whole bytes. +test_bs_match_string_trailing() -> + ok = match_9(id(<<255, 1:1>>)), + nomatch = match_9(id(<<255, 0:1>>)), + nomatch = match_9(id(<<255>>)), + ok = match_long(id(<<"hello!!!", 5:3>>)), + nomatch = match_long(id(<<"hello!!!", 4:3>>)), + ok. + +match_9(<<255, 1:1>>) -> ok; +match_9(_) -> nomatch. + +match_long(<<"hello!!!", 5:3>>) -> ok; +match_long(_) -> nomatch. + +% Dynamic-size binary/bitstring segment extraction compiles to bs_get_binary2 +% (OTP 26 through at least 29 emit it; fixed sizes go through bs_match). +% Sizes are bit-granular and the source offset may be unaligned. +test_dynamic_size_extraction() -> + {<<1, 2>>, <<3>>} = dyn_binary(id(2), id(<<1, 2, 3>>)), + nope = dyn_binary(id(4), id(<<1, 2, 3>>)), + {<<5:3>>, <<1:1>>} = dyn_bits(id(3), id(<<5:3, 1:1>>)), + {<<255, 1:1>>, <<5:3>>} = dyn_bits(id(9), id(<<255, 1:1, 5:3>>)), + nope = dyn_bits(id(5), id(<<5:3, 1:1>>)), + % dynamic size at an unaligned offset + {<<2:2>>, <<1:1>>} = dyn_bits_after3(id(2), id(<<5:3, 2:2, 1:1>>)), + % an all-remaining binary tail fails on a non-byte-aligned remainder + nope = bin_tail_of(id(<<1:12>>)), + % a negative dynamic size fails the match, it does not raise + nope = dyn_binary(id(-1), id(<<1, 2, 3>>)), + nope = dyn_bits(id(-1), id(<<5:3, 1:1>>)), + % a bound segment with a unit other than 8 is only supported since + % bit-granular extraction; a negative size must still be rejected before it + % is scaled, as it is for the units the parent commit already covers + nope = dyn_binary_unit64(id(-1), id(<<1>>)), + nope = dyn_binary_unit64(id(-(1 bsl 58)), id(<<1>>)), + nope = dyn_binary_unit64(id(-(1 bsl 58) - 1), id(<<1>>)), + % a positive size whose scaling by the unit wraps must also fail the match: + % (1 bsl 58) * 64 wraps to 0 on a 64-bit target, so without an overflow-aware + % scaler the JIT would match an empty slice while BEAM/interpreter fail. + nope = dyn_binary_unit64(id(1 bsl 58), id(<<1>>)), + nope = dyn_binary_unit64(id(1 bsl 62), id(<<1>>)), + nope = u3(id(-(1 bsl 62)), id(<<5:6, 1:3>>)), + ok. + +% A non-power-of-two unit on a variable-size binary segment (BEAM emits +% bs_get_binary2 with that unit); the JIT must handle what the interpreter does. +test_non_pow2_unit() -> + {<<5:6>>, <<1:3>>} = u3(id(2), id(<<5:6, 1:3>>)), + nope = u3(id(4), id(<<5:6, 1:3>>)), + % `all` with a non-power-of-two unit: the remainder must be tested with a + % real remainder, not a mask, and the JIT must agree with the interpreter + <<5:6, 1:3>> = u3_all(id(<<5:6, 1:3>>)), + <<1:1, 2:2>> = u3_all(id(<<1:1, 2:2>>)), + <<>> = u3_all(id(<<>>)), + nope = u3_all(id(<<5:6, 1:2>>)), + nope = u3_all(id(<<1>>)), + % same, at a non-byte-aligned starting offset + <<3:3, 1:3>> = u3_all_after5(id(<<9:5, 3:3, 1:3>>)), + nope = u3_all_after5(id(<<9:5, 3:2>>)), + ok. + +u3(N, B) -> + case B of + <> -> {X, R}; + _ -> nope + end. + +u3_all(B) -> + case B of + <> -> X; + _ -> nope + end. + +u3_all_after5(B) -> + case B of + <<_:5, X/binary-unit:3>> -> X; + _ -> nope + end. + +% A signed integer of the full 64-bit width at a non-byte-aligned offset must +% sign-extend correctly (guards a shift by the whole type width in the extractor). +test_signed_int_unaligned() -> + AllOnes = id(<<16#0F, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#F0>>), + -1 = sig64_at4(AllOnes), + 18446744073709551615 = uns64_at4(AllOnes), + -1000000000000 = sig64_at4(id(<<0:4, -1000000000000:64/signed, 0:4>>)), + 42 = sig64_at4(id(<<0:4, 42:64/signed, 0:4>>)), + ok. + +% A zero-width field has no sign bit to extend. Only a runtime size reaches the +% extractor: the compiler folds a literal 0 away. +test_signed_int_zero_width() -> + 0 = sig_at(id(<<>>), id(0)), + 0 = sig_at(id(<<16#FF>>), id(0)), + 0 = sig_at_little(id(<<16#FF>>), id(0)), + % a sign bit that is present still extends, at the same call site + -1 = sig_at(id(<<16#FF>>), id(8)), + ok. + +sig_at(B, N) -> + <> = B, + X. + +sig_at_little(B, N) -> + <> = B, + X. + +% A >64-bit integer field at a non-byte-aligned offset is not supported yet +% (BEAM matches it); AtomVM raises unsupported, as the construction side does. +test_big_int_unaligned_unsupported() -> + ok = atom_unsupported(fun() -> big72_at1(id(<<0:1, 42:72, 0:7>>)) end), + % a runtime size takes the legacy bs_get_integer2 path, which must raise + % unsupported too rather than failing the match + ok = atom_unsupported(fun() -> bigdyn_at1(id(<<0:1, 42:72, 0:7>>), id(72)) end), + % byte-aligned offset and size are supported at any width up to 256 bits. + % The value must not fit in 64 bits: a >64-bit field yields an unnormalized + % bignum, so a small value would not compare equal to a small integer. + (1 bsl 70) = big72_at0(id(<<(1 bsl 70):72, 0:8>>)), + ok. + +big72_at1(B) -> + <<_:1, X:72, _:7>> = B, + X. + +big72_at0(B) -> + <> = B, + X. + +bigdyn_at1(B, N) -> + <<_:1, X:N, _:7>> = B, + X. + +sig64_at4(B) -> + <<_:4, X:64/signed, _:4>> = B, + X. + +uns64_at4(B) -> + <<_:4, X:64/unsigned, _:4>> = B, + X. + +dyn_binary(N, B) -> + case B of + <> -> {A, Rest}; + _ -> nope + end. + +dyn_bits(N, B) -> + case B of + <> -> {X, R}; + _ -> nope + end. + +dyn_binary_unit64(N, B) -> + case B of + <> -> {X, R}; + _ -> nope + end. + +dyn_bits_after3(N, B) -> + case B of + <<_:3, X:N/bits, R/bits>> -> {X, R}; + _ -> nope + end. + +bin_tail_of(B) -> + case B of + <> -> X; + _ -> nope + end. + +sort_bitstrings(L) -> sort_bitstrings(L, []). + +sort_bitstrings([], Sorted) -> Sorted; +sort_bitstrings([H | T], Sorted) -> sort_bitstrings(T, insert_bitstring(Sorted, H)). + +insert_bitstring([], B) -> [B]; +insert_bitstring([H | T], B) when B < H -> [B, H | T]; +insert_bitstring([H | T], B) -> [H | insert_bitstring(T, B)]. test_create_with_int_little_endian() -> <<2, 1>> = create_int_binary_little_endian(16#0102, 16), @@ -187,7 +515,10 @@ test_create_with_binary_size_out_of_range() -> expect_error(fun() -> create_binary_binary(<<"foo">>, id(4)) end, badarg). test_create_with_unsupported_binary_unit() -> - atom_unsupported(fun() -> create_binary_binary_unit_3(<<"foo">>, id(3)) end). + % A binary segment whose size in bits is not a multiple of 8 takes a + % bit-granular prefix of the source (9 bits of <<"foo">> here). + <<102, 0:1>> = create_binary_binary_unit_3(<<"foo">>, id(3)), + ok. % Things are very broken here, we get {badmatch, <<16#FFFFFFFF:32>>} but % this term isn't equal to {badmatch, <<16#FFFFFFFF:32>>} @@ -277,9 +608,99 @@ test_get_with_int_signed() -> ok. test_get_with_unaligned_binary() -> - atom_unsupported(fun() -> get_int_then_binary(<<1, 2, 3, 4>>, id(4), id(1)) end, fun(T) -> - T =:= badarg - end). + % A dynamic-size binary segment may start at an unaligned offset (the + % extracted slice is copied in that case). + {0, <<16>>} = get_int_then_binary(<<1, 2, 3, 4>>, id(4), id(1)), + ok. + +% A slice of 64 bytes or more is allocated as a refc binary instead of a heap +% binary. Extract such slices at a non-byte-aligned offset, where the bits have +% to be copied and cannot be shared with the source. +test_large_unaligned_slice() -> + Payload = id(seq_binary(72, <<>>)), + 576 = bit_size(Payload), + + % fixed size, byte-aligned length at an unaligned offset + Src1 = <<0:3, Payload/binary, 0:5>>, + 73 = byte_size(Src1), + <<_:3, Slice1:72/binary, _:5>> = Src1, + Payload = Slice1, + true = is_binary(Slice1), + + % same, with a size only known at run time + N = id(72), + <<_:3, Slice2:N/binary, _:5>> = Src1, + Payload = Slice2, + + % get_tail of a byte-aligned remainder at an unaligned offset + Src2 = <<0:3, Payload/binary>>, + 579 = bit_size(Src2), + <<_:3, Tail1/bits>> = Src2, + true = is_binary(Tail1), + Payload = Tail1, + + % get_tail of a remainder that is itself not byte-aligned: the copy is + % wrapped in a sub-binary carrying the trailing bits + Src3 = <<0:3, Payload/binary, 1:1>>, + 580 = bit_size(Src3), + <<_:3, Tail2/bits>> = Src3, + 577 = bit_size(Tail2), + false = is_binary(Tail2), + <> = Tail2, + + % a large slice that is not byte-aligned either: 2 remaining leading zero + % bits followed by the first 569 bits of the payload + <<_:1, Slice3:571/bits, _/bits>> = Src3, + 571 = bit_size(Slice3), + false = is_binary(Slice3), + <> = Payload, + <<0:2, PayloadPrefix/bits>> = Slice3, + ok. + +seq_binary(0, Acc) -> Acc; +seq_binary(N, Acc) -> seq_binary(N - 1, <<(N - 1):8, Acc/binary>>). + +% The accumulator of the destructive append path (private_append) may itself be +% a bitstring: its trailing bits must be preserved, and a result that is not +% byte-aligned must be returned as a bitstring. +test_private_append_bitstring() -> + <<>> = accum_bits(id([])), + <<5:3>> = accum_bits(id([5])), + <<5:3, 2:3>> = accum_bits(id([5, 2])), + 6 = bit_size(accum_bits(id([5, 2]))), + + % after 8 three-bit segments the total is byte-aligned again, while the + % accumulator entering the last append is not + Eight = accum_bits(id([1, 2, 3, 4, 5, 6, 7, 0])), + 24 = bit_size(Eight), + true = is_binary(Eight), + <<1:3, 2:3, 3:3, 4:3, 5:3, 6:3, 7:3, 0:3>> = Eight, + + % long enough that the accumulator is a refc binary being reallocated + Long = accum_bits(id(repeat(200, 5, []))), + 600 = bit_size(Long), + true = is_binary(Long), + <<5:3, 5:3, Rest/bits>> = Long, + 594 = bit_size(Rest), + + % appending a bitstring, rather than an integer segment, to an unaligned + % accumulator + <<1:3, 2:5, 3:4>> = accum_tail(<<1:3>>, <<2:5>>, <<3:4>>), + 12 = bit_size(accum_tail(<<1:3>>, <<2:5>>, <<3:4>>)), + ok. + +accum_bits(L) -> accum_bits(L, <<>>). + +accum_bits([], Acc) -> Acc; +accum_bits([H | T], Acc) -> accum_bits(T, <>). + +accum_tail(A, B, C) -> accum_tail2([A, B, C], <<>>). + +accum_tail2([], Acc) -> Acc; +accum_tail2([H | T], Acc) -> accum_tail2(T, <>). + +repeat(0, _V, Acc) -> Acc; +repeat(N, V, Acc) -> repeat(N - 1, V, [V | Acc]). create_int_binary_unit_3(Value, Size) -> <>. @@ -722,6 +1143,47 @@ test_oversized_dynamic_size() -> nope = bin_unit8(id(1 bsl 61), B), ok. +% Construction (bs_create_bin) with a size whose scaling by the segment unit +% wraps must fail rather than silently truncate: (1 bsl 58) * 64 wraps to 0 on a +% 64-bit target, so without an overflow check the source-capacity test and the +% allocation size would both be bypassed. BEAM raises system_limit; AtomVM may +% raise system_limit or badarg for the binary-source case, so accept either. +test_construction_size_overflow() -> + ok = expect_construction_error(fun() -> make_bin_unit64(id(<<>>), id(1 bsl 58)) end), + ok = expect_construction_error(fun() -> make_bin_unit64(id(<<1>>), id(1 bsl 58)) end), + ok = expect_construction_error(fun() -> make_int_unit64(id(0), id(1 bsl 58)) end), + ok = expect_construction_error(fun() -> make_int_unit64(id(0), id(1 bsl 62)) end), + ok. + +make_bin_unit64(B, N) -> <>. +make_int_unit64(V, N) -> <>. + +expect_construction_error(Fun) -> + try Fun() of + R -> {unexpected, R} + catch + error:badarg -> ok; + error:system_limit -> ok + end. + +% Integers narrower than the field but wider than 64 bits must sign-extend a +% negative value (fill the high bits with 1s), not zero-extend it, and this must +% hold for both endiannesses. Expected values match BEAM. +test_wide_integer_construction() -> + <<255, 255, 255, 255, 255, 255, 255, 255, 1:1>> = wide_be(id(-1), id(65)), + <<255, 255, 255, 255, 255, 255, 255, 255, 1:1>> = wide_le(id(-1), id(65)), + <<255, 255, 255, 255, 255, 255, 255, 255, 127:7>> = wide_be(id(-1), id(71)), + <<255, 255, 255, 255, 255, 255, 255, 253, 1:1>> = wide_be(id(-5), id(65)), + <<251, 255, 255, 255, 255, 255, 255, 255, 127:7>> = wide_le(id(-5), id(71)), + % positive values still zero-extend + <<0, 0, 0, 0, 0, 0, 0, 2, 1:1>> = wide_be(id(5), id(65)), + <<0, 0, 0, 0, 0, 0, 0, 0, 5:7>> = wide_be(id(5), id(71)), + <<5, 0, 0, 0, 0, 0, 0, 0, 0:7>> = wide_le(id(5), id(71)), + ok. + +wide_be(V, N) -> <>. +wide_le(V, N) -> <>. + skip_unit64(N, B) -> case B of <<_:N/binary-unit:64, R/binary>> -> R; diff --git a/tests/erlang_tests/test_bs_gettail.erl b/tests/erlang_tests/test_bs_gettail.erl new file mode 100644 index 0000000000..83252918e3 --- /dev/null +++ b/tests/erlang_tests/test_bs_gettail.erl @@ -0,0 +1,42 @@ +% +% This file is part of AtomVM. +% +% Copyright 2026 Paul Guyot +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. +% +% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +% + +-module(test_bs_gettail). +-export([start/0, id/1]). + +%% Matching-only (get_tail) validation: byte-aligned construction, then match a +%% non-byte-aligned prefix capturing the remainder as a bitstring, then re-match. +start() -> + Bin = <<(id(16#12345678)):32>>, + <> = Bin, + 152709948 = I, + 31 = bit_size(Bin) - bit_size(V), + 1 = bit_size(V), + false = is_binary(V), + true = is_bitstring(V), + <<0:1>> = V, + Bin2 = <<(id(16#12345679)):32>>, + <> = Bin2, + false = is_binary(W), + true = is_bitstring(W), + <<1:1>> = W, + 0. + +id(X) -> X. diff --git a/tests/erlang_tests/test_bs_int_unaligned.erl b/tests/erlang_tests/test_bs_int_unaligned.erl index 25ce755d35..20709a4db4 100644 --- a/tests/erlang_tests/test_bs_int_unaligned.erl +++ b/tests/erlang_tests/test_bs_int_unaligned.erl @@ -32,14 +32,9 @@ start() -> %% expressions that traverse a byte boundary ok = test_pack_unpack(13, 3, 1, 7, false), ok = test_pack_unpack(3, 13, 1, 7, false), - %% expressions not aligned on 8 bit boundary (expect failure with AtomVM) - ExpectFailure = - case erlang:system_info(machine) of - "BEAM" -> false; - _ -> true - end, - ok = test_pack_unpack(1, 1, 1, 1, ExpectFailure), - ok = test_pack_unpack(3, 13, 1, 1, ExpectFailure), + %% expressions not aligned on 8 bit boundary + ok = test_pack_unpack(1, 1, 1, 1, false), + ok = test_pack_unpack(3, 13, 1, 1, false), 0. test_pack_unpack(ALen, BLen, CLen, DLen, ExpectFailure) -> diff --git a/tests/erlang_tests/test_bs_size_limits.erl b/tests/erlang_tests/test_bs_size_limits.erl new file mode 100644 index 0000000000..bfe72b8824 --- /dev/null +++ b/tests/erlang_tests/test_bs_size_limits.erl @@ -0,0 +1,93 @@ +% +% This file is part of AtomVM. +% +% Copyright 2026 Paul Guyot +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. +% +% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +% + +%% Sizes that bs_create_bin cannot represent must be reported as system_limit, +%% and the interpreter and the JIT must agree on where that boundary is. This is +%% not run against BEAM: it builds segments one at a time and aborts the whole +%% node trying to allocate them, rather than rejecting the total up front. +-module(test_bs_size_limits). + +-export([start/0, id/1]). + +start() -> + ok = test_representable(), + ok = test_total_overflows_word(), + ok = test_large_but_representable(), + 0. + +%% An ordinary construction with runtime sizes is unaffected by the checks. +test_representable() -> + {ok, 80} = build_8(id(10)), + {ok, 400} = build_40(id(10)), + ok. + +%% 40 segments of nearly the largest small integer: the total wraps a machine +%% word, so no allocation could ever match it. +test_total_overflows_word() -> + system_limit = build_40(id(max_segment_bits())), + ok. + +%% 16 of the same segments: the total is huge but does not wrap and stays within +%% the representable range. Whether it can be allocated depends on the platform, +%% but it must not be rejected as unrepresentable -- that is the boundary the +%% interpreter and the JIT have to agree on. +test_large_but_representable() -> + case build_16(id(max_segment_bits())) of + {ok, _} -> ok; + out_of_memory -> ok + end. + +%% Just below the largest small integer, so the size is not a boxed integer. +max_segment_bits() -> + (1 bsl (word_bits() - 5)) - 1. + +word_bits() -> + case erlang:system_info(wordsize) of + 4 -> 32; + 8 -> 64 + end. + +build_8(N) -> + try <<0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N>> of + R -> {ok, bit_size(R)} + catch + error:E -> E + end. + +build_16(N) -> + try <<0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N>> of + R -> {ok, bit_size(R)} + catch + error:E -> E + end. + +build_40(N) -> + try + <<0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, + 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, 0:N, + 0:N, 0:N, 0:N, 0:N, 0:N>> + of + R -> {ok, bit_size(R)} + catch + error:E -> E + end. + +id(X) -> + X. diff --git a/tests/erlang_tests/test_is_bitstring_is_binary.erl b/tests/erlang_tests/test_is_bitstring_is_binary.erl index e727c6ff1d..03f086e641 100644 --- a/tests/erlang_tests/test_is_bitstring_is_binary.erl +++ b/tests/erlang_tests/test_is_bitstring_is_binary.erl @@ -31,8 +31,8 @@ id(X) -> X. test_is_bitstring() -> true = is_bitstring(id(<<"hello">>)), - % bitstrings are currently unsupported - % true = is_bitstring(id(<<1:1>>)), + true = is_bitstring(id(<<1:1>>)), + true = is_bitstring(id(<<"hello", 3:7>>)), true = is_bitstring(id(<<>>)), false = is_bitstring(id(binary)), false = is_bitstring(id("hello")), @@ -41,8 +41,8 @@ test_is_bitstring() -> test_is_binary() -> true = is_binary(id(<<"hello">>)), - % bitstrings are currently unsupported - % false = is_binary(id(<<1:1>>)), + false = is_binary(id(<<1:1>>)), + false = is_binary(id(<<"hello", 3:7>>)), true = is_binary(id(<<>>)), false = is_binary(id(binary)), false = is_binary(id("hello")), diff --git a/tests/erlang_tests/test_list_to_binary.erl b/tests/erlang_tests/test_list_to_binary.erl index 7714b23ecf..7f23933e3a 100644 --- a/tests/erlang_tests/test_list_to_binary.erl +++ b/tests/erlang_tests/test_list_to_binary.erl @@ -27,6 +27,7 @@ start() -> ok = test_list_to_binary(), ok = test_iolist_to_binary(), + ok = test_improper_tail(), 0. test_list_to_binary() -> <<"Hello world">> = erlang:list_to_binary(?ID([<<"Hello ">>, [<<"wor">>, [$l, $d]]])), @@ -58,6 +59,26 @@ test_iolist_to_binary() -> ok = raises_badarg(fun() -> iolist_size(?ID([12450, 12488, 12512])) end), ok. +% Bytes are only accepted as list elements: an improper tail may be a binary, +% but never an integer. +test_improper_tail() -> + <<1, 2>> = erlang:list_to_binary(?ID([1 | <<2>>])), + <<1, 2, 3>> = erlang:list_to_binary(?ID([1, 2 | <<3>>])), + <<1, 2>> = erlang:list_to_binary(?ID([[1 | <<2>>]])), + <<1, 2>> = erlang:iolist_to_binary(?ID([1 | <<2>>])), + 2 = erlang:iolist_size(?ID([1 | <<2>>])), + 3 = erlang:iolist_size(?ID([[1 | <<2>>], <<3>>])), + + ok = raises_badarg(fun() -> erlang:list_to_binary(?ID([1 | 2])) end), + ok = raises_badarg(fun() -> erlang:list_to_binary(?ID([1, 2 | 3])) end), + ok = raises_badarg(fun() -> erlang:list_to_binary(?ID([[1 | 2]])) end), + ok = raises_badarg(fun() -> erlang:list_to_binary(?ID([<<1>> | 2])) end), + ok = raises_badarg(fun() -> erlang:list_to_binary(?ID([1 | some_atom])) end), + ok = raises_badarg(fun() -> erlang:iolist_to_binary(?ID([1 | 2])) end), + ok = raises_badarg(fun() -> erlang:iolist_size(?ID([1 | 2])) end), + ok = raises_badarg(fun() -> erlang:iolist_size(?ID([[1 | 2], <<3>>])) end), + ok. + concat_space(A, B) -> list_to_binary(A ++ " " ++ B). diff --git a/tests/erlang_tests/test_list_to_bitstring.erl b/tests/erlang_tests/test_list_to_bitstring.erl index c2bca8bae1..77de3a5108 100644 --- a/tests/erlang_tests/test_list_to_bitstring.erl +++ b/tests/erlang_tests/test_list_to_bitstring.erl @@ -26,6 +26,8 @@ start() -> ok = test_concat(), ok = test_iolist(), ok = test_empty_list_to_binary(), + ok = test_bit_level(), + ok = test_large_elements(), 0. test_concat() -> @@ -43,6 +45,56 @@ test_empty_list_to_binary() -> <<"">> = erlang:list_to_bitstring(?MODULE:id([])), ok. +test_bit_level() -> + <<2:2>> = list_to_bitstring(?MODULE:id([<<1:1>>, <<0:1>>])), + % improper tail may be a bitstring + <<2:2>> = list_to_bitstring(?MODULE:id([<<1:1>> | <<0:1>>])), + <<15:4>> = list_to_bitstring(?MODULE:id([[<<1:1>> | <<7:3>>]])), + % mixed bytes and bit fragments, nested + <<"ab", 1:3, 0:5>> = list_to_bitstring(?MODULE:id([<<"ab">>, [<<1:3>>], <<0:5>>])), + <<65, 1:1, 66:8, 3:7, 67>> = list_to_bitstring(?MODULE:id([65, <<1:1>>, 66, <<3:7>>, 67])), + 3000 = bit_size(list_to_bitstring(?MODULE:id(duplicate(1000, <<5:3>>, [])))), + % non-bitstring elements are rejected + 0 = invalid([<<1:1>>, 3.14]), + 0 = invalid([<<1:1>> | some_atom]), + 0 = invalid([256]), + % bytes are only accepted as elements: an improper tail may be a bitstring + % but never an integer + <<1, 2>> = list_to_bitstring(?MODULE:id([1 | <<2>>])), + <<1, 2, 1:1>> = list_to_bitstring(?MODULE:id([[1 | <<2>>] | <<1:1>>])), + <<"">> = list_to_bitstring(?MODULE:id([[]])), + 0 = invalid([1 | 2]), + 0 = invalid([<<1:1>> | 2]), + 0 = invalid([[1 | 2]]), + 0 = invalid([1, 2 | 3]), + ok. + +% Elements large enough that the result is allocated as a refc binary rather +% than a heap binary, including the same binary referenced several times. +test_large_elements() -> + Big = ?MODULE:id(seq_binary(100, <<>>)), + 100 = byte_size(Big), + Twice = list_to_bitstring(?MODULE:id([Big, Big])), + 200 = byte_size(Twice), + <> = Twice, + % same, with a trailing partial element: the whole bytes are copied into a + % refc binary that gets wrapped in a sub-binary + Odd = list_to_bitstring(?MODULE:id([Big, Big, <<1:1>>])), + 1601 = bit_size(Odd), + false = is_binary(Odd), + <> = Odd, + % a leading partial element shifts every following byte + Shifted = list_to_bitstring(?MODULE:id([<<0:1>>, Big])), + 801 = bit_size(Shifted), + <<0:1, Big:100/binary>> = Shifted, + ok. + +seq_binary(0, Acc) -> Acc; +seq_binary(N, Acc) -> seq_binary(N - 1, <<(N - 1):8, Acc/binary>>). + +duplicate(0, _T, Acc) -> Acc; +duplicate(N, T, Acc) -> duplicate(N - 1, T, [T | Acc]). + concat(A, B) -> list_to_bitstring(?MODULE:id(A ++ " " ++ B)). diff --git a/tests/erlang_tests/test_no_bs_create_bin.erl b/tests/erlang_tests/test_no_bs_create_bin.erl index 0868c345af..856ecc9a24 100644 --- a/tests/erlang_tests/test_no_bs_create_bin.erl +++ b/tests/erlang_tests/test_no_bs_create_bin.erl @@ -39,7 +39,8 @@ put_utf16/1, put_utf32/1, build_from_list/1, - bin_comprehension/1 + bin_comprehension/1, + ext_id/1 ]). -if(?OTP_RELEASE =< 27). @@ -62,6 +63,7 @@ start() -> ok = test_put_utf32(), ok = test_private_append(), ok = test_bin_comprehension(), + ok = test_bitstring_source(), 0. %% bs_add + bs_init_bits + bs_put_integer with dynamic size. @@ -133,6 +135,100 @@ test_bin_comprehension() -> <<42>> = bin_comprehension([42]), ok. +%% bs_append / bs_private_append / bs_put_binary with a non-byte-aligned +%% source. These opcodes are bit-granular: a partial bitstring keeps its +%% trailing bits in the result and every segment written after it shifts by +%% that many bits, exactly as bs_create_bin does. A /binary segment (unit 8) +%% still rejects a partial source with badarg, as on BEAM. +test_bitstring_source() -> + %% a byte-aligned source is unaffected + <<1, 2, 3, 4>> = concat_bins(id(<<1, 2>>), id(<<3, 4>>)), + <<1, 2, 3>> = build_from_list(id([1, 2, 3])), + <<1, 2>> = copy_bitstring(id(<<1, 2>>)), + <<>> = copy_bitstring(id(<<>>)), + <<1, 2, 3>> = build_from_list_onto(id(<<>>), [1, 2, 3]), + + Bits1 = id(<<1:1>>), + Bits3 = id(<<5:3>>), + Bits9 = id(<<16#AB:8, 1:1>>), + + %% bs_append: a partial source is copied whole, alone and followed by a + %% segment that shifts by the trailing bit count + <<1:1>> = copy_bitstring(Bits1), + <<5:3>> = copy_bitstring(Bits3), + <<16#AB:8, 1:1>> = copy_bitstring(Bits9), + <<1:1, 16#AB:8>> = append_byte_to_bitstring(Bits1, 16#AB), + <<5:3, 16#AB:8>> = append_byte_to_bitstring(Bits3, 16#AB), + <<16#AB:8, 1:1, 16#CD:8>> = append_byte_to_bitstring(Bits9, 16#CD), + + %% bs_put_binary with an explicit bit size, and with a partial source + %% written at an already unaligned destination offset + <<5:3>> = put_bitstring_sized(Bits3, 3), + <<5:3, 1:1>> = append_bits_to_bitstring(Bits3, id(<<1:1>>)), + <<16#AB:8, 1:1, 5:3>> = append_bits_to_bitstring(Bits9, Bits3), + + %% bs_private_append: the accumulator itself is a partial bitstring, so it + %% cannot be grown in place + <<1:1, 1, 2, 3>> = build_from_list_onto(Bits1, [1, 2, 3]), + <<5:3>> = build_from_list_onto(Bits3, []), + + %% a /binary segment rejects a partial source rather than truncating it + badarg = expect_error(fun() -> append_binary_to(Bits1, id(<<1, 2>>)) end), + + %% a big integer (> 64 bits) written by the legacy bs_put_integer opcode at a + %% non-byte-aligned offset is not supported: the whole-byte bignum writer + %% cannot place it, so AtomVM rejects it rather than misplacing the bits. + %% BEAM builds it, so check only on AtomVM. + case erlang:system_info(machine) of + "ATOM" -> + %% the interpreter raises unsupported, the JIT badarg; either way it + %% must reject rather than misplace the bits + case expect_error(fun() -> put_wide_int_after(Bits1, id(1 bsl 100)) end) of + unsupported -> ok; + badarg -> ok + end; + _ -> + ok + end, + ok. + +put_wide_int_after(Bits, V) -> + <>. + +copy_bitstring(Bits) -> + <>. + +append_binary_to(Bits, Bin) -> + <>. + +expect_error(Fun) -> + try + Fun(), + no_error + catch + error:Reason -> Reason + end. + +append_byte_to_bitstring(Bits, Byte) -> + <>. + +append_bits_to_bitstring(Bits, More) -> + <>. + +put_bitstring_sized(Bits, Size) -> + <>. + +build_from_list_onto(Acc, []) -> + Acc; +build_from_list_onto(Acc, [H | T]) -> + build_from_list_onto(<>, T). + +id(X) -> + ?MODULE:ext_id(X). + +ext_id(X) -> + X. + make_bin(Size, Val) -> <>. diff --git a/tests/erlang_tests/test_unicode.erl b/tests/erlang_tests/test_unicode.erl index f743cd4297..9f927cf8a0 100644 --- a/tests/erlang_tests/test_unicode.erl +++ b/tests/erlang_tests/test_unicode.erl @@ -27,8 +27,36 @@ start() -> ok = test_to_list_utf8(), ok = test_to_binary_latin1(), ok = test_to_binary_utf8(), + ok = test_improper_tail(), 0. +% Characters are only accepted as list elements: an improper tail may be a +% binary, but never an integer. +test_improper_tail() -> + "ab" = unicode:characters_to_list(id([$a | <<"b">>])), + <<"ab">> = unicode:characters_to_binary(id([$a | <<"b">>])), + <<"ab">> = unicode:characters_to_binary(id([[$a | <<"b">>]])), + + ok = raises_badarg(fun() -> unicode:characters_to_list(id([$a | $b])) end), + ok = raises_badarg(fun() -> unicode:characters_to_list(id([$a | $b]), latin1) end), + ok = raises_badarg(fun() -> unicode:characters_to_list(id([[$a | $b]])) end), + ok = raises_badarg(fun() -> unicode:characters_to_binary(id([$a | $b])) end), + ok = raises_badarg(fun() -> unicode:characters_to_binary(id([[$a | $b]])) end), + ok = raises_badarg(fun() -> unicode:characters_to_binary(id(["x", [$a | $b]])) end), + ok = raises_badarg(fun() -> unicode:characters_to_binary(id([$a | $b]), latin1, latin1) end), + ok = raises_badarg(fun() -> unicode:characters_to_binary(id([$a | bad])) end), + ok. + +raises_badarg(Fun) -> + try Fun() of + Any -> {unexpected, Any} + catch + error:badarg -> ok; + C:E -> {unexpected_exception, C, E} + end. + +id(X) -> X. + test_to_list_latin1() -> "hello" = unicode:characters_to_list(<<"hello">>, latin1), "hello" = unicode:characters_to_list("hello", latin1), diff --git a/tests/libs/estdlib/test_binary.erl b/tests/libs/estdlib/test_binary.erl index bc7039624e..676f187204 100644 --- a/tests/libs/estdlib/test_binary.erl +++ b/tests/libs/estdlib/test_binary.erl @@ -64,4 +64,11 @@ test_hex() -> ?ASSERT_MATCH(RawBinary, binary:decode_hex(<<"48656C6C6F2C2041746F6D564D21">>)), ?ASSERT_EXCEPTION(binary:decode_hex(<<"48656C6C6F2C2041746F6D564D2">>), error, badarg), ?ASSERT_EXCEPTION(binary:decode_hex(<<"ABCDEFGH">>), error, badarg), + % bitstrings that are not a whole number of bytes are rejected rather than + % silently truncated + ?ASSERT_EXCEPTION(binary:decode_hex(<<1:4>>), error, badarg), + ?ASSERT_EXCEPTION(binary:decode_hex(<<"AB", 1:1>>), error, badarg), + ?ASSERT_EXCEPTION(binary:encode_hex(<<1:4>>), error, badarg), + ?ASSERT_EXCEPTION(binary:encode_hex(<<1:4>>, lowercase), error, badarg), + ?ASSERT_EXCEPTION(binary:encode_hex(<<"AB", 1:1>>), error, badarg), ok. diff --git a/tests/libs/estdlib/test_io_lib.erl b/tests/libs/estdlib/test_io_lib.erl index cace878de1..7c8074ffbd 100644 --- a/tests/libs/estdlib/test_io_lib.erl +++ b/tests/libs/estdlib/test_io_lib.erl @@ -61,6 +61,19 @@ test_format() -> ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~p~n", [<<1, 2, 3>>])), "foo: <<1,2,3>>\n"), ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~s~n", [<<1, 2, 3>>])), ?FLT(["foo: ", 1, 2, 3, "\n"])), ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~w~n", [<<1, 2, 3>>])), "foo: <<1,2,3>>\n"), + % bitstrings that are not a whole number of bytes are always printed as + % numbers, even when the leading bytes would be printable + ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~p~n", [<<1:1>>])), "foo: <<1:1>>\n"), + ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~w~n", [<<1:1>>])), "foo: <<1:1>>\n"), + ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~p~n", [<<1, 2, 3:4>>])), "foo: <<1,2,3:4>>\n"), + ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~w~n", [<<1, 2, 3:4>>])), "foo: <<1,2,3:4>>\n"), + ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~p~n", [<<"bar", 1:1>>])), "foo: <<98,97,114,1:1>>\n"), + ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~w~n", [<<"bar", 1:1>>])), "foo: <<98,97,114,1:1>>\n"), + ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~tp~n", [<<1:1>>])), "foo: <<1:1>>\n"), + ?ASSERT_ERROR(io_lib:format("foo: ~s~n", [<<1:1>>]), badarg), + % nested in other terms + ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~p~n", [{<<1:1>>}])), "foo: {<<1:1>>}\n"), + ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~p~n", [[<<1:1>>]])), "foo: [<<1:1>>]\n"), % unprintable strings ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~p~n", [[1, 2, 3]])), "foo: [1,2,3]\n"), ?ASSERT_MATCH(?FLT(io_lib:format("foo: ~s~n", [[1, 2, 3]])), "foo: \1\2\3\n"), diff --git a/tests/libs/jit/jit_tests.erl b/tests/libs/jit/jit_tests.erl index c0f8687ced..cfae69c177 100644 --- a/tests/libs/jit/jit_tests.erl +++ b/tests/libs/jit/jit_tests.erl @@ -687,3 +687,34 @@ fuse_tuple_armv6m_test() -> binary:match(CompiledCode, <<16#7e, 16#68>>) ), ok. + +%% Every backend must be able to emit an unsigned comparison. This is the +%% condition bs_create_bin uses to detect that a bit size accumulation wrapped, +%% and a backend that cannot encode it fails at code generation time, not at +%% compile time: jit_tests has no bs_create_bin coverage, so nothing else here +%% would catch it. +if_block_unsigned_less_than_test_() -> + Backends = [ + jit_aarch64, + jit_x86_64, + jit_riscv32, + jit_riscv64, + jit_arm32, + jit_armv6m, + jit_xtensa, + jit_wasm32 + ], + [ + {atom_to_list(Backend), fun() -> + State0 = Backend:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + {State1, RegA} = Backend:move_to_native_register(State0, {x_reg, 0}), + {State2, RegB} = Backend:move_to_native_register(State1, {x_reg, 1}), + %% Emitting is the assertion: a backend that cannot encode this + %% condition raises function_clause here rather than returning. + State3 = Backend:if_block(State2, {'(unsigned)', RegA, '<', RegB}, fun(BSt0) -> + Backend:move_to_vm_register(BSt0, 0, {x_reg, 2}) + end), + ?assert(is_tuple(State3)) + end} + || Backend <- Backends + ]. diff --git a/tests/libs/jit/jit_x86_64_asm_tests.erl b/tests/libs/jit/jit_x86_64_asm_tests.erl index d3e19d24f1..2b63d0eade 100644 --- a/tests/libs/jit/jit_x86_64_asm_tests.erl +++ b/tests/libs/jit/jit_x86_64_asm_tests.erl @@ -1049,6 +1049,32 @@ jge_rel8_test_() -> ) ]. +jb_test_() -> + [ + ?_assertAsmEqual(<<16#72, 16#f4>>, "jb .-10", jit_x86_64_asm:jb(-10)) + ]. + +jb_rel8_test_() -> + [ + ?_assertEqual( + {1, jit_tests_common:asm(x86_64, <<16#72, 16#05>>, "jb .+7")}, + jit_x86_64_asm:jb_rel8(7) + ) + ]. + +jae_test_() -> + [ + ?_assertAsmEqual(<<16#73, 16#f4>>, "jae .-10", jit_x86_64_asm:jae(-10)) + ]. + +jae_rel8_test_() -> + [ + ?_assertEqual( + {1, jit_tests_common:asm(x86_64, <<16#73, 16#05>>, "jae .+7")}, + jit_x86_64_asm:jae_rel8(7) + ) + ]. + jle_test_() -> [ ?_assertAsmEqual(<<16#7e, 16#f4>>, "jle .-10", jit_x86_64_asm:jle(-10)) diff --git a/tests/libs/jit/jit_xtensa_asm_tests.erl b/tests/libs/jit/jit_xtensa_asm_tests.erl index 4b4c151e95..bf5e3c6dc4 100644 --- a/tests/libs/jit/jit_xtensa_asm_tests.erl +++ b/tests/libs/jit/jit_xtensa_asm_tests.erl @@ -202,6 +202,20 @@ blt_test_() -> ) ]. +bltu_test_() -> + [ + ?_assertAsmEqual( + <<16#003347:24/little>>, "bltu a3, a4, . + 4", jit_xtensa_asm:bltu(a3, a4, 0) + ) + ]. + +bgeu_test_() -> + [ + ?_assertAsmEqual( + <<16#00b347:24/little>>, "bgeu a3, a4, . + 4", jit_xtensa_asm:bgeu(a3, a4, 0) + ) + ]. + beqi_test_() -> [ ?_assertAsmEqual( diff --git a/tests/test-bitstring.c b/tests/test-bitstring.c new file mode 100644 index 0000000000..ced4a0a4e2 --- /dev/null +++ b/tests/test-bitstring.c @@ -0,0 +1,210 @@ +/* + * This file is part of AtomVM. + * + * Copyright 2026 Paul Guyot + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later + */ + +#include +#include +#include + +#include "bitstring.h" +#include "context.h" +#include "globalcontext.h" +#include "memory.h" +#include "term.h" +#include "utils.h" + +// A copy must touch exactly the destination bytes that hold the copied bits, +// and read exactly the source bytes that hold them. Both buffers are bracketed +// with guard bytes so an off-by-one byte is caught here rather than by a +// sanitizer on an unrelated allocation. +#define GUARD 0x5A + +struct guarded +{ + uint8_t before[4]; + uint8_t data[16]; + uint8_t after[4]; +}; + +static void guarded_init(struct guarded *g, uint8_t fill) +{ + memset(g->before, GUARD, sizeof(g->before)); + memset(g->data, fill, sizeof(g->data)); + memset(g->after, GUARD, sizeof(g->after)); +} + +static void guarded_check(const struct guarded *g) +{ + for (size_t i = 0; i < sizeof(g->before); i++) { + assert(g->before[i] == GUARD); + } + for (size_t i = 0; i < sizeof(g->after); i++) { + assert(g->after[i] == GUARD); + } +} + +// Reference implementation: copy bits_count bits from the start of src to +// bits_offset in dst, one bit at a time, touching nothing else. +static void reference_copy_bits( + uint8_t *dst, size_t bits_offset, const uint8_t *src, size_t bits_count) +{ + for (size_t i = 0; i < bits_count; i++) { + size_t dst_bit = bits_offset + i; + uint8_t mask = (uint8_t) (1U << (7 - (dst_bit % 8))); + if (src[i / 8] & (uint8_t) (1U << (7 - (i % 8)))) { + dst[dst_bit / 8] |= mask; + } else { + dst[dst_bit / 8] &= (uint8_t) ~mask; + } + } +} + +static void check_copy_bits(size_t bits_offset, size_t bits_count, uint8_t src_fill) +{ + struct guarded src; + guarded_init(&src, src_fill); + + struct guarded got; + struct guarded expected; + // Two different fills, so a byte the copy must not touch differs from the + // byte it would be overwritten with. + guarded_init(&got, 0xC3); + guarded_init(&expected, 0xC3); + + reference_copy_bits(expected.data, bits_offset, src.data, bits_count); + bitstring_copy_bits(got.data, bits_offset, src.data, bits_count); + + // The bytes past the last copied bit must be untouched, and no guard byte + // on either side of either buffer may have moved. + assert(memcmp(got.data, expected.data, sizeof(got.data)) == 0); + guarded_check(&got); + guarded_check(&src); +} + +void test_copy_bits_boundaries(void) +{ + // Exact byte boundaries at an unaligned destination: the last copied bit + // completes the destination byte, so nothing past it may be read or written. + check_copy_bits(1, 7, 0xFF); + check_copy_bits(1, 7, 0x00); + // The source is consumed to its last bit and must not be read past it. + check_copy_bits(1, 8, 0xFF); + check_copy_bits(1, 8, 0x00); + // A zero-bit copy touches neither buffer. + check_copy_bits(0, 0, 0xFF); + check_copy_bits(1, 0, 0xFF); + check_copy_bits(7, 0, 0xFF); + + // Sweep offsets and counts, aligned and unaligned, including whole bytes. + for (size_t offset = 0; offset < 16; offset++) { + for (size_t count = 0; count <= 64; count++) { + check_copy_bits(offset, count, 0xFF); + check_copy_bits(offset, count, 0x00); + check_copy_bits(offset, count, 0xA5); + } + } +} + +// Copying into a destination sized to exactly the bits it holds, as a refc +// binary is: a byte of slack would hide a one-byte overrun. +void test_copy_bits_exact_allocation(void) +{ + for (size_t bits = 1; bits <= 24; bits++) { + size_t bytes = (bits + 7) / 8; + uint8_t *dst = calloc(bytes, 1); + assert(dst != NULL); + uint8_t src[4] = { 0xFF, 0xFF, 0xFF, 0xFF }; + // starting at bit 1 leaves the copy ending mid-byte or exactly on a + // byte boundary depending on the count + if (1 + bits <= bytes * 8) { + bitstring_copy_bits(dst, 1, src, bits); + } + free(dst); + } +} + +void test_print_bitstring(void) +{ + GlobalContext *glb = globalcontext_new(); + Context *ctx = context_new(glb); + + // Reserve every term this function builds in one go: memory_ensure_free may + // collect, and the terms below are plain C locals rather than roots, so a + // second reservation would leave the earlier ones dangling. + size_t heap_size = 3 * TERM_BOXED_SUB_BINARY_SIZE + 3 * term_binary_heap_size(1) + + 2 * term_binary_heap_size(2); + assert(memory_ensure_free(ctx, heap_size) == MEMORY_GC_OK); + + // <<1:1>>: no complete byte, a single trailing bit + term bin1 = term_create_empty_binary(1, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin1))[0] = 0x80; + term bits1 = term_alloc_sub_binary_bits(bin1, 0, 0, 1, &ctx->heap); + char buf[64]; + int len = term_snprint(buf, sizeof(buf), bits1, glb); + assert(len > 0); + assert(strcmp(buf, "<<1:1>>") == 0); + + // <<255, 5:7>>: a complete byte followed by a partial one + term bin2 = term_create_empty_binary(2, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin2))[0] = 0xFF; + ((uint8_t *) term_binary_data(bin2))[1] = (uint8_t) (5 << 1); + term bits2 = term_alloc_sub_binary_bits(bin2, 0, 1, 7, &ctx->heap); + len = term_snprint(buf, sizeof(buf), bits2, glb); + assert(len > 0); + assert(strcmp(buf, "<<255,5:7>>") == 0); + + // a printable byte followed by trailing bits: OTP prints <<97,1:3>>, not + // <<"a",1:3>>, so trailing bits defeat the printable-string form entirely + term bin3 = term_create_empty_binary(2, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin3))[0] = 'a'; + ((uint8_t *) term_binary_data(bin3))[1] = (uint8_t) (1 << 5); + term bits3 = term_alloc_sub_binary_bits(bin3, 0, 1, 3, &ctx->heap); + len = term_snprint(buf, sizeof(buf), bits3, glb); + assert(len > 0); + assert(strcmp(buf, "<<97,1:3>>") == 0); + + // a byte-aligned bitstring still prints as a plain binary, and a printable + // one still prints as a quoted string + term bin4 = term_create_empty_binary(1, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin4))[0] = 0; + len = term_snprint(buf, sizeof(buf), bin4, glb); + assert(len > 0); + assert(strcmp(buf, "<<0>>") == 0); + + term bin5 = term_create_empty_binary(1, &ctx->heap, glb); + ((uint8_t *) term_binary_data(bin5))[0] = 42; + len = term_snprint(buf, sizeof(buf), bin5, glb); + assert(len > 0); + assert(strcmp(buf, "<<\"*\">>") == 0); + + context_destroy(ctx); + globalcontext_destroy(glb); +} + +int main(int argc, char **argv) +{ + UNUSED(argc); + UNUSED(argv); + + test_copy_bits_boundaries(); + test_copy_bits_exact_allocation(); + test_print_bitstring(); + + return EXIT_SUCCESS; +} diff --git a/tests/test.c b/tests/test.c index 18ae322d2a..87212e15af 100644 --- a/tests/test.c +++ b/tests/test.c @@ -416,9 +416,11 @@ struct Test tests[] = { TEST_CASE(test_timeout_not_integer), TEST_CASE(test_undef), TEST_CASE(test_bs), + TEST_CASE(test_bs_gettail), TEST_CASE(test_bs_int), TEST_CASE(test_bs_int_any_flags), TEST_CASE(test_bs_int_unaligned), + TEST_CASE_ATOMVM_ONLY(test_bs_size_limits, 0), TEST_CASE(test_bs_start_match_live), TEST_CASE(test_bs_utf), TEST_CASE_EXPECTED(bs_append_extra_words, 1),