Fix bytes_required over-allocation for negative powers of two

pyiceberg/utils/decimal.py

@@ -49,20 +49,15 @@ def unscaled_to_decimal(unscaled: int, scale: int) -> Decimal:
 
 
 def bytes_required(value: int | Decimal) -> int:
-    """Return the minimum number of bytes needed to serialize a decimal or unscaled value.
-
-    Args:
-        value (int | Decimal): a Decimal value or unscaled int value.
-
-    Returns:
-        int: the minimum number of bytes needed to serialize the value.
-    """
-    if isinstance(value, int):
-        return (value.bit_length() + 8) // 8
-    elif isinstance(value, Decimal):
-        return (decimal_to_unscaled(value).bit_length() + 8) // 8
-
-    raise ValueError(f"Unsupported value: {value}")
+    """Return the minimum number of bytes needed to serialize a decimal or unscaled value."""
+    if isinstance(value, Decimal):
+        value = decimal_to_unscaled(value)
+    if not isinstance(value, int):
+        raise ValueError(f"Unsupported value: {value}")
+    if value == 0:
+        return 1
+    bits = (~value).bit_length() + 1 if value < 0 else value.bit_length() + 1
+    return (bits + 7) // 8
 
 
 def decimal_to_bytes(value: Decimal, byte_length: int | None = None) -> bytes:

tests/utils/test_decimal.py

@@ -18,7 +18,7 @@
 
 import pytest
 
-from pyiceberg.utils.decimal import decimal_required_bytes, decimal_to_bytes
+from pyiceberg.utils.decimal import bytes_required, decimal_required_bytes, decimal_to_bytes
 
 
 def test_decimal_required_bytes() -> None:
@@ -47,3 +47,15 @@ def test_decimal_to_bytes() -> None:
     # 2 bytes has a minimum of -32,768 and a maximum value of 32,767 (inclusive).
     assert decimal_to_bytes(Decimal("32767.")) == b"\x7f\xff"
     assert decimal_to_bytes(Decimal("32768.")) == b"\x00\x80\x00"
+
+
+def test_bytes_required() -> None:
+    # Test boundary limits for signed powers of two
+    assert bytes_required(0) == 1
+    assert bytes_required(127) == 1
+    assert bytes_required(128) == 2
+    assert bytes_required(-127) == 1
+    assert bytes_required(-128) == 1  # Fixes #3522: Should fit perfectly into 1 byte (b\x80)
+    assert bytes_required(-129) == 2
+    assert bytes_required(-32768) == 2  # Fixes #3522: Should fit perfectly into 2 bytes
+    assert bytes_required(-32769) == 3