From 97d0e0141df7493a854a43407c0440c82b431071 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Fri, 4 Sep 2026 00:16:56 -0600 Subject: [PATCH 1/5] Support Parsers 2 and 3 for number parsing; parse typed integers exactly under allownan Parsers 3 removed xparse2/Options, so JSON.jl could not load against it. JSON now validates the number token against the JSON grammar itself (scannumber), which also accumulates the Int64 value, and hands the span to a small backend surface selected at load time: the Parsers 3 kernels (parsefloat/parsebigint/parsenext) or the existing Parsers 2 xparse2 calls. Drops Parsers 1; requires Julia 1.10. parsenumber receives the requested type, so a concrete integer target parses the digits exactly instead of through Float64 (fixes #478). Untyped parsing with allownan=true still returns Float64 for every number. Adds a Parsers 2 CI lane. Tests for the long-mantissa float bugs are gated on Parsers >= 2.8.8, which will carry the fix for Parsers 2 users. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/CI.yml | 15 ++- Project.toml | 4 +- docs/src/reading.md | 2 +- src/lazy.jl | 267 ++++++++++++++++++++++----------------- src/parse.jl | 2 +- test/parse.jl | 46 ++++++- 6 files changed, 211 insertions(+), 125 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2fb6ac73..1cf05db1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -9,8 +9,14 @@ on: jobs: test: - name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}${{ matrix.parsers && format(' - Parsers {0}', matrix.parsers) || '' }} runs-on: ${{ matrix.os }} + env: + # Arrow (test dep) -> TimeZones >= 1.6 -> InlineStrings, whose released + # versions cap Parsers at 2, so resolving Parsers 3 falls back to TimeZones + # 1.5.9, whose build step needs a single thread pool (JuliaTime/TimeZones.jl#429). + # Remove once InlineStrings releases its Parsers 3 support. + JULIA_NUM_THREADS: '1' strategy: fail-fast: false matrix: @@ -30,6 +36,10 @@ jobs: - os: ubuntu-latest arch: x86 version: 1 + - os: ubuntu-latest + arch: x64 + version: 1 + parsers: '2' steps: - uses: actions/checkout@v7 - uses: julia-actions/setup-julia@v3 @@ -38,6 +48,9 @@ jobs: arch: ${{ matrix.arch }} - uses: julia-actions/cache@v3 - uses: julia-actions/julia-buildpkg@v1 + - name: Pin Parsers ${{ matrix.parsers }} + if: matrix.parsers + run: julia --project=. -e 'using Pkg; Pkg.add(name="Parsers", version="${{ matrix.parsers }}")' - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v7 diff --git a/Project.toml b/Project.toml index de0e3b26..fe28bbf3 100644 --- a/Project.toml +++ b/Project.toml @@ -20,10 +20,10 @@ JSONArrowExt = ["ArrowTypes"] [compat] Arrow = "2.8.0" ArrowTypes = "2.2" -Parsers = "1, 2" +Parsers = "2, 3" PrecompileTools = "1" StructUtils = "2.8.4" -julia = "1.9" +julia = "1.10" [extras] Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" diff --git a/docs/src/reading.md b/docs/src/reading.md index 25a10ba2..1933c0ce 100644 --- a/docs/src/reading.md +++ b/docs/src/reading.md @@ -25,7 +25,7 @@ The core JSON parsing machinery is hence built around having an `AbstractVector{ Each entrypoint function first calls [`JSON.lazy`](@ref), which will consume the JSON input until the type of the next JSON value can be identified (`{` for objects, `[` for arrays, `"` for strings, `t` for true, `f` for false, `n` for null, and `-` or a digit for numbers). [`JSON.lazy`](@ref) returns a [`JSON.LazyValue`](@ref), which wraps the JSON input buffer (`AbstractVector{UInt8}` or `AbstractString`), and marks the byte position the value starts at, the type of the value, and any keyword arguments that were provided that may affect parsing. Currently supported parsing-specific keyword arguments to [`JSON.lazy`](@ref) (and thus all other entrypoint functions) include: - - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing + - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, all numbers are parsed as `Float64` unless a specific numeric type is requested, in which case the number is parsed exactly as that type - `ninf::String = "-Infinity"`: the string that will be used to parse `-Inf` if `allownan=true` - `inf::String = "Infinity"`: the string that will be used to parse `Inf` if `allownan=true` - `nan::String = "NaN"`: the string that will be sued to parse `NaN` if `allownan=true` diff --git a/src/lazy.jl b/src/lazy.jl index e1226994..857e5b8f 100644 --- a/src/lazy.jl +++ b/src/lazy.jl @@ -54,7 +54,7 @@ In this example, we only parsed as much of the `very_large_json_object` as was r Then we fully materialized `y` into `z`, which is now a normal Julia object. We can now mutate or access values in `z`. Currently supported keyword arguments include: - - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing + - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, all numbers are parsed as `Float64` unless a specific numeric type is requested, in which case the number is parsed exactly as that type - `ninf::String = "-Infinity"`: the string that will be used to parse `-Inf` if `allownan=true` - `inf::String = "Infinity"`: the string that will be used to parse `Inf` if `allownan=true` - `nan::String = "NaN"`: the string that will be sued to parse `NaN` if `allownan=true` @@ -527,33 +527,92 @@ function parsestring(x::LazyValue) end # core JSON number parsing function -# we rely on functionality in Parsers to help infer what kind -# of number we're parsing; valid return types include: -# Int64, BigInt, Float64 or BigFloat -const INT64_OVERFLOW_VAL = div(typemax(Int64), 10) -const INT64_OVERFLOW_DIGIT = typemax(Int64) % 10 +# JSON validates the number token against the JSON grammar itself, then hands +# the exact byte span to the Parsers kernels for conversion; valid return +# types include: Int64, BigInt, Float64 or BigFloat + +# match `special` at `pos`; returns the position after the match, or 0 if no match +@inline function matchspecial(buf, pos, len, special::String) + bytes = codeunits(special) + n = length(bytes) + (n == 0 || pos + n - 1 > len) && return 0 + for i = 1:n + getbyte(buf, pos + i - 1) == @inbounds(bytes[i]) || return 0 + end + return pos + n +end -macro check_special(special, value) - esc(quote - pos = startpos +# integer digits are accumulated as a negative number so typemin(Int64) fits +const INT64_MIN_DIV10 = div(typemin(Int64), 10) +const INT64_MIN_LASTDIGIT = -rem(typemin(Int64), 10) +# Parsers 2's xparse2 finds the end of a float itself, so the scanner stops at +# the first '.'/'e'; the Parsers 3 kernels need the exact span, so it scans to the end +const SCANFLOATTAIL = !isdefined(Parsers, :xparse2) + +# scan a JSON number token starting at `startpos`; returns the position after +# the token (or after the integer part if !SCANFLOATTAIL), whether it has a +# fraction or exponent, the integer value, and whether that value overflowed Int64; +# nextpos == startpos means the bytes at startpos are not a valid JSON number +@inline function scannumber(buf, startpos, len) + pos = startpos + b = getbyte(buf, pos) + isneg = b == UInt8('-') + # leading '+' only reaches here with allownan=true (see `_lazy`) + if isneg || b == UInt8('+') + pos += 1 + pos > len && return startpos, false, Int64(0), false b = getbyte(buf, pos) - bytes = codeunits($special) - i = 1 - while b == @inbounds(bytes[i]) - pos += 1 - i += 1 - i > length(bytes) && break - if pos > len - error = UnexpectedEOF - @goto invalid + end + val = Int64(0) + overflow = false + # integer part; leading zeros are invalid JSON + if b == UInt8('0') + pos += 1 + pos <= len && UInt8('0') <= getbyte(buf, pos) <= UInt8('9') && return startpos, false, val, false + elseif UInt8('1') <= b <= UInt8('9') + while true + digit = Int64(b - UInt8('0')) + if val < INT64_MIN_DIV10 || (val == INT64_MIN_DIV10 && digit > INT64_MIN_LASTDIGIT) + overflow = true + else + val = Int64(10) * val - digit end + pos += 1 + pos > len && break b = getbyte(buf, pos) - i += 1 + UInt8('0') <= b <= UInt8('9') || break end - if i > length(bytes) - return NumberResult($value), pos + else + return startpos, false, val, false + end + isfloat = false + # fraction: at least one digit required after '.' + if pos <= len && getbyte(buf, pos) == UInt8('.') + isfloat = true + pos += 1 + (pos > len || !(UInt8('0') <= getbyte(buf, pos) <= UInt8('9'))) && return startpos, false, val, false + SCANFLOATTAIL || return pos, true, val, overflow + while pos <= len && UInt8('0') <= getbyte(buf, pos) <= UInt8('9') + pos += 1 end - end) + end + # exponent: optional sign, at least one digit + if pos <= len && (getbyte(buf, pos) == UInt8('e') || getbyte(buf, pos) == UInt8('E')) + isfloat = true + SCANFLOATTAIL || return pos, true, val, overflow + pos += 1 + if pos <= len && (getbyte(buf, pos) == UInt8('+') || getbyte(buf, pos) == UInt8('-')) + pos += 1 + end + (pos > len || !(UInt8('0') <= getbyte(buf, pos) <= UInt8('9'))) && return startpos, false, val, false + while pos <= len && UInt8('0') <= getbyte(buf, pos) <= UInt8('9') + pos += 1 + end + end + if !isneg + val == typemin(Int64) ? (overflow = true) : (val = -val) + end + return pos, isfloat, val, overflow end const INT = 0x00 @@ -579,109 +638,81 @@ isfloat(x::NumberResult) = x.tag == FLOAT isbigint(x::NumberResult) = x.tag == BIGINT isbigfloat(x::NumberResult) = x.tag == BIGFLOAT -@inline function parsenumber(x::LazyValue) +# Parsers converts the validated token span; Parsers 2 and 3 have different APIs +@inline numberbytes(buf::AbstractString) = codeunits(buf) +@inline numberbytes(buf) = buf + +@static if isdefined(Parsers, :xparse2) + # Parsers 2: `xparse2` parses the longest number prefix of buf[first:len] + @inline function parsefloat64(buf, first, nextpos, len, allownan) + res = Parsers.xparse2(Float64, buf, first, len) + nextpos = first + Int(res.tlen) + if !allownan && Parsers.specialvalue(res.code) + # overflowed to Inf; promote to BigFloat + bres = Parsers.xparse2(BigFloat, buf, first, len) + Parsers.invalid(bres.code) || return NumberResult(bres.val), nextpos + end + Parsers.invalid(res.code) && invalid(InvalidNumber, buf, first, "number") + return NumberResult(res.val), nextpos + end + @inline parsebigint(buf, first, last) = NumberResult(Parsers.xparse2(BigInt, buf, first, last).val) + # native `[+-]NaN`, `[+-]Inf`, `[+-]Infinity` spellings (case-insensitive) + @inline function parsenativespecial(buf, pos, len) + res = Parsers.xparse2(Float64, buf, pos, len) + (Parsers.invalid(res.code) || isfinite(res.val)) && return nothing + return NumberResult(res.val), pos + Int(res.tlen) + end +else + # Parsers 3: the kernels convert exactly buf[first:nextpos-1] + @inline function parsefloat64(buf, first, nextpos, len, allownan) + bytes = numberbytes(buf) + val, code = Parsers.parsefloat(Float64, bytes, first, nextpos - 1) + if !allownan && code == Parsers.RC_OVERFLOW + # promote to BigFloat + return NumberResult(Parsers.parse(BigFloat, bytes, first, nextpos - 1)), nextpos + end + return NumberResult(val), nextpos + end + @inline parsebigint(buf, first, last) = NumberResult(Parsers.parsebigint(numberbytes(buf), first, last)[1]) + # native `[+-]NaN`, `[+-]Inf`, `[+-]Infinity` spellings (case-insensitive) + @inline function parsenativespecial(buf, pos, len) + val, nextpos, code = Parsers.parsenext(Float64, numberbytes(buf), pos, len) + (code != Parsers.RC_OK || isfinite(val)) && return nothing + return NumberResult(val), nextpos + end +end + +# `T` is the type requested for this value (`Any` when materializing untyped); +# with `allownan=true`, all numbers materialize as `Float64` unless a specific +# type was requested, in which case the token is parsed exactly for that type +@inline function parsenumber(x::LazyValue, ::Type{T}=Any) where {T} buf = getbuf(x) - pos::Int = getpos(x) + startpos::Int = getpos(x) len = getlength(buf) opts = getopts(x) - b = getbyte(buf, pos) - startpos = pos - isneg = isfloat = overflow = false - if !opts.allownan - val = Int64(0) - isneg = b == UInt8('-') - if isneg || b == UInt8('+') # spec doesn't allow leading +, but we do - pos += 1 - if pos > len - error = UnexpectedEOF - @goto invalid - end - b = getbyte(buf, pos) - end - # Parse integer part, check for leading zeros (invalid JSON) - if b == UInt8('0') - pos += 1 - if pos <= len - b = getbyte(buf, pos) - if UInt8('0') <= b <= UInt8('9') - error = InvalidNumber - @goto invalid - end - end - elseif UInt8('1') <= b <= UInt8('9') - while UInt8('0') <= b <= UInt8('9') - digit = Int64(b - UInt8('0')) - if val > INT64_OVERFLOW_VAL || (val == INT64_OVERFLOW_VAL && digit > INT64_OVERFLOW_DIGIT) - overflow = true - break - end - val = Int64(10) * val + digit - pos += 1 - pos > len && break - b = getbyte(buf, pos) - end - if overflow - bval = BigInt(val) - while UInt8('0') <= b <= UInt8('9') - digit = BigInt(b - UInt8('0')) - bval = BigInt(10) * bval + digit - pos += 1 - pos > len && break - b = getbyte(buf, pos) - end - end - else - error = InvalidNumber - @goto invalid - end - # Check for decimal or exponent - if b == UInt8('.') || b == UInt8('e') || b == UInt8('E') - isfloat = true - # in strict JSON spec, we need at least one digit after the decimal - if b == UInt8('.') - pos += 1 - if pos > len - error = UnexpectedEOF - @goto invalid - end - b = getbyte(buf, pos) - if !(UInt8('0') <= b <= UInt8('9')) - error = InvalidNumber - @goto invalid - end - end - end + if opts.allownan + # check for configured NaN, Inf, -Inf spellings + pos = matchspecial(buf, startpos, len, opts.nan) + pos != 0 && return NumberResult(NaN), pos + pos = matchspecial(buf, startpos, len, opts.inf) + pos != 0 && return NumberResult(Inf), pos + pos = matchspecial(buf, startpos, len, opts.ninf) + pos != 0 && return NumberResult(-Inf), pos end - if isfloat || opts.allownan + nextpos, isfloat, val, overflow = scannumber(buf, startpos, len) + if nextpos == startpos if opts.allownan - # check for NaN, Inf, -Inf - @check_special(opts.nan, NaN) - @check_special(opts.inf, Inf) - @check_special(opts.ninf, -Inf) - end - res = Parsers.xparse2(Float64, buf, startpos, len) - if !opts.allownan && Parsers.specialvalue(res.code) - # if we overflowed, then let's try BigFloat - bres = Parsers.xparse2(BigFloat, buf, startpos, len) - if !Parsers.invalid(bres.code) - return NumberResult(bres.val), startpos + Int(bres.tlen) - end - end - if Parsers.invalid(res.code) - error = InvalidNumber - @goto invalid - end - return NumberResult(res.val), Int(startpos + res.tlen) - else - if overflow - return NumberResult(isneg ? -bval : bval), pos - else - return NumberResult(isneg ? -val : val), pos + res = parsenativespecial(buf, startpos, len) + res === nothing || return res end + invalid(InvalidNumber, buf, startpos, "number") end - -@label invalid - invalid(InvalidNumber, buf, startpos, "number") + if isfloat || (opts.allownan && T === Any) + return parsefloat64(buf, startpos, nextpos, len, opts.allownan) + end + overflow || return NumberResult(val), nextpos + # promote to BigInt + return parsebigint(buf, startpos, nextpos - 1), nextpos end # efficiently skip over a JSON value diff --git a/src/parse.jl b/src/parse.jl index 36dba098..3628db2c 100644 --- a/src/parse.jl +++ b/src/parse.jl @@ -471,7 +471,7 @@ function StructUtils.lift(style::JSONReadStyle, ::Type{T}, x::LazyValues, tags=( end return str, pos elseif type == JSONTypes.NUMBER - num, pos = parsenumber(x) + num, pos = parsenumber(x, T) if isint(num) T === Int64 && return num.int, pos int, _ = StructUtils.lift(style, T, num.int, tags) diff --git a/test/parse.jl b/test/parse.jl index bbea56ba..58e4cf52 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -5,6 +5,9 @@ struct RefValueStyle <: JSON.JSONStyle end struct DateStringStyle <: JSON.JSONStyle end struct DateObjectStyle <: JSON.JSONStyle end struct DateMaterializedObjectStyle <: JSON.JSONStyle end +struct TestAllownanInt + a::Int64 +end struct A a::Int @@ -407,6 +410,27 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ # allownan for parsing normally invalid json values @test JSON.parse("NaN"; allownan=true) === NaN @test JSON.parse("Inf"; inf="Inf", allownan=true) === Inf + @test JSON.parse("Infinity"; allownan=true) === Inf + @test JSON.parse("-Infinity"; allownan=true) === -Inf + @test JSON.parse("Inf"; allownan=true) === Inf + @test JSON.parse("-Inf"; allownan=true) === -Inf + @test isequal(JSON.parse("[Inf,NaN,-Infinity]"; allownan=true), [Inf, NaN, -Inf]) + @test_throws ArgumentError JSON.parse("-"; allownan=true) + @test_throws ArgumentError JSON.parse("+"; allownan=true) + @test JSON.parse("+1"; allownan=true) === 1.0 + @test JSON.parse("+Inf"; allownan=true) === Inf + # allownan=true materializes all numbers as Float64 when no type is requested... + @test JSON.parse("1"; allownan=true) === 1.0 + @test JSON.parse("[1,2.5]"; allownan=true) == [1.0, 2.5] + @test JSON.parse("[1]", Vector{Any}; allownan=true) == [1.0] + @test JSON.parse("1", Float64; allownan=true) === 1.0 + @test JSON.parse("[1,2]", Vector{Float64}; allownan=true) == [1.0, 2.0] + # ...but a requested type parses the token exactly (#478) + @test JSON.parse(string(typemax(Int64)), Int64; allownan=true) === typemax(Int64) + @test JSON.parse(string(typemin(Int64)), Int64; allownan=true) === typemin(Int64) + @test JSON.parse(string(typemax(UInt64)), UInt64; allownan=true) === typemax(UInt64) + @test JSON.parse(string(typemax(Int128)), Int128; allownan=true) === typemax(Int128) + @test JSON.parse("{\"a\":$(typemax(Int64))}", TestAllownanInt; allownan=true).a === typemax(Int64) # jsonlines support @test JSON.parse("1"; jsonlines=true) == [1] @test JSON.parse("1 \t"; jsonlines=true) == [1] @@ -493,6 +517,23 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test JSON.parse("9223372036854775805") === 9223372036854775805 @test JSON.parse("9223372036854775806") === 9223372036854775806 @test JSON.parse("9223372036854775807") === 9223372036854775807 + @test JSON.parse("-9223372036854775808") === typemin(Int64) + x = JSON.parse("-9223372036854775809") + @test x isa BigInt && x == -9223372036854775809 + # long mantissas and negative zeros that Parsers < 2.8.8 crashed on or misparsed + if pkgversion(JSON.Parsers) >= v"2.8.8" + # (Base.parse(Float64, ...) itself throws on this one, so compare directly) + @test JSON.parse("295574326048237151328925.8099133506971425945276929554326e-440") === 0.0 + for source in ("-773185451005006305224330936226383685.195e3", + "0.72741733550162454424961322208253163690E+61", + "-75738806850214820018096823497.7e229") + @test JSON.parse(source) === Base.parse(Float64, source) + @test JSON.parse(Vector{UInt8}(codeunits(source))) === Base.parse(Float64, source) + end + @test JSON.parse("-0e291") === -0.0 + @test JSON.parse("-0e292") === -0.0 + @test JSON.parse("-0.0e100") === -0.0 + end # promote to BigInt x = JSON.parse("9223372036854775808") # only == here because BigInt don't compare w/ === @@ -501,6 +542,7 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test x isa BigInt && x == 170141183460469231731687303715884105727 x = JSON.parse("170141183460469231731687303715884105728") @test x isa BigInt && x == 170141183460469231731687303715884105728 + @test JSON.parse("170141183460469231731687303715884105728") !== x # BigFloat @test JSON.parse("1.7976931348623157e310") == big"1.7976931348623157e310" @@ -520,8 +562,8 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test JSON.parse("0e292") === 0.0 @test JSON.parse("0e347") == big"0.0" @test JSON.parse("0e348") == big"0.0" - @test JSON.parse("-0e291") === 0.0 - @test JSON.parse("-0e292") === 0.0 + @test JSON.parse("-0e291") == 0.0 + @test JSON.parse("-0e292") == 0.0 @test JSON.parse("-0e347") == big"0.0" @test JSON.parse("-0e348") == big"0.0" @test JSON.parse("2e-324") === 0.0 From bcb8e334682e8135c08913781bf8200832cf752e Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Fri, 4 Sep 2026 11:46:55 -0600 Subject: [PATCH 2/5] fix(parse): preserve exact integer tokens Parse decimal and exponent forms for requested integer types without an intermediate Float64 conversion. Preserve Float64 output for untyped allownan array values, require the first compatible Parsers 2 release, and test that exact floor in CI. --- .github/workflows/CI.yml | 2 +- Project.toml | 2 +- docs/src/reading.md | 2 +- src/lazy.jl | 81 +++++++++++++++++++++++++++++++++++++--- test/parse.jl | 12 +++++- 5 files changed, 90 insertions(+), 9 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1cf05db1..72807ba3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -39,7 +39,7 @@ jobs: - os: ubuntu-latest arch: x64 version: 1 - parsers: '2' + parsers: '2.8.8' steps: - uses: actions/checkout@v7 - uses: julia-actions/setup-julia@v3 diff --git a/Project.toml b/Project.toml index fe28bbf3..5258b2d2 100644 --- a/Project.toml +++ b/Project.toml @@ -20,7 +20,7 @@ JSONArrowExt = ["ArrowTypes"] [compat] Arrow = "2.8.0" ArrowTypes = "2.2" -Parsers = "2, 3" +Parsers = "2.8.8, 3" PrecompileTools = "1" StructUtils = "2.8.4" julia = "1.10" diff --git a/docs/src/reading.md b/docs/src/reading.md index 1933c0ce..dde79c3d 100644 --- a/docs/src/reading.md +++ b/docs/src/reading.md @@ -25,7 +25,7 @@ The core JSON parsing machinery is hence built around having an `AbstractVector{ Each entrypoint function first calls [`JSON.lazy`](@ref), which will consume the JSON input until the type of the next JSON value can be identified (`{` for objects, `[` for arrays, `"` for strings, `t` for true, `f` for false, `n` for null, and `-` or a digit for numbers). [`JSON.lazy`](@ref) returns a [`JSON.LazyValue`](@ref), which wraps the JSON input buffer (`AbstractVector{UInt8}` or `AbstractString`), and marks the byte position the value starts at, the type of the value, and any keyword arguments that were provided that may affect parsing. Currently supported parsing-specific keyword arguments to [`JSON.lazy`](@ref) (and thus all other entrypoint functions) include: - - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, all numbers are parsed as `Float64` unless a specific numeric type is requested, in which case the number is parsed exactly as that type + - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, untyped numbers are parsed as `Float64`. A requested integer type is parsed exactly from the number token - `ninf::String = "-Infinity"`: the string that will be used to parse `-Inf` if `allownan=true` - `inf::String = "Infinity"`: the string that will be used to parse `Inf` if `allownan=true` - `nan::String = "NaN"`: the string that will be sued to parse `NaN` if `allownan=true` diff --git a/src/lazy.jl b/src/lazy.jl index 857e5b8f..eede04ab 100644 --- a/src/lazy.jl +++ b/src/lazy.jl @@ -54,7 +54,7 @@ In this example, we only parsed as much of the `very_large_json_object` as was r Then we fully materialized `y` into `z`, which is now a normal Julia object. We can now mutate or access values in `z`. Currently supported keyword arguments include: - - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, all numbers are parsed as `Float64` unless a specific numeric type is requested, in which case the number is parsed exactly as that type + - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, untyped numbers are parsed as `Float64`. A requested integer type is parsed exactly from the number token - `ninf::String = "-Infinity"`: the string that will be used to parse `-Inf` if `allownan=true` - `inf::String = "Infinity"`: the string that will be used to parse `Inf` if `allownan=true` - `nan::String = "NaN"`: the string that will be sued to parse `NaN` if `allownan=true` @@ -662,6 +662,11 @@ isbigfloat(x::NumberResult) = x.tag == BIGFLOAT (Parsers.invalid(res.code) || isfinite(res.val)) && return nothing return NumberResult(res.val), pos + Int(res.tlen) end + @inline function exactintegerend(buf, first, nextpos, len) + res = Parsers.xparse2(Float64, buf, first, len) + Parsers.invalid(res.code) && invalid(InvalidNumber, buf, first, "number") + return first + Int(res.tlen) + end else # Parsers 3: the kernels convert exactly buf[first:nextpos-1] @inline function parsefloat64(buf, first, nextpos, len, allownan) @@ -680,11 +685,75 @@ else (code != Parsers.RC_OK || isfinite(val)) && return nothing return NumberResult(val), nextpos end + @inline exactintegerend(buf, first, nextpos, len) = nextpos +end + +# Convert a validated decimal or exponent token to an exact integer. This avoids +# losing precision by first materializing the token as Float64. +function parseexactinteger(buf, first, nextpos, len, ::Type{T}) where {T} + nextpos = exactintegerend(buf, first, nextpos, len) + pos = first + b = getbyte(buf, pos) + isneg = b == UInt8('-') + if isneg || b == UInt8('+') + pos += 1 + end + + coefficient = BigInt(0) + fractiondigits = 0 + infraction = false + while pos < nextpos + b = getbyte(buf, pos) + if UInt8('0') <= b <= UInt8('9') + coefficient = coefficient * 10 + (b - UInt8('0')) + infraction && (fractiondigits += 1) + elseif b == UInt8('.') + infraction = true + else + break + end + pos += 1 + end + + exponent = BigInt(0) + if pos < nextpos + pos += 1 # skip 'e' or 'E' + b = getbyte(buf, pos) + expneg = b == UInt8('-') + if expneg || b == UInt8('+') + pos += 1 + end + while pos < nextpos + exponent = exponent * 10 + (getbyte(buf, pos) - UInt8('0')) + pos += 1 + end + expneg && (exponent = -exponent) + end + + iszero(coefficient) && return NumberResult(coefficient), nextpos + scale = exponent - fractiondigits + if scale >= 0 + if isconcretetype(T) && T !== BigInt + maxmagnitude = max(abs(BigInt(typemin(T))), abs(BigInt(typemax(T)))) + ndigits(coefficient) + scale <= ndigits(maxmagnitude) || + throw(InexactError(:parse, T, coefficient)) + end + scale <= typemax(Int) || throw(OverflowError("JSON integer exponent is too large")) + coefficient *= big(10)^Int(scale) + else + -scale < ndigits(coefficient) || throw(InexactError(:parse, T, coefficient)) + -scale <= typemax(Int) || throw(InexactError(:parse, T, coefficient)) + quotient, remainder = divrem(coefficient, big(10)^Int(-scale)) + iszero(remainder) || throw(InexactError(:parse, T, coefficient)) + coefficient = quotient + end + isneg && (coefficient = -coefficient) + return NumberResult(coefficient), nextpos end -# `T` is the type requested for this value (`Any` when materializing untyped); -# with `allownan=true`, all numbers materialize as `Float64` unless a specific -# type was requested, in which case the token is parsed exactly for that type +# `T` is the type requested for this value (`Number` when materializing untyped); +# with `allownan=true`, untyped numbers materialize as `Float64`, while a +# requested integer type is converted exactly from the number token @inline function parsenumber(x::LazyValue, ::Type{T}=Any) where {T} buf = getbuf(x) startpos::Int = getpos(x) @@ -707,7 +776,9 @@ end end invalid(InvalidNumber, buf, startpos, "number") end - if isfloat || (opts.allownan && T === Any) + if isfloat && opts.allownan && T <: Integer + return parseexactinteger(buf, startpos, nextpos, len, T) + elseif isfloat || (opts.allownan && (T === Any || T === Number)) return parsefloat64(buf, startpos, nextpos, len, opts.allownan) end overflow || return NumberResult(val), nextpos diff --git a/test/parse.jl b/test/parse.jl index 58e4cf52..5b5837db 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -422,7 +422,7 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ # allownan=true materializes all numbers as Float64 when no type is requested... @test JSON.parse("1"; allownan=true) === 1.0 @test JSON.parse("[1,2.5]"; allownan=true) == [1.0, 2.5] - @test JSON.parse("[1]", Vector{Any}; allownan=true) == [1.0] + @test only(JSON.parse("[1]", Vector{Any}; allownan=true)) === 1.0 @test JSON.parse("1", Float64; allownan=true) === 1.0 @test JSON.parse("[1,2]", Vector{Float64}; allownan=true) == [1.0, 2.0] # ...but a requested type parses the token exactly (#478) @@ -430,6 +430,16 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test JSON.parse(string(typemin(Int64)), Int64; allownan=true) === typemin(Int64) @test JSON.parse(string(typemax(UInt64)), UInt64; allownan=true) === typemax(UInt64) @test JSON.parse(string(typemax(Int128)), Int128; allownan=true) === typemax(Int128) + @test JSON.parse("9007199254740993e0", Int64; allownan=true) === Int64(9007199254740993) + @test JSON.parse("[9007199254740993e0]", Vector{Int64}; allownan=true) == Int64[9007199254740993] + @test JSON.parse("9223372036854775807.0", Int64; allownan=true) === typemax(Int64) + @test JSON.parse("-9223372036854775808.0", Int64; allownan=true) === typemin(Int64) + @test JSON.parse("18446744073709551615.0", UInt64; allownan=true) === typemax(UInt64) + @test JSON.parse("1.25e2", BigInt; allownan=true) == big(125) + @test_throws InexactError JSON.parse("1.5", Int64; allownan=true) + @test_throws InexactError JSON.parse("1.25e1", Int64; allownan=true) + @test_throws InexactError JSON.parse("1e1000000000", Int64; allownan=true) + @test_throws InexactError JSON.parse("1e-1000000000", Int64; allownan=true) @test JSON.parse("{\"a\":$(typemax(Int64))}", TestAllownanInt; allownan=true).a === typemax(Int64) # jsonlines support @test JSON.parse("1"; jsonlines=true) == [1] From 2cb59a0645d762dd215cc705659724c40f2349a6 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Fri, 4 Sep 2026 12:10:45 -0600 Subject: [PATCH 3/5] perf(parse): avoid BigInt on exact fast path Accumulate common fixed-width decimal and exponent tokens in UInt128, bound exponent expansion before multiplication, and reserve BigInt for long coefficients that can shrink to fit. Exclude BigInt targets from exponent expansion to prevent small inputs from forcing very large allocations. --- docs/src/reading.md | 2 +- src/lazy.jl | 113 +++++++++++++++++++++++++++++++++++++++----- test/parse.jl | 2 +- 3 files changed, 103 insertions(+), 14 deletions(-) diff --git a/docs/src/reading.md b/docs/src/reading.md index dde79c3d..ae1c5c3f 100644 --- a/docs/src/reading.md +++ b/docs/src/reading.md @@ -25,7 +25,7 @@ The core JSON parsing machinery is hence built around having an `AbstractVector{ Each entrypoint function first calls [`JSON.lazy`](@ref), which will consume the JSON input until the type of the next JSON value can be identified (`{` for objects, `[` for arrays, `"` for strings, `t` for true, `f` for false, `n` for null, and `-` or a digit for numbers). [`JSON.lazy`](@ref) returns a [`JSON.LazyValue`](@ref), which wraps the JSON input buffer (`AbstractVector{UInt8}` or `AbstractString`), and marks the byte position the value starts at, the type of the value, and any keyword arguments that were provided that may affect parsing. Currently supported parsing-specific keyword arguments to [`JSON.lazy`](@ref) (and thus all other entrypoint functions) include: - - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, untyped numbers are parsed as `Float64`. A requested integer type is parsed exactly from the number token + - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, untyped numbers are parsed as `Float64`. A requested fixed-width integer type of up to 128 bits is parsed exactly from the number token - `ninf::String = "-Infinity"`: the string that will be used to parse `-Inf` if `allownan=true` - `inf::String = "Infinity"`: the string that will be used to parse `Inf` if `allownan=true` - `nan::String = "NaN"`: the string that will be sued to parse `NaN` if `allownan=true` diff --git a/src/lazy.jl b/src/lazy.jl index eede04ab..f2a970bf 100644 --- a/src/lazy.jl +++ b/src/lazy.jl @@ -54,7 +54,7 @@ In this example, we only parsed as much of the `very_large_json_object` as was r Then we fully materialized `y` into `z`, which is now a normal Julia object. We can now mutate or access values in `z`. Currently supported keyword arguments include: - - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, untyped numbers are parsed as `Float64`. A requested integer type is parsed exactly from the number token + - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. When `true`, untyped numbers are parsed as `Float64`. A requested fixed-width integer type of up to 128 bits is parsed exactly from the number token - `ninf::String = "-Infinity"`: the string that will be used to parse `-Inf` if `allownan=true` - `inf::String = "Infinity"`: the string that will be used to parse `Inf` if `allownan=true` - `nan::String = "NaN"`: the string that will be sued to parse `NaN` if `allownan=true` @@ -688,10 +688,9 @@ else @inline exactintegerend(buf, first, nextpos, len) = nextpos end -# Convert a validated decimal or exponent token to an exact integer. This avoids -# losing precision by first materializing the token as Float64. -function parseexactinteger(buf, first, nextpos, len, ::Type{T}) where {T} - nextpos = exactintegerend(buf, first, nextpos, len) +# BigInt fallback for tokens whose coefficient does not fit in UInt128. This is +# only needed when a large coefficient is reduced by a negative decimal scale. +function parseexactintegerbig(buf, first, nextpos, ::Type{T}) where {T} pos = first b = getbyte(buf, pos) isneg = b == UInt8('-') @@ -733,11 +732,9 @@ function parseexactinteger(buf, first, nextpos, len, ::Type{T}) where {T} iszero(coefficient) && return NumberResult(coefficient), nextpos scale = exponent - fractiondigits if scale >= 0 - if isconcretetype(T) && T !== BigInt - maxmagnitude = max(abs(BigInt(typemin(T))), abs(BigInt(typemax(T)))) - ndigits(coefficient) + scale <= ndigits(maxmagnitude) || - throw(InexactError(:parse, T, coefficient)) - end + maxmagnitude = max(abs(BigInt(typemin(T))), abs(BigInt(typemax(T)))) + ndigits(coefficient) + scale <= ndigits(maxmagnitude) || + throw(InexactError(:parse, T, coefficient)) scale <= typemax(Int) || throw(OverflowError("JSON integer exponent is too large")) coefficient *= big(10)^Int(scale) else @@ -751,9 +748,99 @@ function parseexactinteger(buf, first, nextpos, len, ::Type{T}) where {T} return NumberResult(coefficient), nextpos end +# Convert a validated decimal or exponent token to an exact fixed-width integer. +# UInt128 keeps the common path allocation-free; unusually long coefficients +# fall back to BigInt only when a negative scale can reduce them to fit. +function parseexactinteger(buf, first, nextpos, len, ::Type{T}) where {T} + nextpos = exactintegerend(buf, first, nextpos, len) + pos = first + b = getbyte(buf, pos) + isneg = b == UInt8('-') + if isneg || b == UInt8('+') + pos += 1 + end + + coefficient = UInt128(0) + fractiondigits = 0 + infraction = false + while pos < nextpos + b = getbyte(buf, pos) + if UInt8('0') <= b <= UInt8('9') + digit = UInt128(b - UInt8('0')) + coefficient <= div(typemax(UInt128) - digit, 10) || + return parseexactintegerbig(buf, first, nextpos, T) + coefficient = coefficient * 10 + digit + infraction && (fractiondigits += 1) + elseif b == UInt8('.') + infraction = true + else + break + end + pos += 1 + end + + iszero(coefficient) && return NumberResult(Int64(0)), nextpos + exponent = 0 + if pos < nextpos + pos += 1 # skip 'e' or 'E' + b = getbyte(buf, pos) + expneg = b == UInt8('-') + if expneg || b == UInt8('+') + pos += 1 + end + # A UInt128 coefficient cannot survive an absolute decimal scale of 39. + # Saturating here prevents exponent text from overflowing Int. + explimit = expneg ? 39 : min(fractiondigits, typemax(Int) - 39) + 39 + while pos < nextpos + digit = Int(getbyte(buf, pos) - UInt8('0')) + if exponent > div(explimit - digit, 10) + exponent = explimit + break + end + exponent = exponent * 10 + digit + pos += 1 + end + expneg && (exponent = -exponent) + end + + scale = exponent - fractiondigits + if scale >= 0 + scale < 39 || throw(InexactError(:parse, T, coefficient)) + for _ = 1:scale + coefficient <= div(typemax(UInt128), 10) || + throw(InexactError(:parse, T, coefficient)) + coefficient *= 10 + end + else + -scale < 39 || throw(InexactError(:parse, T, coefficient)) + divisor = UInt128(1) + for _ = 1:-scale + divisor *= 10 + end + quotient, remainder = divrem(coefficient, divisor) + iszero(remainder) || throw(InexactError(:parse, T, coefficient)) + coefficient = quotient + end + + limit = if isneg + T <: Signed ? UInt128(typemax(T)) + 1 : UInt128(0) + else + UInt128(typemax(T)) + end + coefficient <= limit || throw(InexactError(:parse, T, coefficient)) + if coefficient <= UInt128(typemax(Int64)) + val = Int64(coefficient) + return NumberResult(isneg ? -val : val), nextpos + elseif isneg && coefficient == UInt128(typemax(Int64)) + 1 + return NumberResult(typemin(Int64)), nextpos + end + return NumberResult(isneg ? -BigInt(coefficient) : BigInt(coefficient)), nextpos +end + # `T` is the type requested for this value (`Number` when materializing untyped); # with `allownan=true`, untyped numbers materialize as `Float64`, while a -# requested integer type is converted exactly from the number token +# requested fixed-width integer type up to 128 bits is converted exactly from +# the number token @inline function parsenumber(x::LazyValue, ::Type{T}=Any) where {T} buf = getbuf(x) startpos::Int = getpos(x) @@ -776,7 +863,9 @@ end end invalid(InvalidNumber, buf, startpos, "number") end - if isfloat && opts.allownan && T <: Integer + exactint = isbitstype(T) && sizeof(T) <= sizeof(UInt128) && + (T === Bool || T <: Signed || T <: Unsigned) + if isfloat && opts.allownan && exactint return parseexactinteger(buf, startpos, nextpos, len, T) elseif isfloat || (opts.allownan && (T === Any || T === Number)) return parsefloat64(buf, startpos, nextpos, len, opts.allownan) diff --git a/test/parse.jl b/test/parse.jl index 5b5837db..7986bc3e 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -435,11 +435,11 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test JSON.parse("9223372036854775807.0", Int64; allownan=true) === typemax(Int64) @test JSON.parse("-9223372036854775808.0", Int64; allownan=true) === typemin(Int64) @test JSON.parse("18446744073709551615.0", UInt64; allownan=true) === typemax(UInt64) - @test JSON.parse("1.25e2", BigInt; allownan=true) == big(125) @test_throws InexactError JSON.parse("1.5", Int64; allownan=true) @test_throws InexactError JSON.parse("1.25e1", Int64; allownan=true) @test_throws InexactError JSON.parse("1e1000000000", Int64; allownan=true) @test_throws InexactError JSON.parse("1e-1000000000", Int64; allownan=true) + @test_throws InexactError JSON.parse("1e1000000000", BigInt; allownan=true) @test JSON.parse("{\"a\":$(typemax(Int64))}", TestAllownanInt; allownan=true).a === typemax(Int64) # jsonlines support @test JSON.parse("1"; jsonlines=true) == [1] From 472becdf9b954723eee4749791166a86d6deca4e Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Fri, 4 Sep 2026 12:27:31 -0600 Subject: [PATCH 4/5] fix(parse): bound exact integer conversion --- src/lazy.jl | 158 +++++++++++++++++++++++++++++++++----------------- test/parse.jl | 6 ++ 2 files changed, 111 insertions(+), 53 deletions(-) diff --git a/src/lazy.jl b/src/lazy.jl index f2a970bf..44e68c76 100644 --- a/src/lazy.jl +++ b/src/lazy.jl @@ -663,9 +663,32 @@ isbigfloat(x::NumberResult) = x.tag == BIGFLOAT return NumberResult(res.val), pos + Int(res.tlen) end @inline function exactintegerend(buf, first, nextpos, len) - res = Parsers.xparse2(Float64, buf, first, len) - Parsers.invalid(res.code) && invalid(InvalidNumber, buf, first, "number") - return first + Int(res.tlen) + pos = first + b = getbyte(buf, pos) + (b == UInt8('-') || b == UInt8('+')) && (pos += 1) + while pos <= len && UInt8('0') <= getbyte(buf, pos) <= UInt8('9') + pos += 1 + end + if pos <= len && getbyte(buf, pos) == UInt8('.') + pos += 1 + (pos > len || !(UInt8('0') <= getbyte(buf, pos) <= UInt8('9'))) && + invalid(InvalidNumber, buf, pos, "number") + while pos <= len && UInt8('0') <= getbyte(buf, pos) <= UInt8('9') + pos += 1 + end + end + if pos <= len && (getbyte(buf, pos) == UInt8('e') || getbyte(buf, pos) == UInt8('E')) + pos += 1 + if pos <= len && (getbyte(buf, pos) == UInt8('+') || getbyte(buf, pos) == UInt8('-')) + pos += 1 + end + (pos > len || !(UInt8('0') <= getbyte(buf, pos) <= UInt8('9'))) && + invalid(InvalidNumber, buf, pos, "number") + while pos <= len && UInt8('0') <= getbyte(buf, pos) <= UInt8('9') + pos += 1 + end + end + return pos end else # Parsers 3: the kernels convert exactly buf[first:nextpos-1] @@ -688,9 +711,40 @@ else @inline exactintegerend(buf, first, nextpos, len) = nextpos end -# BigInt fallback for tokens whose coefficient does not fit in UInt128. This is -# only needed when a large coefficient is reduced by a negative decimal scale. -function parseexactintegerbig(buf, first, nextpos, ::Type{T}) where {T} +@inline function parseboundedexponent(buf, pos, nextpos, limit) + exponent = 0 + tens, ones = divrem(limit, 10) + while pos < nextpos + digit = Int(getbyte(buf, pos) - UInt8('0')) + if exponent > tens || (exponent == tens && digit > ones) + return limit + end + exponent = exponent * 10 + digit + pos += 1 + end + return exponent +end + +@inline function exactintegerresult(coefficient::UInt128, isneg, ::Type{T}, nextpos) where {T} + limit = if isneg + T <: Signed ? UInt128(typemax(T)) + 1 : UInt128(0) + else + UInt128(typemax(T)) + end + coefficient <= limit || throw(InexactError(:parse, T, coefficient)) + if coefficient <= UInt128(typemax(Int64)) + val = Int64(coefficient) + return NumberResult(isneg ? -val : val), nextpos + elseif isneg && coefficient == UInt128(typemax(Int64)) + 1 + return NumberResult(typemin(Int64)), nextpos + end + return NumberResult(isneg ? -BigInt(coefficient) : BigInt(coefficient)), nextpos +end + +# Bounded fallback for tokens whose coefficient does not fit in UInt128. Count +# the decimal scale first, then retain at most the 39 significant digits that +# can contribute to a fixed-width result. This keeps work linear in token size. +function parseexactintegerlong(buf, first, nextpos, ::Type{T}) where {T} pos = first b = getbyte(buf, pos) isneg = b == UInt8('-') @@ -698,13 +752,15 @@ function parseexactintegerbig(buf, first, nextpos, ::Type{T}) where {T} pos += 1 end - coefficient = BigInt(0) + digitcount = 0 + firstnonzero = 0 fractiondigits = 0 infraction = false while pos < nextpos b = getbyte(buf, pos) if UInt8('0') <= b <= UInt8('9') - coefficient = coefficient * 10 + (b - UInt8('0')) + digitcount += 1 + b != UInt8('0') && firstnonzero == 0 && (firstnonzero = digitcount) infraction && (fractiondigits += 1) elseif b == UInt8('.') infraction = true @@ -714,7 +770,9 @@ function parseexactintegerbig(buf, first, nextpos, ::Type{T}) where {T} pos += 1 end - exponent = BigInt(0) + firstnonzero == 0 && return NumberResult(Int64(0)), nextpos + significantdigits = digitcount - firstnonzero + 1 + exponent = 0 if pos < nextpos pos += 1 # skip 'e' or 'E' b = getbyte(buf, pos) @@ -722,35 +780,49 @@ function parseexactintegerbig(buf, first, nextpos, ::Type{T}) where {T} if expneg || b == UInt8('+') pos += 1 end - while pos < nextpos - exponent = exponent * 10 + (getbyte(buf, pos) - UInt8('0')) - pos += 1 - end + # Once the scale removes all significant digits, the nonzero value is + # fractional. Positive exponents at least as large as fractiondigits + # cannot reduce an oversized coefficient. + explimit = expneg ? max(significantdigits - fractiondigits, 0) : fractiondigits + exponent = parseboundedexponent(buf, pos, nextpos, explimit) expneg && (exponent = -exponent) end - iszero(coefficient) && return NumberResult(coefficient), nextpos scale = exponent - fractiondigits - if scale >= 0 - maxmagnitude = max(abs(BigInt(typemin(T))), abs(BigInt(typemax(T)))) - ndigits(coefficient) + scale <= ndigits(maxmagnitude) || - throw(InexactError(:parse, T, coefficient)) - scale <= typemax(Int) || throw(OverflowError("JSON integer exponent is too large")) - coefficient *= big(10)^Int(scale) - else - -scale < ndigits(coefficient) || throw(InexactError(:parse, T, coefficient)) - -scale <= typemax(Int) || throw(InexactError(:parse, T, coefficient)) - quotient, remainder = divrem(coefficient, big(10)^Int(-scale)) - iszero(remainder) || throw(InexactError(:parse, T, coefficient)) - coefficient = quotient + scale < 0 || throw(InexactError(:parse, T, "out-of-range JSON number")) + removed = -scale + removed < significantdigits || + throw(InexactError(:parse, T, "non-integral JSON number")) + significantdigits - removed <= 39 || + throw(InexactError(:parse, T, "out-of-range JSON number")) + + cutoff = digitcount - removed + coefficient = UInt128(0) + digitindex = 0 + pos = first + (isneg || getbyte(buf, first) == UInt8('+')) + while pos < nextpos + b = getbyte(buf, pos) + if UInt8('0') <= b <= UInt8('9') + digitindex += 1 + digit = UInt128(b - UInt8('0')) + if digitindex <= cutoff + coefficient <= div(typemax(UInt128) - digit, 10) || + throw(InexactError(:parse, T, coefficient)) + coefficient = coefficient * 10 + digit + elseif !iszero(digit) + throw(InexactError(:parse, T, coefficient)) + end + elseif b != UInt8('.') + break + end + pos += 1 end - isneg && (coefficient = -coefficient) - return NumberResult(coefficient), nextpos + return exactintegerresult(coefficient, isneg, T, nextpos) end # Convert a validated decimal or exponent token to an exact fixed-width integer. -# UInt128 keeps the common path allocation-free; unusually long coefficients -# fall back to BigInt only when a negative scale can reduce them to fit. +# UInt128 keeps the common path allocation-free. The fallback for longer +# coefficients also retains only a bounded UInt128 result. function parseexactinteger(buf, first, nextpos, len, ::Type{T}) where {T} nextpos = exactintegerend(buf, first, nextpos, len) pos = first @@ -768,7 +840,7 @@ function parseexactinteger(buf, first, nextpos, len, ::Type{T}) where {T} if UInt8('0') <= b <= UInt8('9') digit = UInt128(b - UInt8('0')) coefficient <= div(typemax(UInt128) - digit, 10) || - return parseexactintegerbig(buf, first, nextpos, T) + return parseexactintegerlong(buf, first, nextpos, T) coefficient = coefficient * 10 + digit infraction && (fractiondigits += 1) elseif b == UInt8('.') @@ -791,15 +863,7 @@ function parseexactinteger(buf, first, nextpos, len, ::Type{T}) where {T} # A UInt128 coefficient cannot survive an absolute decimal scale of 39. # Saturating here prevents exponent text from overflowing Int. explimit = expneg ? 39 : min(fractiondigits, typemax(Int) - 39) + 39 - while pos < nextpos - digit = Int(getbyte(buf, pos) - UInt8('0')) - if exponent > div(explimit - digit, 10) - exponent = explimit - break - end - exponent = exponent * 10 + digit - pos += 1 - end + exponent = parseboundedexponent(buf, pos, nextpos, explimit) expneg && (exponent = -exponent) end @@ -822,19 +886,7 @@ function parseexactinteger(buf, first, nextpos, len, ::Type{T}) where {T} coefficient = quotient end - limit = if isneg - T <: Signed ? UInt128(typemax(T)) + 1 : UInt128(0) - else - UInt128(typemax(T)) - end - coefficient <= limit || throw(InexactError(:parse, T, coefficient)) - if coefficient <= UInt128(typemax(Int64)) - val = Int64(coefficient) - return NumberResult(isneg ? -val : val), nextpos - elseif isneg && coefficient == UInt128(typemax(Int64)) + 1 - return NumberResult(typemin(Int64)), nextpos - end - return NumberResult(isneg ? -BigInt(coefficient) : BigInt(coefficient)), nextpos + return exactintegerresult(coefficient, isneg, T, nextpos) end # `T` is the type requested for this value (`Number` when materializing untyped); diff --git a/test/parse.jl b/test/parse.jl index 7986bc3e..b3c07e1f 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -435,10 +435,16 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test JSON.parse("9223372036854775807.0", Int64; allownan=true) === typemax(Int64) @test JSON.parse("-9223372036854775808.0", Int64; allownan=true) === typemin(Int64) @test JSON.parse("18446744073709551615.0", UInt64; allownan=true) === typemax(UInt64) + @test JSON.parse("1000000000000000000000000000000000000000e-39", Int64; allownan=true) === 1 + @test JSON.parse("-1000000000000000000000000000000000000000e-39", Int64; allownan=true) === -1 @test_throws InexactError JSON.parse("1.5", Int64; allownan=true) @test_throws InexactError JSON.parse("1.25e1", Int64; allownan=true) + @test_throws InexactError JSON.parse("1000000000000000000000000000000000000000e-40", Int64; allownan=true) @test_throws InexactError JSON.parse("1e1000000000", Int64; allownan=true) @test_throws InexactError JSON.parse("1e-1000000000", Int64; allownan=true) + @test_throws InexactError JSON.parse("1e-$(repeat("9", 1000))", Int64; allownan=true) + @test_throws InexactError JSON.parse("$(repeat("9", 1000))e-1", Int64; allownan=true) + @test_throws InexactError JSON.parse("$(repeat("9", 40))e-$(repeat("9", 1000))", Int64; allownan=true) @test_throws InexactError JSON.parse("1e1000000000", BigInt; allownan=true) @test JSON.parse("{\"a\":$(typemax(Int64))}", TestAllownanInt; allownan=true).a === typemax(Int64) # jsonlines support From 9da7baf77acb9ba01987d727fae48aea4097c8b9 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Sat, 5 Sep 2026 13:02:19 -0600 Subject: [PATCH 5/5] Prepare JSON 1.8 and preserve typed float behavior --- .github/workflows/CI.yml | 15 ++++------ Project.toml | 2 +- src/lazy.jl | 2 +- test/parse.jl | 59 +++++++++++++++++++++++++++----------- test/trim_compile_tests.jl | 1 + 5 files changed, 52 insertions(+), 27 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 72807ba3..6cb6f512 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -9,14 +9,8 @@ on: jobs: test: - name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}${{ matrix.parsers && format(' - Parsers {0}', matrix.parsers) || '' }} + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} - Parsers ${{ matrix.parsers }} runs-on: ${{ matrix.os }} - env: - # Arrow (test dep) -> TimeZones >= 1.6 -> InlineStrings, whose released - # versions cap Parsers at 2, so resolving Parsers 3 falls back to TimeZones - # 1.5.9, whose build step needs a single thread pool (JuliaTime/TimeZones.jl#429). - # Remove once InlineStrings releases its Parsers 3 support. - JULIA_NUM_THREADS: '1' strategy: fail-fast: false matrix: @@ -29,13 +23,17 @@ jobs: - windows-latest arch: - x64 + parsers: + - '3' include: - os: macOS-latest arch: aarch64 version: 1 + parsers: '3' - os: ubuntu-latest arch: x86 version: 1 + parsers: '3' - os: ubuntu-latest arch: x64 version: 1 @@ -49,8 +47,7 @@ jobs: - uses: julia-actions/cache@v3 - uses: julia-actions/julia-buildpkg@v1 - name: Pin Parsers ${{ matrix.parsers }} - if: matrix.parsers - run: julia --project=. -e 'using Pkg; Pkg.add(name="Parsers", version="${{ matrix.parsers }}")' + run: julia --project=. -e 'using Pkg; Pkg.add(name="Parsers", version="${{ matrix.parsers }}"); Pkg.pin("Parsers")' - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v7 diff --git a/Project.toml b/Project.toml index 5258b2d2..49d5ae52 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "JSON" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "1.7.1" +version = "1.8.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/src/lazy.jl b/src/lazy.jl index 44e68c76..92c5fd44 100644 --- a/src/lazy.jl +++ b/src/lazy.jl @@ -919,7 +919,7 @@ end (T === Bool || T <: Signed || T <: Unsigned) if isfloat && opts.allownan && exactint return parseexactinteger(buf, startpos, nextpos, len, T) - elseif isfloat || (opts.allownan && (T === Any || T === Number)) + elseif isfloat || (opts.allownan && !(T <: Integer)) return parsefloat64(buf, startpos, nextpos, len, opts.allownan) end overflow || return NumberResult(val), nextpos diff --git a/test/parse.jl b/test/parse.jl index b3c07e1f..94fed49c 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -419,12 +419,24 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test_throws ArgumentError JSON.parse("+"; allownan=true) @test JSON.parse("+1"; allownan=true) === 1.0 @test JSON.parse("+Inf"; allownan=true) === Inf + @test isnan(JSON.parse("0"; allownan=true, nan="0")) + @test JSON.parse("1"; allownan=true, inf="1") === Inf + @test JSON.parse("-1"; allownan=true, ninf="-1") === -Inf + @test_throws InexactError JSON.parse("0", Int64; allownan=true, nan="0") + for source in ("01", "1.", "1e", "1e+", "1.0e-", "nan", "inf") + @test_throws ArgumentError JSON.parse(source; allownan=true) + @test_throws ArgumentError JSON.parse(source, Int64; allownan=true) + end # allownan=true materializes all numbers as Float64 when no type is requested... @test JSON.parse("1"; allownan=true) === 1.0 @test JSON.parse("[1,2.5]"; allownan=true) == [1.0, 2.5] @test only(JSON.parse("[1]", Vector{Any}; allownan=true)) === 1.0 @test JSON.parse("1", Float64; allownan=true) === 1.0 @test JSON.parse("[1,2]", Vector{Float64}; allownan=true) == [1.0, 2.0] + for T in (Float16, Float32, Float64, BigFloat, Real, Number, Any) + @test signbit(JSON.parse("-0", T; allownan=true)) + @test signbit(only(JSON.parse(b"[-0]", Vector{T}; allownan=true))) + end # ...but a requested type parses the token exactly (#478) @test JSON.parse(string(typemax(Int64)), Int64; allownan=true) === typemax(Int64) @test JSON.parse(string(typemin(Int64)), Int64; allownan=true) === typemin(Int64) @@ -435,8 +447,8 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test JSON.parse("9223372036854775807.0", Int64; allownan=true) === typemax(Int64) @test JSON.parse("-9223372036854775808.0", Int64; allownan=true) === typemin(Int64) @test JSON.parse("18446744073709551615.0", UInt64; allownan=true) === typemax(UInt64) - @test JSON.parse("1000000000000000000000000000000000000000e-39", Int64; allownan=true) === 1 - @test JSON.parse("-1000000000000000000000000000000000000000e-39", Int64; allownan=true) === -1 + @test JSON.parse("1000000000000000000000000000000000000000e-39", Int64; allownan=true) === Int64(1) + @test JSON.parse("-1000000000000000000000000000000000000000e-39", Int64; allownan=true) === Int64(-1) @test_throws InexactError JSON.parse("1.5", Int64; allownan=true) @test_throws InexactError JSON.parse("1.25e1", Int64; allownan=true) @test_throws InexactError JSON.parse("1000000000000000000000000000000000000000e-40", Int64; allownan=true) @@ -447,6 +459,25 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test_throws InexactError JSON.parse("$(repeat("9", 40))e-$(repeat("9", 1000))", Int64; allownan=true) @test_throws InexactError JSON.parse("1e1000000000", BigInt; allownan=true) @test JSON.parse("{\"a\":$(typemax(Int64))}", TestAllownanInt; allownan=true).a === typemax(Int64) + mktempdir() do dir + file = joinpath(dir, "typed-integer.json") + for value in (typemin(Int64), typemax(Int64), Int64(2)^53 + 1) + JSON.json(file, TestAllownanInt(value); allownan=true) + @test JSON.parsefile(file, TestAllownanInt; allownan=true).a === value + end + end + for T in (Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128) + for value in (typemin(T), typemax(T)), source in ( + string(BigInt(value), ".0"), + string(BigInt(value), "e0"), + string(BigInt(value) * 10, "e-1"), + ) + @test JSON.parse(source, T; allownan=true) === value + @test only(JSON.parse(Vector{UInt8}(codeunits("[ $source ]")), Vector{T}; allownan=true)) === value + end + @test_throws InexactError JSON.parse(string(BigInt(typemax(T)) + 1, ".0"), T; allownan=true) + @test_throws InexactError JSON.parse(string(BigInt(typemin(T)) - 1, "e0"), T; allownan=true) + end # jsonlines support @test JSON.parse("1"; jsonlines=true) == [1] @test JSON.parse("1 \t"; jsonlines=true) == [1] @@ -537,19 +568,15 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ x = JSON.parse("-9223372036854775809") @test x isa BigInt && x == -9223372036854775809 # long mantissas and negative zeros that Parsers < 2.8.8 crashed on or misparsed - if pkgversion(JSON.Parsers) >= v"2.8.8" - # (Base.parse(Float64, ...) itself throws on this one, so compare directly) - @test JSON.parse("295574326048237151328925.8099133506971425945276929554326e-440") === 0.0 - for source in ("-773185451005006305224330936226383685.195e3", - "0.72741733550162454424961322208253163690E+61", - "-75738806850214820018096823497.7e229") - @test JSON.parse(source) === Base.parse(Float64, source) - @test JSON.parse(Vector{UInt8}(codeunits(source))) === Base.parse(Float64, source) - end - @test JSON.parse("-0e291") === -0.0 - @test JSON.parse("-0e292") === -0.0 - @test JSON.parse("-0.0e100") === -0.0 + # (Base.parse(Float64, ...) itself throws on this one, so compare directly) + @test JSON.parse("295574326048237151328925.8099133506971425945276929554326e-440") === 0.0 + for source in ("-773185451005006305224330936226383685.195e3", + "0.72741733550162454424961322208253163690E+61", + "-75738806850214820018096823497.7e229") + @test JSON.parse(source) === Base.parse(Float64, source) + @test JSON.parse(Vector{UInt8}(codeunits(source))) === Base.parse(Float64, source) end + @test JSON.parse("-0.0e100") === -0.0 # promote to BigInt x = JSON.parse("9223372036854775808") # only == here because BigInt don't compare w/ === @@ -578,8 +605,8 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test JSON.parse("0e292") === 0.0 @test JSON.parse("0e347") == big"0.0" @test JSON.parse("0e348") == big"0.0" - @test JSON.parse("-0e291") == 0.0 - @test JSON.parse("-0e292") == 0.0 + @test JSON.parse("-0e291") === -0.0 + @test JSON.parse("-0e292") === -0.0 @test JSON.parse("-0e347") == big"0.0" @test JSON.parse("-0e348") == big"0.0" @test JSON.parse("2e-324") === 0.0 diff --git a/test/trim_compile_tests.jl b/test/trim_compile_tests.jl index 8caddc0c..fcab9a67 100644 --- a/test/trim_compile_tests.jl +++ b/test/trim_compile_tests.jl @@ -18,6 +18,7 @@ function _prepare_trim_project(project_path::String, trim_project::String)::Noth try Pkg.activate(trim_project) Pkg.develop(Pkg.PackageSpec(path = project_path)) + Pkg.add(Pkg.PackageSpec(name = "Parsers", version = pkgversion(JSON.Parsers))) Pkg.instantiate() finally if original_project !== nothing