From 218939999b1ef26ed8e282b9cb2d38d81ca63d36 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Sun, 19 Jul 2026 22:25:18 +0530 Subject: [PATCH 1/5] parser_syslog: accept a space after RFC3164 priority Signed-off-by: Akash Kumar --- lib/fluent/plugin/parser_syslog.rb | 81 ++++++++++- test/plugin/test_in_syslog.rb | 72 ++++++++++ test/plugin/test_parser_syslog.rb | 214 +++++++++++++++++++++++++++++ 3 files changed, 361 insertions(+), 6 deletions(-) diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb index 2b6fcca479..4bbc41b89c 100644 --- a/lib/fluent/plugin/parser_syslog.rb +++ b/lib/fluent/plugin/parser_syslog.rb @@ -71,6 +71,7 @@ def initialize @time_parser_rfc5424 = nil @space_count_rfc3164 = nil @space_count_rfc5424 = nil + @time_format_starts_with_space_rfc3164 = false @skip_space_count_rfc3164 = false @skip_space_count_rfc5424 = false @time_parser_rfc5424_without_subseconds = nil @@ -123,10 +124,12 @@ class << self def setup_time_parser_3164(time_fmt) @time_parser_rfc3164 = time_parser_create(format: time_fmt) + @time_format_starts_with_space_rfc3164 = time_fmt.start_with?(SPLIT_CHAR) if ['%b %d %H:%M:%S', '%b %d %H:%M:%S.%N'].include?(time_fmt) @skip_space_count_rfc3164 = true end @space_count_rfc3164 = time_fmt.squeeze(' ').count(' ') + 1 + @space_count_rfc3164 -= 1 if @time_format_starts_with_space_rfc3164 end def setup_time_parser_5424(time_fmt) @@ -162,6 +165,13 @@ def parse_auto(text, &block) end SPLIT_CHAR = ' '.freeze + SPLIT_BYTE = 0x20 + DOT_CHAR = '.'.freeze + DOT_BYTE = 0x2e + GREATER_THAN_BYTE = 0x3e + ZERO_BYTE = 0x30 + NINE_BYTE = 0x39 + RFC3164_PRI_MINIMUM = [nil, nil, nil, 1, 10, 100].freeze def parse_rfc3164_regex(text, &block) idx = 0 @@ -180,12 +190,21 @@ def parse_rfc3164_regex(text, &block) i = idx - 1 sq = false + first_time_field = true @space_count_rfc3164.times do while text[i + 1] == SPLIT_CHAR + if first_time_field + unless @time_format_starts_with_space_rfc3164 + idx += 1 + i += 1 + break + end + end sq = true i += 1 end + first_time_field = false i = text.index(SPLIT_CHAR, i + 1) end @@ -287,14 +306,43 @@ def parse_rfc3164(text, &block) end end + byte = text.getbyte(cursor) + space_after_priority = if byte == SPLIT_BYTE + true + elsif @with_priority && text.getbyte(cursor - 1) != GREATER_THAN_BYTE + # String#index returns a character index. Mark + # non-ASCII PRI for the same timestamp lookup. + byte = nil + text[cursor] == SPLIT_CHAR + else + false + end + if space_after_priority + cursor = rfc3164_space_cursor(text, cursor, pri) + unless cursor + yield nil, nil + return + end + end + if @skip_space_count_rfc3164 # header part time_size = 15 # skip Mmm dd hh:mm:ss - time_end = text[cursor + time_size] - if time_end == SPLIT_CHAR + time_end_index = cursor + time_size + if byte + time_end = text.getbyte(time_end_index) + split_delimiter = SPLIT_BYTE + dot_delimiter = DOT_BYTE + else + time_end = text[time_end_index] + split_delimiter = SPLIT_CHAR + dot_delimiter = DOT_CHAR + end + + if time_end == split_delimiter time_str = text.slice(cursor, time_size) cursor += 16 # time + ' ' - elsif time_end == '.'.freeze + elsif time_end == dot_delimiter # support subsecond time i = text.index(SPLIT_CHAR, time_size) time_str = text.slice(cursor, i - cursor) @@ -314,12 +362,12 @@ def parse_rfc3164(text, &block) i = text.index(SPLIT_CHAR, i + 1) end - time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor) + time_str = sq ? text.slice(cursor, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor) cursor = i + 1 end i = text.index(SPLIT_CHAR, cursor) - if i.nil? + unless i yield nil, nil return end @@ -333,7 +381,7 @@ def parse_rfc3164(text, &block) i = text.index(SPLIT_CHAR, cursor) # message part - msg = if i.nil? # for 'only non-space content case' + msg = unless i # for 'only non-space content case' text.slice(cursor, text.bytesize) else if text[i - 1] == ':'.freeze @@ -369,6 +417,27 @@ def parse_rfc3164(text, &block) yield time, record end + def rfc3164_space_cursor(text, cursor, pri) + if @with_priority + # A nonzero value with the expected decimal width proves that the + # whole PRI is numeric. Leading-zero values take the bytewise path. + minimum = RFC3164_PRI_MINIMUM[cursor] + return unless minimum + + unless pri >= minimum + priority_byte = 1 + while priority_byte < cursor - 1 + byte = text.getbyte(priority_byte) + return unless byte >= ZERO_BYTE && byte <= NINE_BYTE + + priority_byte += 1 + end + end + end + + @time_format_starts_with_space_rfc3164 ? cursor : cursor + 1 + end + NILVALUE = '-'.freeze def parse_rfc5424(text, &block) diff --git a/test/plugin/test_in_syslog.rb b/test/plugin/test_in_syslog.rb index 868e77bb04..09b5136524 100755 --- a/test/plugin/test_in_syslog.rb +++ b/test/plugin/test_in_syslog.rb @@ -124,6 +124,78 @@ def test_time_format(data) } end + data( + 'regexp/rfc3164/parser priority' => ['regexp', 'rfc3164', true], + 'string/rfc3164/parser priority' => ['string', 'rfc3164', true], + 'regexp/auto/parser priority' => ['regexp', 'auto', true], + 'string/auto/parser priority' => ['string', 'auto', true], + 'regexp/rfc3164/input priority' => ['regexp', 'rfc3164', false], + 'string/rfc3164/input priority' => ['string', 'rfc3164', false], + 'regexp/auto/input priority' => ['regexp', 'auto', false], + 'string/auto/input priority' => ['string', 'auto', false], + ) + def test_space_between_rfc3164_priority_and_header(data) + parser_engine, message_format, with_priority = data + d = create_driver([ + ipv4_config, + 'severity_key severity', + 'facility_key facility', + '', + " parser_engine #{parser_engine}", + " message_format #{message_format}", + " with_priority #{with_priority}", + '', + ].join("\n")) + + message = 'Apr 25 16:43:29 PAA-SW1-1 General[procLOG]: main.c(257) 272264 %% Stopping System API application' + d.run(expect_emits: 2) do + u = UDPSocket.new + u.connect('127.0.0.1', @port) + u.send("<14>#{message}", 0) + u.send("<14> #{message}", 0) + end + + assert_equal(2, d.events.size) + assert_equal(d.events[0][1], d.events[1][1]) + assert_equal(d.events[0][2], d.events[1][2]) + d.events.each do |tag, time, record| + assert_equal('syslog.user.info', tag) + assert_equal(event_time('Apr 25 16:43:29', format: '%b %d %H:%M:%S'), time) + assert_equal('user', record['facility']) + assert_equal('info', record['severity']) + assert_equal('PAA-SW1-1', record['host']) + assert_equal('General', record['ident']) + assert_equal('main.c(257) 272264 %% Stopping System API application', record['message']) + end + end + + data('regexp' => 'regexp', 'string' => 'string') + def test_space_after_input_owned_priority_preserves_input_priority_syntax(parser_engine) + d = create_driver([ + ipv4_config, + 'emit_unmatched_lines true', + '', + " parser_engine #{parser_engine}", + ' with_priority false', + '', + ].join("\n")) + + messages = [ + '<1234>Apr 25 16:43:29 host app: message', + '<1234> Apr 25 16:43:29 host app: message', + ' Apr 25 16:43:29 host app: message', + ] + d.run(expect_emits: 3) do + u = UDPSocket.new + u.connect('127.0.0.1', @port) + messages.each { |message| u.send(message, 0) } + end + + assert_equal(d.events[0], d.events[1]) + assert_equal('syslog.unmatched', d.events[2][0]) + assert_equal(messages[2], d.events[2][2]['unmatched_line']) + end + def test_msg_size d = create_driver tests = create_test_case diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb index dcdbca8388..1eb884ce7f 100644 --- a/test/plugin/test_parser_syslog.rb +++ b/test/plugin/test_parser_syslog.rb @@ -75,6 +75,220 @@ def test_parse_with_priority(param) assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format']) end + data( + 'regexp/rfc3164/parser priority' => ['regexp', 'rfc3164', true], + 'string/rfc3164/parser priority' => ['string', 'rfc3164', true], + 'regexp/auto/parser priority' => ['regexp', 'auto', true], + 'string/auto/parser priority' => ['string', 'auto', true], + 'regexp/rfc3164/without priority' => ['regexp', 'rfc3164', false], + 'string/rfc3164/without priority' => ['string', 'rfc3164', false], + 'regexp/auto/without priority' => ['regexp', 'auto', false], + 'string/auto/without priority' => ['string', 'auto', false], + ) + def test_parse_with_space_between_rfc3164_priority_and_header(data) + parser_engine, message_format, with_priority = data + @parser.configure( + 'parser_engine' => parser_engine, + 'message_format' => message_format, + 'with_priority' => with_priority, + 'keep_time_key' => true, + ) + + prefix = with_priority ? '<14>' : '' + message = 'Apr 25 16:43:29 PAA-SW1-1 General[procLOG]: main.c(257) 272264 %% Stopping System API application' + results = ["#{prefix}#{message}", "#{prefix} #{message}"].map do |text| + result = nil + @parser.instance.parse(text) do |time, record| + result = [time, record] + end + result + end + + assert_equal(results[0], results[1]) + time, record = results[1] + assert_equal(event_time('Apr 25 16:43:29', format: '%b %d %H:%M:%S'), time) + assert_equal(14, record['pri']) if with_priority + assert_equal('Apr 25 16:43:29', record['time']) + assert_equal('PAA-SW1-1', record['host']) + assert_equal('General', record['ident']) + assert_equal('main.c(257) 272264 %% Stopping System API application', record['message']) + end + + data( + 'regexp/single digit day/with priority' => ['regexp', true, '%b %d %H:%M:%S', 'Apr 5 16:43:29'], + 'string/single digit day/with priority' => ['string', true, '%b %d %H:%M:%S', 'Apr 5 16:43:29'], + 'regexp/subseconds/with priority' => ['regexp', true, '%b %d %H:%M:%S.%N', 'Apr 25 16:43:29.123456789'], + 'string/subseconds/with priority' => ['string', true, '%b %d %H:%M:%S.%N', 'Apr 25 16:43:29.123456789'], + 'regexp/custom format/with priority' => ['regexp', true, '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29'], + 'string/custom format/with priority' => ['string', true, '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29'], + 'regexp/single digit day/without priority' => ['regexp', false, '%b %d %H:%M:%S', 'Apr 5 16:43:29'], + 'string/single digit day/without priority' => ['string', false, '%b %d %H:%M:%S', 'Apr 5 16:43:29'], + 'regexp/subseconds/without priority' => ['regexp', false, '%b %d %H:%M:%S.%N', 'Apr 25 16:43:29.123456789'], + 'string/subseconds/without priority' => ['string', false, '%b %d %H:%M:%S.%N', 'Apr 25 16:43:29.123456789'], + 'regexp/custom format/without priority' => ['regexp', false, '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29'], + 'string/custom format/without priority' => ['string', false, '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29'], + ) + def test_parse_rfc3164_time_variants_with_space_after_priority(data) + parser_engine, with_priority, time_format, timestamp = data + @parser.configure( + 'parser_engine' => parser_engine, + 'time_format' => time_format, + 'with_priority' => with_priority, + 'keep_time_key' => true, + ) + + suffix = "#{timestamp} host app[123]: message" + prefix = with_priority ? '<14>' : '' + results = ["#{prefix}#{suffix}", "#{prefix} #{suffix}"].map do |text| + result = nil + @parser.instance.parse(text) do |time, record| + result = [time, record] + end + result + end + + assert_equal(results[0], results[1]) + expected_time_key = parser_engine == 'regexp' ? timestamp.squeeze(' ') : timestamp + assert_equal(expected_time_key, results[1][1]['time']) + assert_equal('host', results[1][1]['host']) + assert_equal('app', results[1][1]['ident']) + assert_equal('123', results[1][1]['pid']) + assert_equal('message', results[1][1]['message']) + end + + data( + 'regexp/parser priority' => ['regexp', true], + 'string/parser priority' => ['string', true], + 'regexp/input priority' => ['regexp', false], + 'string/input priority' => ['string', false], + ) + def test_parse_with_leading_space_time_format(data) + parser_engine, with_priority = data + @parser.configure( + 'parser_engine' => parser_engine, + 'time_format' => ' %b %d %H:%M:%S', + 'with_priority' => with_priority, + 'keep_time_key' => true, + ) + + prefix = with_priority ? '<14>' : '' + result = nil + @parser.instance.parse("#{prefix} Apr 25 16:43:29 host app[123]: message") do |time, record| + result = [time, record] + end + + time, record = result + assert_equal(event_time('Apr 25 16:43:29', format: '%b %d %H:%M:%S'), time) + assert_equal(14, record['pri']) if with_priority + assert_equal(' Apr 25 16:43:29', record['time']) + assert_equal( + {'host' => 'host', 'ident' => 'app', 'pid' => '123', 'message' => 'message'}, + record.reject { |key, _| ['pri', 'time'].include?(key) }, + ) + end + + data( + 'regexp/two spaces' => ['regexp', true, '<14> Apr 25 16:43:29 host app: message'], + 'string/two spaces' => ['string', true, '<14> Apr 25 16:43:29 host app: message'], + 'regexp/tab' => ['regexp', true, "<14>\tApr 25 16:43:29 host app: message"], + 'string/tab' => ['string', true, "<14>\tApr 25 16:43:29 host app: message"], + 'regexp/two leading spaces' => ['regexp', false, ' Apr 25 16:43:29 host app: message'], + 'string/two leading spaces' => ['string', false, ' Apr 25 16:43:29 host app: message'], + 'regexp/leading tab' => ['regexp', false, "\tApr 25 16:43:29 host app: message"], + 'string/leading tab' => ['string', false, "\tApr 25 16:43:29 host app: message"], + 'regexp/long priority' => ['regexp', true, '<1234> Apr 25 16:43:29 host app: message'], + 'string/long priority' => ['string', true, '<1234> Apr 25 16:43:29 host app: message'], + 'regexp/non-numeric priority' => ['regexp', true, ' Apr 25 16:43:29 host app: message'], + 'string/non-numeric priority' => ['string', true, ' Apr 25 16:43:29 host app: message'], + ) + def test_parse_does_not_broaden_rfc3164_priority_separator(data) + parser_engine, with_priority, text = data + @parser.configure('parser_engine' => parser_engine, 'with_priority' => with_priority) + parsed = false + + begin + @parser.instance.parse(text) do |time, record| + parsed = !!(time && record) + end + rescue Fluent::TimeParser::TimeParseError + # The regexp and string parsers report invalid input differently. + end + + assert_false(parsed) + end + + data( + 'zero' => ['<0> Apr 25 16:43:29 host app: message', 0], + 'two digits with leading zero' => ['<01> Apr 25 16:43:29 host app: message', 1], + 'three digits with leading zeros' => ['<001> Apr 25 16:43:29 host app: message', 1], + ) + def test_string_parser_accepts_spaced_numeric_priority_with_leading_zeros(data) + text, expected_priority = data + @parser.configure('parser_engine' => 'string', 'with_priority' => true) + result = nil + @parser.instance.parse(text) do |time, record| + result = [time, record] + end + + time, record = result + assert_not_nil(time) + assert_equal(expected_priority, record['pri']) + assert_equal('host', record['host']) + assert_equal('app', record['ident']) + assert_equal('message', record['message']) + end + + data( + 'long priority' => '<1234> Apr 25 16:43:29 host app: message', + 'non-numeric priority' => ' Apr 25 16:43:29 host app: message', + 'partially numeric priority' => '<1a> Apr 25 16:43:29 host app: message', + 'non-ASCII priority' => '<é> Apr 25 16:43:29 host app: message', + ) + def test_leading_space_time_format_does_not_broaden_priority_syntax(text) + @parser.configure( + 'parser_engine' => 'string', + 'time_format' => ' %b %d %H:%M:%S', + 'with_priority' => true, + ) + parsed = false + @parser.instance.parse(text) do |time, record| + parsed = !!(time && record) + end + + assert_false(parsed) + end + + data( + 'regexp/rfc5424/with priority' => ['regexp', 'rfc5424', true], + 'string/rfc5424/with priority' => ['string', 'rfc5424', true], + 'regexp/auto/with priority' => ['regexp', 'auto', true], + 'string/auto/with priority' => ['string', 'auto', true], + 'regexp/rfc5424/without priority' => ['regexp', 'rfc5424', false], + 'string/rfc5424/without priority' => ['string', 'rfc5424', false], + 'regexp/auto/without priority' => ['regexp', 'auto', false], + 'string/auto/without priority' => ['string', 'auto', false], + ) + def test_parse_does_not_accept_space_after_rfc5424_priority(data) + parser_engine, message_format, with_priority = data + @parser.configure( + 'parser_engine' => parser_engine, + 'message_format' => message_format, + 'with_priority' => with_priority, + ) + text = with_priority ? '<14> 1 2026-04-25T16:43:29Z host app 123 ID - message' : ' 1 2026-04-25T16:43:29Z host app 123 ID - message' + parsed = false + + begin + @parser.instance.parse(text) do |time, record| + parsed = !!(time && record) + end + rescue Fluent::TimeParser::TimeParseError + # The regexp and string parsers report invalid input differently. + end + + assert_false(parsed) + end + data('regexp' => 'regexp', 'string' => 'string') def test_parse_rfc5452_with_priority(param) @parser.configure('with_priority' => true, 'parser_type' => param, 'message_format' => 'rfc5424') From 24c01fb475d0ca53703a6d4b99d83b34f6d1c3d5 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Tue, 28 Jul 2026 22:07:09 +0530 Subject: [PATCH 2/5] parser_syslog: reject truncated subsecond records cleanly Signed-off-by: Akash Kumar --- lib/fluent/plugin/parser_syslog.rb | 4 ++++ test/plugin/test_parser_syslog.rb | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb index 4bbc41b89c..0c7264cf86 100644 --- a/lib/fluent/plugin/parser_syslog.rb +++ b/lib/fluent/plugin/parser_syslog.rb @@ -345,6 +345,10 @@ def parse_rfc3164(text, &block) elsif time_end == dot_delimiter # support subsecond time i = text.index(SPLIT_CHAR, time_size) + unless i + yield nil, nil + return + end time_str = text.slice(cursor, i - cursor) cursor = i + 1 else diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb index 1eb884ce7f..fff94ea099 100644 --- a/test/plugin/test_parser_syslog.rb +++ b/test/plugin/test_parser_syslog.rb @@ -156,6 +156,26 @@ def test_parse_rfc3164_time_variants_with_space_after_priority(data) assert_equal('message', results[1][1]['message']) end + data( + 'boundary space' => '<14> Apr 25 16:43:29.5', + ) + def test_truncated_subsecond_rfc3164_is_rejected_without_raising(text) + @parser.configure( + 'parser_engine' => 'string', + 'message_format' => 'rfc3164', + 'with_priority' => true, + 'time_format' => '%b %d %H:%M:%S.%N', + ) + + result = :not_yielded + assert_nothing_raised do + @parser.instance.parse(text) do |time, record| + result = [time, record] + end + end + assert_equal([nil, nil], result) + end + data( 'regexp/parser priority' => ['regexp', true], 'string/parser priority' => ['string', true], From c45c7daf9510ab40d8ea86aa5701d6054f60bff3 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Tue, 28 Jul 2026 22:15:28 +0530 Subject: [PATCH 3/5] test: cover truncated subsecond parser routes Signed-off-by: Akash Kumar --- test/plugin/test_parser_syslog.rb | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb index fff94ea099..10345d6bc9 100644 --- a/test/plugin/test_parser_syslog.rb +++ b/test/plugin/test_parser_syslog.rb @@ -157,19 +157,36 @@ def test_parse_rfc3164_time_variants_with_space_after_priority(data) end data( - 'boundary space' => '<14> Apr 25 16:43:29.5', + 'rfc3164/parser priority' => ['rfc3164', true], + 'auto/parser priority' => ['auto', true], + 'rfc3164/input priority' => ['rfc3164', false], + 'auto/input priority' => ['auto', false], ) - def test_truncated_subsecond_rfc3164_is_rejected_without_raising(text) + def test_truncated_subsecond_rfc3164_is_rejected_without_raising(data) + message_format, with_priority = data @parser.configure( 'parser_engine' => 'string', - 'message_format' => 'rfc3164', - 'with_priority' => true, + 'message_format' => message_format, + 'with_priority' => with_priority, 'time_format' => '%b %d %H:%M:%S.%N', ) + prefix = with_priority ? '<14> ' : ' ' + timestamp = 'Apr 25 16:43:29.5' + valid_result = nil + @parser.instance.parse("#{prefix}#{timestamp} host app: message") do |time, record| + valid_result = [time, record] + end + expected_record = {'host' => 'host', 'ident' => 'app', 'message' => 'message'} + expected_record['pri'] = 14 if with_priority + assert_equal( + [event_time(timestamp, format: '%b %d %H:%M:%S.%N'), expected_record], + valid_result, + ) + result = :not_yielded assert_nothing_raised do - @parser.instance.parse(text) do |time, record| + @parser.instance.parse("#{prefix}#{timestamp}") do |time, record| result = [time, record] end end From d46926fdf0d2f59983535c4fcb0ac8a13e0deaa5 Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Wed, 9 Sep 2026 12:29:15 +0900 Subject: [PATCH 4/5] parser_syslog: use character indexes consistently in the string parser The byte-index path added for the space after the RFC3164 priority mixed byte offsets with the character offsets used by String#slice. A UTF-8 header carrying a multibyte character was handed to the time parser instead of being rejected. Drop that path and check the priority digits on the substring itself, guard the space_count branch against a truncated header in both the string and the regexp parser, and search for the subsecond delimiter from the end of the timestamp. Signed-off-by: Shizuo Fujita Co-authored-by: Claude Opus 5 (1M context) --- lib/fluent/plugin/parser_syslog.rb | 67 ++++---------- test/plugin/test_in_syslog.rb | 1 + test/plugin/test_parser_syslog.rb | 141 +++++++++++++++++++---------- 3 files changed, 112 insertions(+), 97 deletions(-) diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb index 0c7264cf86..901a5e66ca 100644 --- a/lib/fluent/plugin/parser_syslog.rb +++ b/lib/fluent/plugin/parser_syslog.rb @@ -165,13 +165,8 @@ def parse_auto(text, &block) end SPLIT_CHAR = ' '.freeze - SPLIT_BYTE = 0x20 DOT_CHAR = '.'.freeze - DOT_BYTE = 0x2e - GREATER_THAN_BYTE = 0x3e - ZERO_BYTE = 0x30 - NINE_BYTE = 0x39 - RFC3164_PRI_MINIMUM = [nil, nil, nil, 1, 10, 100].freeze + RFC3164_PRI_DIGITS_REGEXP = /\A[0-9]{1,3}\z/ def parse_rfc3164_regex(text, &block) idx = 0 @@ -206,6 +201,10 @@ def parse_rfc3164_regex(text, &block) first_time_field = false i = text.index(SPLIT_CHAR, i + 1) + unless i + yield nil, nil + return + end end time_str = sq ? text.slice(idx, i - idx).squeeze(SPLIT_CHAR) : text.slice(idx, i - idx) @@ -306,19 +305,8 @@ def parse_rfc3164(text, &block) end end - byte = text.getbyte(cursor) - space_after_priority = if byte == SPLIT_BYTE - true - elsif @with_priority && text.getbyte(cursor - 1) != GREATER_THAN_BYTE - # String#index returns a character index. Mark - # non-ASCII PRI for the same timestamp lookup. - byte = nil - text[cursor] == SPLIT_CHAR - else - false - end - if space_after_priority - cursor = rfc3164_space_cursor(text, cursor, pri) + if text[cursor] == SPLIT_CHAR + cursor = rfc3164_space_cursor(text, cursor) unless cursor yield nil, nil return @@ -329,22 +317,14 @@ def parse_rfc3164(text, &block) # header part time_size = 15 # skip Mmm dd hh:mm:ss time_end_index = cursor + time_size - if byte - time_end = text.getbyte(time_end_index) - split_delimiter = SPLIT_BYTE - dot_delimiter = DOT_BYTE - else - time_end = text[time_end_index] - split_delimiter = SPLIT_CHAR - dot_delimiter = DOT_CHAR - end + time_end = text[time_end_index] - if time_end == split_delimiter + if time_end == SPLIT_CHAR time_str = text.slice(cursor, time_size) cursor += 16 # time + ' ' - elsif time_end == dot_delimiter + elsif time_end == DOT_CHAR # support subsecond time - i = text.index(SPLIT_CHAR, time_size) + i = text.index(SPLIT_CHAR, time_end_index) unless i yield nil, nil return @@ -364,6 +344,10 @@ def parse_rfc3164(text, &block) i += 1 end i = text.index(SPLIT_CHAR, i + 1) + unless i + yield nil, nil + return + end end time_str = sq ? text.slice(cursor, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor) @@ -385,7 +369,7 @@ def parse_rfc3164(text, &block) i = text.index(SPLIT_CHAR, cursor) # message part - msg = unless i # for 'only non-space content case' + msg = if i.nil? # for 'only non-space content case' text.slice(cursor, text.bytesize) else if text[i - 1] == ':'.freeze @@ -421,23 +405,8 @@ def parse_rfc3164(text, &block) yield time, record end - def rfc3164_space_cursor(text, cursor, pri) - if @with_priority - # A nonzero value with the expected decimal width proves that the - # whole PRI is numeric. Leading-zero values take the bytewise path. - minimum = RFC3164_PRI_MINIMUM[cursor] - return unless minimum - - unless pri >= minimum - priority_byte = 1 - while priority_byte < cursor - 1 - byte = text.getbyte(priority_byte) - return unless byte >= ZERO_BYTE && byte <= NINE_BYTE - - priority_byte += 1 - end - end - end + def rfc3164_space_cursor(text, cursor) + return if @with_priority && !RFC3164_PRI_DIGITS_REGEXP.match?(text.slice(1, cursor - 2)) @time_format_starts_with_space_rfc3164 ? cursor : cursor + 1 end diff --git a/test/plugin/test_in_syslog.rb b/test/plugin/test_in_syslog.rb index 09b5136524..9132a7f27f 100755 --- a/test/plugin/test_in_syslog.rb +++ b/test/plugin/test_in_syslog.rb @@ -191,6 +191,7 @@ def test_space_after_input_owned_priority_preserves_input_priority_syntax(parser messages.each { |message| u.send(message, 0) } end + assert_equal(3, d.events.size) assert_equal(d.events[0], d.events[1]) assert_equal('syslog.unmatched', d.events[2][0]) assert_equal(messages[2], d.events[2][2]['unmatched_line']) diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb index 10345d6bc9..e16c19254e 100644 --- a/test/plugin/test_parser_syslog.rb +++ b/test/plugin/test_parser_syslog.rb @@ -15,6 +15,25 @@ def setup } end + def assert_input_rejected(text, expectation) + case expectation + when :raises + assert_raise(Fluent::TimeParser::TimeParseError) do + @parser.instance.parse(text) { |time, record| } + end + when :rejects + result = :not_yielded + assert_nothing_raised do + @parser.instance.parse(text) do |time, record| + result = [time, record] + end + end + assert_equal([nil, nil], result) + else + flunk("unknown expectation: #{expectation.inspect}") + end + end + data('regexp' => 'regexp', 'string' => 'string') def test_parse(param) @parser.configure({'parser_type' => param}) @@ -157,15 +176,19 @@ def test_parse_rfc3164_time_variants_with_space_after_priority(data) end data( - 'rfc3164/parser priority' => ['rfc3164', true], - 'auto/parser priority' => ['auto', true], - 'rfc3164/input priority' => ['rfc3164', false], - 'auto/input priority' => ['auto', false], + 'regexp/rfc3164/parser priority' => ['regexp', 'rfc3164', true], + 'string/rfc3164/parser priority' => ['string', 'rfc3164', true], + 'regexp/auto/parser priority' => ['regexp', 'auto', true], + 'string/auto/parser priority' => ['string', 'auto', true], + 'regexp/rfc3164/input priority' => ['regexp', 'rfc3164', false], + 'string/rfc3164/input priority' => ['string', 'rfc3164', false], + 'regexp/auto/input priority' => ['regexp', 'auto', false], + 'string/auto/input priority' => ['string', 'auto', false], ) def test_truncated_subsecond_rfc3164_is_rejected_without_raising(data) - message_format, with_priority = data + parser_engine, message_format, with_priority = data @parser.configure( - 'parser_engine' => 'string', + 'parser_engine' => parser_engine, 'message_format' => message_format, 'with_priority' => with_priority, 'time_format' => '%b %d %H:%M:%S.%N', @@ -193,6 +216,46 @@ def test_truncated_subsecond_rfc3164_is_rejected_without_raising(data) assert_equal([nil, nil], result) end + data( + 'rfc3164/parser priority' => ['rfc3164', true], + 'auto/parser priority' => ['auto', true], + 'rfc3164/input priority' => ['rfc3164', false], + 'auto/input priority' => ['auto', false], + ) + def test_non_ascii_rfc3164_header_is_rejected_without_raising(data) + message_format, with_priority = data + @parser.configure( + 'parser_engine' => 'string', + 'message_format' => message_format, + 'with_priority' => with_priority, + ) + prefix = with_priority ? '<14>' : '' + + assert_input_rejected("#{prefix}Apr 25 16:43:é host app: message", :rejects) + end + + data( + 'regexp/subsecond/parser priority' => ['regexp', '%b %d %H:%M:%S.%L', 'Apr 25', true], + 'string/subsecond/parser priority' => ['string', '%b %d %H:%M:%S.%L', 'Apr 25', true], + 'regexp/subsecond/input priority' => ['regexp', '%b %d %H:%M:%S.%L', 'Apr 25', false], + 'string/subsecond/input priority' => ['string', '%b %d %H:%M:%S.%L', 'Apr 25', false], + 'regexp/iso8601/parser priority' => ['regexp', '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29', true], + 'string/iso8601/parser priority' => ['string', '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29', true], + 'regexp/iso8601/input priority' => ['regexp', '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29', false], + 'string/iso8601/input priority' => ['string', '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29', false], + ) + def test_truncated_rfc3164_header_is_rejected_without_raising(data) + parser_engine, time_format, timestamp, with_priority = data + @parser.configure( + 'parser_engine' => parser_engine, + 'with_priority' => with_priority, + 'time_format' => time_format, + ) + prefix = with_priority ? '<14> ' : ' ' + + assert_input_rejected("#{prefix}#{timestamp}", :rejects) + end + data( 'regexp/parser priority' => ['regexp', true], 'string/parser priority' => ['string', true], @@ -225,33 +288,24 @@ def test_parse_with_leading_space_time_format(data) end data( - 'regexp/two spaces' => ['regexp', true, '<14> Apr 25 16:43:29 host app: message'], - 'string/two spaces' => ['string', true, '<14> Apr 25 16:43:29 host app: message'], - 'regexp/tab' => ['regexp', true, "<14>\tApr 25 16:43:29 host app: message"], - 'string/tab' => ['string', true, "<14>\tApr 25 16:43:29 host app: message"], - 'regexp/two leading spaces' => ['regexp', false, ' Apr 25 16:43:29 host app: message'], - 'string/two leading spaces' => ['string', false, ' Apr 25 16:43:29 host app: message'], - 'regexp/leading tab' => ['regexp', false, "\tApr 25 16:43:29 host app: message"], - 'string/leading tab' => ['string', false, "\tApr 25 16:43:29 host app: message"], - 'regexp/long priority' => ['regexp', true, '<1234> Apr 25 16:43:29 host app: message'], - 'string/long priority' => ['string', true, '<1234> Apr 25 16:43:29 host app: message'], - 'regexp/non-numeric priority' => ['regexp', true, ' Apr 25 16:43:29 host app: message'], - 'string/non-numeric priority' => ['string', true, ' Apr 25 16:43:29 host app: message'], + 'regexp/two spaces' => ['regexp', true, '<14> Apr 25 16:43:29 host app: message', :raises], + 'string/two spaces' => ['string', true, '<14> Apr 25 16:43:29 host app: message', :rejects], + 'regexp/tab' => ['regexp', true, "<14>\tApr 25 16:43:29 host app: message", :raises], + 'string/tab' => ['string', true, "<14>\tApr 25 16:43:29 host app: message", :rejects], + 'regexp/two leading spaces' => ['regexp', false, ' Apr 25 16:43:29 host app: message', :raises], + 'string/two leading spaces' => ['string', false, ' Apr 25 16:43:29 host app: message', :rejects], + 'regexp/leading tab' => ['regexp', false, "\tApr 25 16:43:29 host app: message", :raises], + 'string/leading tab' => ['string', false, "\tApr 25 16:43:29 host app: message", :rejects], + 'regexp/long priority' => ['regexp', true, '<1234> Apr 25 16:43:29 host app: message', :rejects], + 'string/long priority' => ['string', true, '<1234> Apr 25 16:43:29 host app: message', :rejects], + 'regexp/non-numeric priority' => ['regexp', true, ' Apr 25 16:43:29 host app: message', :rejects], + 'string/non-numeric priority' => ['string', true, ' Apr 25 16:43:29 host app: message', :rejects], ) def test_parse_does_not_broaden_rfc3164_priority_separator(data) - parser_engine, with_priority, text = data + parser_engine, with_priority, text, expectation = data @parser.configure('parser_engine' => parser_engine, 'with_priority' => with_priority) - parsed = false - begin - @parser.instance.parse(text) do |time, record| - parsed = !!(time && record) - end - rescue Fluent::TimeParser::TimeParseError - # The regexp and string parsers report invalid input differently. - end - - assert_false(parsed) + assert_input_rejected(text, expectation) end data( @@ -296,34 +350,25 @@ def test_leading_space_time_format_does_not_broaden_priority_syntax(text) end data( - 'regexp/rfc5424/with priority' => ['regexp', 'rfc5424', true], - 'string/rfc5424/with priority' => ['string', 'rfc5424', true], - 'regexp/auto/with priority' => ['regexp', 'auto', true], - 'string/auto/with priority' => ['string', 'auto', true], - 'regexp/rfc5424/without priority' => ['regexp', 'rfc5424', false], - 'string/rfc5424/without priority' => ['string', 'rfc5424', false], - 'regexp/auto/without priority' => ['regexp', 'auto', false], - 'string/auto/without priority' => ['string', 'auto', false], + 'regexp/rfc5424/with priority' => ['regexp', 'rfc5424', true, :rejects], + 'string/rfc5424/with priority' => ['string', 'rfc5424', true, :raises], + 'regexp/auto/with priority' => ['regexp', 'auto', true, :raises], + 'string/auto/with priority' => ['string', 'auto', true, :rejects], + 'regexp/rfc5424/without priority' => ['regexp', 'rfc5424', false, :raises], + 'string/rfc5424/without priority' => ['string', 'rfc5424', false, :raises], + 'regexp/auto/without priority' => ['regexp', 'auto', false, :raises], + 'string/auto/without priority' => ['string', 'auto', false, :rejects], ) def test_parse_does_not_accept_space_after_rfc5424_priority(data) - parser_engine, message_format, with_priority = data + parser_engine, message_format, with_priority, expectation = data @parser.configure( 'parser_engine' => parser_engine, 'message_format' => message_format, 'with_priority' => with_priority, ) text = with_priority ? '<14> 1 2026-04-25T16:43:29Z host app 123 ID - message' : ' 1 2026-04-25T16:43:29Z host app 123 ID - message' - parsed = false - begin - @parser.instance.parse(text) do |time, record| - parsed = !!(time && record) - end - rescue Fluent::TimeParser::TimeParseError - # The regexp and string parsers report invalid input differently. - end - - assert_false(parsed) + assert_input_rejected(text, expectation) end data('regexp' => 'regexp', 'string' => 'string') From 942ef831bdf618227ab7462ab90e4547eb7f3979 Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Wed, 9 Sep 2026 17:30:48 +0900 Subject: [PATCH 5/5] parser_syslog: reject a broken timestamp instead of raising Accepting the boundary space moves the cursor one character forward, so a record whose timestamp is malformed can line up with the fixed-width lookup and reach the time parser. The string parser then raised Fluent::TimeParser::TimeParseError where it used to yield nil, nil. RFC 3164 asks receivers to be liberal in what they accept and not to malfunction on malformed messages, and in_tail cannot apply emit_unmatched_lines once the parser raises. Turn the error back into the documented nil, nil rejection. Signed-off-by: Shizuo Fujita Co-authored-by: Claude Opus 5 (1M context) --- lib/fluent/plugin/parser_syslog.rb | 7 ++++++- test/plugin/test_parser_syslog.rb | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb index 901a5e66ca..3938c13ed6 100644 --- a/lib/fluent/plugin/parser_syslog.rb +++ b/lib/fluent/plugin/parser_syslog.rb @@ -399,7 +399,12 @@ def parse_rfc3164(text, &block) msg.chomp! record['message'] = msg - time = @time_parser_rfc3164.parse(time_str) + begin + time = @time_parser_rfc3164.parse(time_str) + rescue Fluent::TimeParser::TimeParseError + yield nil, nil + return + end record['time'] = time_str if @keep_time_key yield time, record diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb index e16c19254e..f7e8655c45 100644 --- a/test/plugin/test_parser_syslog.rb +++ b/test/plugin/test_parser_syslog.rb @@ -234,6 +234,20 @@ def test_non_ascii_rfc3164_header_is_rejected_without_raising(data) assert_input_rejected("#{prefix}Apr 25 16:43:é host app: message", :rejects) end + data( + 'two boundary spaces' => [nil, '<14> Apr 25 16:43:2 host app: message'], + 'subsecond format without subsecond' => ['%b %d %H:%M:%S.%N', '<14> Apr 25 16:43:29 host app: message'], + 'subsecond format truncated second' => ['%b %d %H:%M:%S.%N', '<14> Apr 25 16:43:2 host app: message'], + ) + def test_broken_timestamp_is_rejected_without_raising(data) + time_format, text = data + config = {'parser_engine' => 'string', 'with_priority' => true} + config['time_format'] = time_format if time_format + @parser.configure(config) + + assert_input_rejected(text, :rejects) + end + data( 'regexp/subsecond/parser priority' => ['regexp', '%b %d %H:%M:%S.%L', 'Apr 25', true], 'string/subsecond/parser priority' => ['string', '%b %d %H:%M:%S.%L', 'Apr 25', true],