Skip to content

Commit dd1c96b

Browse files
kyleconroykyleconroy
authored
ast: tagged, snake_case JSON output for parse and analyze (#4592)
1 parent 2249e14 commit dd1c96b

266 files changed

Lines changed: 2074 additions & 1514 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/howto/analyze.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,4 +106,6 @@ reports the result columns and parameters:
106106
]
107107
```
108108

109-
Pass `--ast` to also include each statement's parsed AST under an `ast` key.
109+
Pass `--ast` to also include each statement's parsed AST under an `ast` key. It
110+
has the same shape as the output of [`parse`](parse.md), with every node tagged
111+
by type.

‎docs/howto/parse.md‎

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,64 @@ The output is a JSON array with one object per statement:
4848
"name": "GetAuthor",
4949
"cmd": ":one",
5050
"ast": {
51-
"Stmt": {
51+
"tag": "RawStmt",
52+
"stmt": {
5253
"...": "..."
5354
},
54-
"StmtLocation": 0,
55-
"StmtLen": 42
55+
"stmt_location": 0,
56+
"stmt_len": 42
5657
}
5758
}
5859
]
5960
```
6061

6162
Statements without a `-- name:` annotation (for example schema DDL) omit the
62-
`name` and `cmd` fields.
63+
`name` and `cmd` fields. Field names are `snake_case` versions of the AST node
64+
field names.
65+
66+
## Node types
67+
68+
Every node in the AST carries a `tag` naming its type. Some nodes have no
69+
fields of their own, so without it a star, a null literal and an untranslated
70+
clause would all print as `{}`.
71+
72+
```json
73+
"val": {
74+
"tag": "ColumnRef",
75+
"name": "",
76+
"fields": {
77+
"tag": "List",
78+
"items": [
79+
{
80+
"tag": "A_Star"
81+
}
82+
]
83+
},
84+
"location": 93
85+
}
86+
```
87+
88+
A `tag` of `TODO` marks a clause the dialect's converter does not translate
89+
yet. It means the clause was parsed but is not represented in the AST, not that
90+
the clause was absent from the query.
91+
92+
## Absent fields
93+
94+
A field the statement does not use is left out rather than printed as `null`.
95+
An `A_Const` carrying an integer reports only what it has:
96+
97+
```json
98+
"val": {
99+
"tag": "A_Const",
100+
"val": {
101+
"tag": "Integer",
102+
"ival": 1
103+
},
104+
"location": 30
105+
}
106+
```
107+
108+
Only absent fields — and empty lists, which engines construct differently for
109+
the same SQL — are omitted. A zero keeps its place, because zero is a value the
110+
parser can find: `stmt_location` is 0 for the first statement in a file, and
111+
`LIMIT 0` parses to an `ival` of 0.

‎internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt‎

Lines changed: 75 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -24,99 +24,113 @@
2424
}
2525
],
2626
"ast": {
27-
"Stmt": {
28-
"DistinctClause": {
29-
"Items": null
27+
"tag": "RawStmt",
28+
"stmt": {
29+
"tag": "SelectStmt",
30+
"distinct_clause": {
31+
"tag": "List"
3032
},
31-
"IntoClause": null,
32-
"TargetList": {
33-
"Items": [
33+
"target_list": {
34+
"tag": "List",
35+
"items": [
3436
{
35-
"Name": null,
36-
"Indirection": {
37-
"Items": null
37+
"tag": "ResTarget",
38+
"indirection": {
39+
"tag": "List"
3840
},
39-
"Val": {
40-
"Name": "",
41-
"Fields": {
42-
"Items": [
41+
"val": {
42+
"tag": "ColumnRef",
43+
"name": "",
44+
"fields": {
45+
"tag": "List",
46+
"items": [
4347
{
44-
"Str": "name"
48+
"tag": "String",
49+
"str": "name"
4550
}
4651
]
4752
},
48-
"Location": 35
53+
"location": 35
4954
},
50-
"Location": 35
55+
"location": 35
5156
}
5257
]
5358
},
54-
"FromClause": {
55-
"Items": [
59+
"from_clause": {
60+
"tag": "List",
61+
"items": [
5662
{
57-
"Catalogname": null,
58-
"Schemaname": null,
59-
"Relname": "authors",
60-
"Inh": true,
61-
"Relpersistence": 112,
62-
"Alias": null,
63-
"Location": 45
63+
"tag": "RangeVar",
64+
"relname": "authors",
65+
"inh": true,
66+
"relpersistence": 112,
67+
"location": 45
6468
}
6569
]
6670
},
67-
"WhereClause": {
68-
"Kind": 1,
69-
"Name": {
70-
"Items": [
71+
"where_clause": {
72+
"tag": "A_Expr",
73+
"kind": 1,
74+
"name": {
75+
"tag": "List",
76+
"items": [
7177
{
72-
"Str": "="
78+
"tag": "String",
79+
"str": "="
7380
}
7481
]
7582
},
76-
"Lexpr": {
77-
"Name": "",
78-
"Fields": {
79-
"Items": [
83+
"lexpr": {
84+
"tag": "ColumnRef",
85+
"name": "",
86+
"fields": {
87+
"tag": "List",
88+
"items": [
8089
{
81-
"Str": "id"
90+
"tag": "String",
91+
"str": "id"
8292
}
8393
]
8494
},
85-
"Location": 59
95+
"location": 59
8696
},
87-
"Rexpr": {
88-
"Number": 1,
89-
"Location": 64,
90-
"Dollar": true
97+
"rexpr": {
98+
"tag": "ParamRef",
99+
"number": 1,
100+
"location": 64,
101+
"dollar": true
91102
},
92-
"Location": 62
103+
"location": 62
93104
},
94-
"GroupClause": {
95-
"Items": null
105+
"group_clause": {
106+
"tag": "List"
96107
},
97-
"HavingClause": {},
98-
"WindowClause": {
99-
"Items": null
108+
"having_clause": {
109+
"tag": "TODO"
100110
},
101-
"ValuesLists": {
102-
"Items": null
111+
"window_clause": {
112+
"tag": "List"
103113
},
104-
"SortClause": {
105-
"Items": null
114+
"values_lists": {
115+
"tag": "List"
106116
},
107-
"LimitOffset": {},
108-
"LimitCount": {},
109-
"LockingClause": {
110-
"Items": null
117+
"sort_clause": {
118+
"tag": "List"
111119
},
112-
"WithClause": null,
113-
"Op": 0,
114-
"All": false,
115-
"Larg": null,
116-
"Rarg": null
120+
"limit_offset": {
121+
"tag": "TODO"
122+
},
123+
"limit_count": {
124+
"tag": "TODO"
125+
},
126+
"locking_clause": {
127+
"tag": "List"
128+
},
129+
"op": 0,
130+
"all": false
117131
},
118-
"StmtLocation": 0,
119-
"StmtLen": 66
132+
"stmt_location": 0,
133+
"stmt_len": 66
120134
}
121135
}
122136
]

‎internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt‎

Lines changed: 18 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,42 +3,31 @@
33
"name": "GetValue",
44
"cmd": ":one",
55
"ast": {
6-
"Stmt": {
7-
"DistinctClause": null,
8-
"IntoClause": null,
9-
"TargetList": {
10-
"Items": [
6+
"tag": "RawStmt",
7+
"stmt": {
8+
"tag": "SelectStmt",
9+
"target_list": {
10+
"tag": "List",
11+
"items": [
1112
{
12-
"Name": null,
13-
"Indirection": null,
14-
"Val": {
15-
"Val": {
16-
"Ival": 1
13+
"tag": "ResTarget",
14+
"val": {
15+
"tag": "A_Const",
16+
"val": {
17+
"tag": "Integer",
18+
"ival": 1
1719
},
18-
"Location": 31
20+
"location": 31
1921
},
20-
"Location": 31
22+
"location": 31
2123
}
2224
]
2325
},
24-
"FromClause": null,
25-
"WhereClause": null,
26-
"GroupClause": null,
27-
"HavingClause": null,
28-
"WindowClause": null,
29-
"ValuesLists": null,
30-
"SortClause": null,
31-
"LimitOffset": null,
32-
"LimitCount": null,
33-
"LockingClause": null,
34-
"WithClause": null,
35-
"Op": 0,
36-
"All": false,
37-
"Larg": null,
38-
"Rarg": null
26+
"op": 0,
27+
"all": false
3928
},
40-
"StmtLocation": 0,
41-
"StmtLen": 32
29+
"stmt_location": 0,
30+
"stmt_len": 32
4231
}
4332
}
4433
]

‎internal/endtoend/testdata/parse_basic/duckdb/stdout.txt‎

Lines changed: 18 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,42 +3,31 @@
33
"name": "GetValue",
44
"cmd": ":one",
55
"ast": {
6-
"Stmt": {
7-
"DistinctClause": null,
8-
"IntoClause": null,
9-
"TargetList": {
10-
"Items": [
6+
"tag": "RawStmt",
7+
"stmt": {
8+
"tag": "SelectStmt",
9+
"target_list": {
10+
"tag": "List",
11+
"items": [
1112
{
12-
"Name": null,
13-
"Indirection": null,
14-
"Val": {
15-
"Val": {
16-
"Ival": 1
13+
"tag": "ResTarget",
14+
"val": {
15+
"tag": "A_Const",
16+
"val": {
17+
"tag": "Integer",
18+
"ival": 1
1719
},
18-
"Location": 30
20+
"location": 30
1921
},
20-
"Location": 30
22+
"location": 30
2123
}
2224
]
2325
},
24-
"FromClause": null,
25-
"WhereClause": null,
26-
"GroupClause": null,
27-
"HavingClause": null,
28-
"WindowClause": null,
29-
"ValuesLists": null,
30-
"SortClause": null,
31-
"LimitOffset": null,
32-
"LimitCount": null,
33-
"LockingClause": null,
34-
"WithClause": null,
35-
"Op": 0,
36-
"All": false,
37-
"Larg": null,
38-
"Rarg": null
26+
"op": 0,
27+
"all": false
3928
},
40-
"StmtLocation": 0,
41-
"StmtLen": 33
29+
"stmt_location": 0,
30+
"stmt_len": 33
4231
}
4332
}
4433
]

0 commit comments

Comments
 (0)