From 2114d1bb5dbd411b0800fdd44b75636d9b18588f Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 02:59:43 +0000 Subject: [PATCH 01/11] fix(oracle): accept non-reserved keywords as column names, gate on Oracle's own PL/SQL (BYT-10301) Two engine-driven gates, from the BYT-10301 investigation: TestOraclePLSQLDictionary parses every unwrapped, VALID procedure, function, package, package body, type body, and trigger of the Oracle-maintained schemas in the pinned image (875 units) through Split and ParseRange, as Bytebase runs a script. The 102 units that fail today are recorded with their category in plsql_dictionary_known_failures.tsv; a new failure is a regression and a fixed unit must leave the list, so it only shrinks (-update regenerates it). Oracle's source is read at test time, never committed. TestOracleNonReservedKeywordsAsColumns takes every word omni lexes as a keyword that V$RESERVED_WORDS does not reserve (232) and checks it as a column against the engine: defined in CREATE TABLE, then referenced from a select list, WHERE, ORDER BY, UPDATE, an INSERT column list, and an index (1,624 checks). The keyword manifest only covered the definition, which is how OFFSET regressed in #118/#154 unnoticed. The audit found 36 disagreements; 35 were omni rejecting valid SQL and are fixed: - FETCH, JOIN, MODEL, USING (like OFFSET) start an operand as a column; only reserved clause keywords end one. - CAST, DECODE, INTERVAL name a column unless followed by '(' or a string literal; CASE names one when the next token cannot continue a CASE expression (Oracle reports `SELECT case` as ORA-00904, not a syntax error, so strictness row expr_014 now uses SELECT CASE 1). - PRIMARY and FOREIGN start a table constraint only before KEY. - PRIVILEGES is not reserved (V$RESERVED_WORDS, and absent from the 26ai reserved-word list); CONTENT and JOIN work as schema-qualified table names. The keyword manifest rows now say so. - A statement may end with a column named CONTENT, JOIN, or USING, so they leave the incomplete-statement heuristic; truncated clauses still fail in their own parsers. The one remaining gap, XMLELEMENT as an index column (Oracle rejects, omni accepts), stays in oracle_keyword_column_gaps.tsv. The Oracle image is pinned to gvenzl/oracle-free:23.26.3-slim-faststart, the same digest as the 23-slim-faststart tag used so far, so the dictionary the manifest describes cannot drift; AGENTS.md lists it. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- oracle/parser/create_table.go | 8 +- oracle/parser/expr.go | 40 ++- oracle/parser/keyword_column_oracle_test.go | 138 ++++++++++ oracle/parser/keywords.go | 2 +- oracle/parser/oracle_helper_test.go | 2 +- oracle/parser/parser.go | 9 +- oracle/parser/plsql_dictionary_oracle_test.go | 255 ++++++++++++++++++ oracle/parser/select.go | 7 +- .../coverage/oracle_keyword_column_gaps.tsv | 2 + .../testdata/coverage/oracle_keywords.tsv | 10 +- .../plsql_dictionary_known_failures.tsv | 103 +++++++ .../testdata/coverage/strictness_v2.tsv | 2 +- 13 files changed, 557 insertions(+), 23 deletions(-) create mode 100644 oracle/parser/keyword_column_oracle_test.go create mode 100644 oracle/parser/plsql_dictionary_oracle_test.go create mode 100644 oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv create mode 100644 oracle/parser/testdata/coverage/plsql_dictionary_known_failures.tsv diff --git a/AGENTS.md b/AGENTS.md index 6e59ad77..f8aacae5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ The rules above are the target. These places do not meet them yet; do not copy t - Engine dialect and DDL fidelity tests live here, beside the package they cover. Bytebase tests its API and workflow against PostgreSQL only and trusts omni's result, so a behavior Bytebase depends on is pinned by an omni test. - Test against the engine, not against memory. Parser acceptance, catalog results, and generated DDL are verified against a real engine wherever one can run: testcontainers for pg, mysql, mariadb, tidb, mssql, doris, and starrocks; Trino and the Spanner emulator as CI services; Oracle and Redshift through `ORACLE_PARSER_REF_DSN` and `REDSHIFT_COMPAT_DSN`. Gate a test on an environment variable only when it needs a server CI cannot start. - Container tests fail when Docker is missing; they do not skip. CI runs every test on every PR with no `-short` and no nightly. Never check `testing.Short()`, and never park a known gap behind a skip. -- Pinned images are part of the recorded expectations: `trinodb/trino:482` and the Spanner emulator `1.5.54` in `.github/workflows/ci.yml`, `postgres:17-alpine` in `pg/parser`. Upgrading one means rerunning its oracle tests and reading the drift, not editing expectations to match. +- Pinned images are part of the recorded expectations: `trinodb/trino:482` and the Spanner emulator `1.5.54` in `.github/workflows/ci.yml`, `postgres:17-alpine` in `pg/parser`, `gvenzl/oracle-free:23.26.3-slim-faststart` in `oracle/parser`. Upgrading one means rerunning its oracle tests and reading the drift, not editing expectations to match. - CI runs the sweeps at `PAREN_FUZZ_SIZE=10000`, `SPLITTEST_N=1000000`, and `S3DIFF_N=2000`; local defaults are smaller. Trino packages run one at a time because they share the service and run stateful DDL against the same names. - A PR runs the packages `scripts/affected-packages.sh` derives from the diff. A change to `go.mod`, `proto/`, `scripts/`, `harness/`, `docs/`, or any root file runs everything. diff --git a/oracle/parser/create_table.go b/oracle/parser/create_table.go index 2b4ed83d..baeffcf9 100644 --- a/oracle/parser/create_table.go +++ b/oracle/parser/create_table.go @@ -564,10 +564,10 @@ func (p *Parser) isTableConstraintStart() bool { case kwCONSTRAINT: next := p.peekNext() return !isOracleTypeToken(next.Type) - case kwPRIMARY: - return true - case kwFOREIGN: - return true + case kwPRIMARY, kwFOREIGN: + // PRIMARY and FOREIGN are not reserved: without KEY they start a + // column definition (engine-checked by TestOracleNonReservedKeywordsAsColumns). + return p.peekNext().Type == kwKEY case kwUNIQUE: // UNIQUE could be a column constraint when it follows a type, // but at the top level of the column list it's a table constraint diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index 54f00193..6385d87a 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -583,9 +583,17 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { return p.parseParenExpr() case kwCASE: + if p.caseNamesColumn() { + return p.parseIdentExpr() + } return p.parseCaseExpr() + // CAST, DECODE, and INTERVAL are not reserved; without the '(' or the + // string literal that opens the function or literal, they name a column. case kwCAST: + if p.peekNext().Type != '(' { + return p.parseIdentExpr() + } return p.parseCastExpr() case kwDATE, kwTIMESTAMP: @@ -595,6 +603,9 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { return p.parseIdentExpr() case kwDECODE: + if p.peekNext().Type != '(' { + return p.parseIdentExpr() + } return p.parseDecodeExpr() case kwEXISTS: @@ -633,6 +644,9 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { return p.parseJsonPathFunc("JSON_MERGEPATCH") case kwINTERVAL: + if p.peekNext().Type != tokSCONST { + return p.parseIdentExpr() + } return p.parseIntervalExpr() default: @@ -642,10 +656,12 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { } // Identifier — could be column ref, function call, or keyword-as-identifier. - // OFFSET is not reserved in Oracle and names columns in practice; it - // starts the row_limiting_clause only after a complete expression or - // table reference, never where an operand begins. - if isOracleClauseStarterKeyword(p.cur.Type) && p.cur.Type != kwOFFSET { + // A clause keyword Oracle reserves (FROM, WHERE, ...) ends the operand. + // The ones it does not reserve (FETCH, JOIN, MODEL, OFFSET, USING) name + // columns in practice and start their clauses only after a complete + // expression or table reference, never where an operand begins. + // TestOracleNonReservedKeywordsAsColumns checks them on the engine. + if isOracleClauseStarterKeyword(p.cur.Type) && isOracleSQLReservedKeyword(p.cur) { return nil, nil } if p.isIdentLike() { @@ -656,6 +672,22 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { } } +// caseNamesColumn reports whether CASE at the current token is a column +// reference rather than a CASE expression. Oracle does not reserve CASE, and +// a CASE expression always continues with WHEN or an operand, never with a +// token that ends an operand or joins two of them. +func (p *Parser) caseNamesColumn() bool { + switch p.peekNext().Type { + case tokEOF, ';', ',', ')', '.', '=', '<', '>', '*', '/', + tokLESSEQ, tokGREATEQ, tokNOTEQ, tokCONCAT, + kwFROM, kwWHERE, kwGROUP, kwORDER, kwHAVING, kwINTO, kwAS, + kwIS, kwIN, kwNOT, kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4, kwBETWEEN, + kwAND, kwOR, kwASC, kwDESC, kwNULLS, kwUNION, kwINTERSECT, kwMINUS: + return true + } + return false +} + // parseDateTimeLiteral parses ANSI datetime literals such as DATE '2020-01-01'. func (p *Parser) parseDateTimeLiteral() (nodes.ExprNode, error) { start := p.pos() diff --git a/oracle/parser/keyword_column_oracle_test.go b/oracle/parser/keyword_column_oracle_test.go new file mode 100644 index 00000000..69905c03 --- /dev/null +++ b/oracle/parser/keyword_column_oracle_test.go @@ -0,0 +1,138 @@ +package parser + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +var oracleKeywordColumnGaps = filepath.Join("testdata", "coverage", "oracle_keyword_column_gaps.tsv") + +// keywordColumnContexts reference a column named by the keyword after it is +// defined; %[1]s is the table and %[2]s the keyword. +var keywordColumnContexts = []struct { + name string + sql string +}{ + {"select list", "SELECT %[2]s FROM %[1]s"}, + {"where", "SELECT a FROM %[1]s WHERE %[2]s IS NULL"}, + {"order by", "SELECT a FROM %[1]s ORDER BY %[2]s"}, + {"update", "UPDATE %[1]s SET %[2]s = NULL WHERE %[2]s IS NULL"}, + {"insert columns", "INSERT INTO %[1]s (%[2]s) VALUES (1)"}, + {"index", "CREATE INDEX %[1]s_i ON %[1]s (%[2]s)"}, +} + +// TestOracleNonReservedKeywordsAsColumns checks every word omni lexes as a +// keyword that Oracle does not reserve (V$RESERVED_WORDS) as a column name: +// defined by CREATE TABLE, then referenced from a select list, a WHERE +// predicate, ORDER BY, UPDATE, an INSERT column list, and an index. omni must +// accept or reject each statement exactly as Oracle does. The keyword +// manifest's column_name context covers only the definition; OFFSET, FETCH, +// JOIN, MODEL, and USING failed in the references alone. +// +// Pseudo-column keywords are left out: a reference resolves to the +// pseudo-column (SYSTIMESTAMP compares as a TIMESTAMP), not to the column. +// Disagreements still open are recorded in oracle_keyword_column_gaps.tsv; +// the list only shrinks, and -update regenerates it. +func TestOracleNonReservedKeywordsAsColumns(t *testing.T) { + o := startOracleDB(t) + + reserved := map[string]bool{} + rows, err := o.adminDB.QueryContext(o.ctx, ` +SELECT keyword FROM v$reserved_words + WHERE keyword IS NOT NULL + AND (reserved = 'Y' OR res_type = 'Y' OR res_attr = 'Y' OR res_semi = 'Y')`) + if err != nil { + t.Fatalf("query v$reserved_words: %v", err) + } + for rows.Next() { + var keyword string + if err := rows.Scan(&keyword); err != nil { + t.Fatalf("scan v$reserved_words: %v", err) + } + reserved[keyword] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate v$reserved_words: %v", err) + } + rows.Close() + + var words []string + for word, tokenType := range oracleKeywords { + if reserved[word] || isOraclePseudoColumnKeyword(tokenType) { + continue + } + words = append(words, word) + } + sort.Strings(words) + if len(words) < 200 { + t.Fatalf("checked %d non-reserved keywords, want at least 200", len(words)) + } + + known := map[string]coverageRow{} + if !*update { + gaps, _ := readCoverageTSV(t, oracleKeywordColumnGaps) + for _, row := range gaps { + known[row.Key] = row + } + } + + run := fmt.Sprintf("%d", time.Now().UnixNano()%1_000_000_000) + var gaps []string + var newGaps, fixed, checks int + // check records a disagreement with the table renamed to t, so the + // manifest does not change from run to run. + check := func(word, context, table, sqlText string, oracleAccepts bool) { + checks++ + _, omniErr := Parse(sqlText) + key := word + ":" + context + _, wasKnown := known[key] + if oracleAccepts == (omniErr == nil) { + if wasKnown { + fixed++ + t.Errorf("FIXED %s now matches Oracle: remove it from %s", key, oracleKeywordColumnGaps) + } + return + } + gaps = append(gaps, fmt.Sprintf("%s\t%v\t%v\t%s", key, oracleAccepts, omniErr == nil, strings.ReplaceAll(sqlText, table, "t"))) + if !wasKnown && !*update { + newGaps++ + t.Errorf("NEW GAP %s: oracle_accepts=%v omni_err=%v sql=%q", key, oracleAccepts, omniErr, sqlText) + } + } + + for i, word := range words { + table := fmt.Sprintf("omni_kw_%s_%d", run, i) + create := fmt.Sprintf("CREATE TABLE %s (a NUMBER, %s NUMBER)", table, word) + _, createErr := o.db.ExecContext(o.ctx, create) + check(word, "column definition", table, create, createErr == nil) + if createErr != nil { + continue + } + for _, c := range keywordColumnContexts { + sqlText := fmt.Sprintf(c.sql, table, word) + check(word, c.name, table, sqlText, oracleParseOnly(o.ctx, o.db, sqlText) == nil) + } + if _, err := o.db.ExecContext(o.ctx, "DROP TABLE "+table+" PURGE"); err != nil { + t.Logf("drop %s: %v", table, err) + } + } + + t.Logf("non-reserved keywords as columns: %d words, %d checks, %d disagree with Oracle (%d new, %d fixed)", + len(words), checks, len(gaps), newGaps, fixed) + + if *update { + content := "case\toracle_accepts\tomni_accepts\tsql\n" + strings.Join(gaps, "\n") + if len(gaps) > 0 { + content += "\n" + } + if err := os.WriteFile(oracleKeywordColumnGaps, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", oracleKeywordColumnGaps, err) + } + t.Logf("wrote %s (%d gaps)", oracleKeywordColumnGaps, len(gaps)) + } +} diff --git a/oracle/parser/keywords.go b/oracle/parser/keywords.go index 85bd6634..1ab4bbfe 100644 --- a/oracle/parser/keywords.go +++ b/oracle/parser/keywords.go @@ -140,7 +140,7 @@ func isOracleSQLReservedKeyword(tok Token) bool { kwINTO, kwIS, kwLEVEL, kwLIKE, kwLOCK, kwLONG, kwMAXEXTENTS, kwMINUS, kwMLSLABEL, kwMODE, kwMODIFY, kwNESTED_TABLE_ID, kwNOAUDIT, kwNOCOMPRESS, kwNOT, kwNOWAIT, kwNULL, kwNUMBER, kwOF, kwOFFLINE, kwON, kwONLINE, - kwOPTION, kwOR, kwORDER, kwPCTFREE, kwPRIOR, kwPRIVILEGES, + kwOPTION, kwOR, kwORDER, kwPCTFREE, kwPRIOR, kwPUBLIC, kwRAW, kwRENAME, kwRESOURCE, kwREVOKE, kwROW, kwROWID, kwROWNUM, kwROWS, kwSELECT, kwSESSION, kwSET, kwSHARE, kwSIZE, kwSMALLINT, kwSTART, kwSUCCESSFUL, kwSYNONYM, kwSYSDATE, kwTABLE, kwTHEN, diff --git a/oracle/parser/oracle_helper_test.go b/oracle/parser/oracle_helper_test.go index 3ad830d5..f00f9695 100644 --- a/oracle/parser/oracle_helper_test.go +++ b/oracle/parser/oracle_helper_test.go @@ -37,7 +37,7 @@ func startOracleDB(t *testing.T) *oracleDB { ctx := context.Background() req := testcontainers.ContainerRequest{ - Image: "gvenzl/oracle-free:23-slim-faststart", + Image: "gvenzl/oracle-free:23.26.3-slim-faststart", ExposedPorts: []string{"1521/tcp"}, Env: map[string]string{ "ORACLE_PASSWORD": "testpass", diff --git a/oracle/parser/parser.go b/oracle/parser/parser.go index 1eed0648..cfa4971c 100644 --- a/oracle/parser/parser.go +++ b/oracle/parser/parser.go @@ -138,11 +138,14 @@ func (p *Parser) isIncompleteStatementEnd(stmt nodes.StmtNode) bool { return true } + // Keywords Oracle does not reserve (CONTENT, JOIN, USING) are left out: a + // statement can end with a column of that name, and a clause they open + // without its operand ("FROM t JOIN") fails in the clause's own parser. switch stmt.(type) { case *nodes.SelectStmt: switch p.prev.Type { - case kwAND, kwAS, kwBY, kwCONTENT, kwFROM, kwGROUP, kwHAVING, - kwIS, kwJOIN, kwNOT, kwON, kwOR, kwORDER, kwTHEN, kwUNION, + case kwAND, kwAS, kwBY, kwFROM, kwGROUP, kwHAVING, + kwIS, kwNOT, kwON, kwOR, kwORDER, kwTHEN, kwUNION, kwWHERE: return true } @@ -154,7 +157,7 @@ func (p *Parser) isIncompleteStatementEnd(stmt nodes.StmtNode) bool { return p.prev.Type == kwFROM || p.prev.Type == kwWHERE case *nodes.MergeStmt: switch p.prev.Type { - case kwINTO, kwON, kwSET, kwTHEN, kwUSING, kwWHERE: + case kwINTO, kwON, kwSET, kwTHEN, kwWHERE: return true } case *nodes.CreateTableStmt: diff --git a/oracle/parser/plsql_dictionary_oracle_test.go b/oracle/parser/plsql_dictionary_oracle_test.go new file mode 100644 index 00000000..9b75d9e8 --- /dev/null +++ b/oracle/parser/plsql_dictionary_oracle_test.go @@ -0,0 +1,255 @@ +package parser + +import ( + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +var update = flag.Bool("update", false, "rewrite the known-gap manifests of the Oracle dictionary tests from the current results") + +// oracleDictionaryVersion is the release of the pinned Oracle image whose +// dictionary the known-gap manifests were recorded on. +const oracleDictionaryVersion = "23.26.3.0.0" + +var plsqlDictionaryManifest = filepath.Join("testdata", "coverage", "plsql_dictionary_known_failures.tsv") + +// dictionaryUnit is one PL/SQL unit read from DBA_SOURCE. +type dictionaryUnit struct { + key string // OWNER.NAME:TYPE + kind string // DBA_SOURCE type + text string // CREATE OR REPLACE followed by the stored source +} + +// TestOraclePLSQLDictionary parses the PL/SQL that ships with Oracle: every +// unwrapped, VALID procedure, function, package, package body, type body, and +// trigger of an Oracle-maintained schema in the pinned image, about 875 units +// of code the engine compiled. Each unit goes through Split and ParseRange, +// as Bytebase runs a script, and must come out as one segment that parses. +// +// The units that fail today are recorded with a category in +// plsql_dictionary_known_failures.tsv. A unit that starts failing is a +// regression; a recorded unit that now parses must leave the manifest, so the +// list only shrinks. Regenerate it with -update and review the diff. The +// source text itself is Oracle's and is read at test time, never committed. +func TestOraclePLSQLDictionary(t *testing.T) { + units := loadOracleDictionaryUnits(t) + if len(units) < 800 { + t.Fatalf("read %d dictionary units, want at least 800: the DBA_SOURCE query no longer matches the image", len(units)) + } + + known := map[string]coverageRow{} + if !*update { + rows, _ := readCoverageTSV(t, plsqlDictionaryManifest) + for _, row := range rows { + known[row.Key] = row + } + } + + type failure struct { + unit dictionaryUnit + category string + where string + message string + } + var failures []failure + byCategory := map[string]int{} + var newFailures, fixed int + for _, u := range units { + category, where, message, ok := parseDictionaryUnit(u) + if ok { + if _, wasKnown := known[u.key]; wasKnown { + fixed++ + t.Errorf("FIXED %s now parses: remove it from %s", u.key, plsqlDictionaryManifest) + } + continue + } + failures = append(failures, failure{unit: u, category: category, where: where, message: message}) + byCategory[category]++ + if _, wasKnown := known[u.key]; !wasKnown && !*update { + newFailures++ + t.Errorf("NEW FAILURE %s at %s: %s (%s)", u.key, where, message, category) + } + } + + t.Logf("Oracle %s dictionary: %d units, %d fail (%.1f%%), %d new, %d fixed", + oracleDictionaryVersion, len(units), len(failures), + 100*float64(len(failures))/float64(len(units)), newFailures, fixed) + var categories []string + for c := range byCategory { + categories = append(categories, c) + } + sort.Slice(categories, func(i, j int) bool { + if byCategory[categories[i]] != byCategory[categories[j]] { + return byCategory[categories[i]] > byCategory[categories[j]] + } + return categories[i] < categories[j] + }) + for _, c := range categories { + t.Logf(" %4d %s", byCategory[c], c) + } + + if *update { + var b strings.Builder + b.WriteString("unit\tcategory\tposition\terror\n") + for _, f := range failures { + fmt.Fprintf(&b, "%s\t%s\t%s\t%s\n", f.unit.key, f.category, f.where, f.message) + } + if err := os.WriteFile(plsqlDictionaryManifest, []byte(b.String()), 0o644); err != nil { + t.Fatalf("write %s: %v", plsqlDictionaryManifest, err) + } + t.Logf("wrote %s (%d units)", plsqlDictionaryManifest, len(failures)) + } +} + +// loadOracleDictionaryUnits reads the units from the container started by +// startOracleDB, after checking that it runs the release the manifest was +// recorded on. It does not honor ORACLE_PARSER_REF_* DSNs: the manifest +// describes the pinned image's dictionary, not another server's. +func loadOracleDictionaryUnits(t *testing.T) []dictionaryUnit { + t.Helper() + o := startOracleDB(t) + + var version string + if err := o.adminDB.QueryRowContext(o.ctx, + `SELECT version_full FROM product_component_version WHERE product LIKE 'Oracle%'`).Scan(&version); err != nil { + t.Fatalf("read Oracle version: %v", err) + } + if version != oracleDictionaryVersion { + t.Fatalf("Oracle %s is running; the dictionary manifests were recorded on %s. Rerun with -update and review the drift.", + version, oracleDictionaryVersion) + } + + // The dba_objects and dba_users filters are semi-joins: a join repeats + // rows for objects listed in more than one container. + rows, err := o.adminDB.QueryContext(o.ctx, ` +SELECT s.owner, s.name, s.type, s.line, s.text + FROM dba_source s + WHERE s.type IN ('PROCEDURE', 'FUNCTION', 'PACKAGE', 'PACKAGE BODY', 'TYPE BODY', 'TRIGGER') + AND s.owner IN (SELECT username FROM dba_users WHERE oracle_maintained = 'Y') + AND (s.owner, s.name, s.type) IN + (SELECT owner, object_name, object_type FROM dba_objects WHERE status = 'VALID') + AND (s.owner, s.name, s.type) NOT IN + (SELECT owner, name, type FROM dba_source WHERE line = 1 AND LOWER(text) LIKE '%wrapped%') + ORDER BY s.owner, s.type, s.name, s.line`) + if err != nil { + t.Fatalf("query DBA_SOURCE: %v", err) + } + defer rows.Close() + + var units []dictionaryUnit + var cur *dictionaryUnit + var body strings.Builder + lastLine := 0 + flush := func() { + if cur != nil { + cur.text = "CREATE OR REPLACE " + body.String() + units = append(units, *cur) + } + body.Reset() + } + for rows.Next() { + var owner, name, kind, text string + var line int + if err := rows.Scan(&owner, &name, &kind, &line, &text); err != nil { + t.Fatalf("scan DBA_SOURCE: %v", err) + } + key := owner + "." + name + ":" + kind + if cur == nil || cur.key != key { + flush() + cur = &dictionaryUnit{key: key, kind: kind} + lastLine = 0 + } + if line == lastLine { + continue + } + lastLine = line + body.WriteString(text) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate DBA_SOURCE: %v", err) + } + flush() + return units +} + +// parseDictionaryUnit splits and parses one unit. On failure it returns the +// category of the construct at the error, the error's line:column within the +// unit, and the message. +func parseDictionaryUnit(u dictionaryUnit) (category, where, message string, ok bool) { + var segments []Segment + for _, seg := range Split(u.text) { + if !seg.Empty() { + segments = append(segments, seg) + } + } + if len(segments) != 1 { + return classifyDictionaryFailure(u, ""), "1:1", fmt.Sprintf("split into %d segments", len(segments)), false + } + _, err := ParseRange(u.text, segments[0].ByteStart, segments[0].ByteEnd) + if err == nil { + return "", "", "", true + } + var pe *ParseError + if !errors.As(err, &pe) { + return "unclassified", "1:1", err.Error(), false + } + pos := pe.Position + if pos < 0 || pos > len(u.text) { + pos = 0 + } + lineStart := strings.LastIndexByte(u.text[:pos], '\n') + 1 + lineEnd := strings.IndexByte(u.text[pos:], '\n') + if lineEnd < 0 { + lineEnd = len(u.text) + } else { + lineEnd += pos + } + line := strings.Count(u.text[:pos], "\n") + 1 + where = fmt.Sprintf("%d:%d", line, len([]rune(u.text[lineStart:pos]))+1) + return classifyDictionaryFailure(u, u.text[lineStart:lineEnd]), where, pe.Message, false +} + +// dictionaryCategories name the construct behind a failure from the source +// line the parser stopped on. First match wins. +var dictionaryCategories = []struct { + name string + kinds string // DBA_SOURCE types the rule applies to; empty for all + pattern *regexp.Regexp +}{ + {"conditional compilation ($IF/$END)", "", regexp.MustCompile(`(?i)^\s*\$|\$(if|then|elsif|else|end|error)\b|\$\$`)}, + {"ACCESSIBLE BY clause", "", regexp.MustCompile(`(?i)accessible\s+by`)}, + {"CHARACTER SET ANY_CS / %CHARSET", "", regexp.MustCompile(`(?i)character\s+set`)}, + {"PIPELINED / AGGREGATE USING implementation type", "", regexp.MustCompile(`(?i)(pipelined|aggregate|parallel_enable)\s+using|^\s*using\b`)}, + {"PARALLEL_ENABLE (PARTITION ...)", "", regexp.MustCompile(`(?i)parallel_enable\s*\(`)}, + {"package initialization section", "PACKAGE BODY", regexp.MustCompile(`(?i)^\s*begin\b`)}, + {"REF type", "", regexp.MustCompile(`(?i)\bref\s`)}, + {"Oracle-internal syntax (PRAGMA INTERFACE, ... parameters)", "", regexp.MustCompile(`(?i)pragma\s+interface|\.\.\.`)}, + {"JSON function clause", "", regexp.MustCompile(`(?i)\breturning\b|json_|^\s*key\b`)}, + {"trigger CALL with arguments", "TRIGGER", regexp.MustCompile(`(?i)^\s*call\b|\(\s*:(new|old)\.`)}, + {"qualified expression (=>)", "", regexp.MustCompile(`=>`)}, + {"type length expression", "", regexp.MustCompile(`(?i)\(\s*[a-z_0-9]*\s*[*+-]`)}, + {"SQL%BULK_EXCEPTIONS", "", regexp.MustCompile(`(?i)sql%bulk`)}, + {"type body method modifiers", "TYPE BODY", regexp.MustCompile(`(?i)\b(final|instantiable|overriding)\b`)}, +} + +func classifyDictionaryFailure(u dictionaryUnit, line string) string { + if line == "" && strings.Contains(strings.ToUpper(u.text), "$END") { + return "conditional compilation ($IF/$END)" + } + for _, c := range dictionaryCategories { + if c.kinds != "" && c.kinds != u.kind { + continue + } + if c.pattern.MatchString(line) { + return c.name + } + } + return "unclassified" +} diff --git a/oracle/parser/select.go b/oracle/parser/select.go index a32ba095..3d8bccab 100644 --- a/oracle/parser/select.go +++ b/oracle/parser/select.go @@ -387,12 +387,13 @@ func (p *Parser) parseSelectList() (*nodes.List, error) { } // isSelectListTerminator reports whether the current token ends the select -// list where an item would start. OFFSET is absent: an item can be a column -// named OFFSET, and the row_limiting_clause never follows an empty list. +// list where an item would start. FETCH and OFFSET are absent: an item can be +// a column of that name, and the row_limiting_clause never follows an empty +// list. func (p *Parser) isSelectListTerminator() bool { switch p.cur.Type { case tokEOF, kwFROM, kwWHERE, kwGROUP, kwHAVING, kwORDER, kwUNION, - kwINTERSECT, kwMINUS, kwFOR, kwCONNECT, kwSTART, kwFETCH: + kwINTERSECT, kwMINUS, kwFOR, kwCONNECT, kwSTART: return true default: return false diff --git a/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv b/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv new file mode 100644 index 00000000..3df089c5 --- /dev/null +++ b/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv @@ -0,0 +1,2 @@ +case oracle_accepts omni_accepts sql +XMLELEMENT:index false true CREATE INDEX t_i ON t (XMLELEMENT) diff --git a/oracle/parser/testdata/coverage/oracle_keywords.tsv b/oracle/parser/testdata/coverage/oracle_keywords.tsv index 6c7033f1..c7713e03 100644 --- a/oracle/parser/testdata/coverage/oracle_keywords.tsv +++ b/oracle/parser/testdata/coverage/oracle_keywords.tsv @@ -44,7 +44,7 @@ kw_042 CONNECT_BY_ROOT 2036 nonreserved allow allow skip allow skip skip lexer k kw_043 CONSTANT 2037 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_044 CONSTRAINT 2038 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_045 CONSTRAINTS 2039 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_046 CONTENT 2309 context allow allow skip reject skip skip lexer keyword classified by local Oracle parser policy +kw_046 CONTENT 2309 context allow allow skip allow skip skip engine-checked: Oracle accepts it as a schema-qualified table name kw_047 CONTEXT 2325 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_048 CONTINUE 2040 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_049 CREATE 2041 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy @@ -95,7 +95,7 @@ kw_093 FOLLOWING 2290 context allow allow skip allow skip skip lexer keyword cla kw_094 FOR 2079 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy kw_095 FORALL 2080 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_096 FORCE 2081 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_097 FOREIGN 2082 nonreserved allow reject skip allow skip skip lexer keyword classified by local Oracle parser policy +kw_097 FOREIGN 2082 nonreserved allow allow skip allow skip skip engine-checked: not reserved in V$RESERVED_WORDS; Oracle accepts it as an identifier kw_098 FORMAT 2331 context allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_099 FROM 2083 clause-starter reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy kw_100 FULL 2084 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy @@ -131,7 +131,7 @@ kw_129 IS 2111 reserved reject reject skip reject skip skip lexer keyword classi kw_130 ISOLATION 2112 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_131 ITERATE 2280 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_132 JAVA 2327 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_133 JOIN 2113 clause-starter allow allow skip reject skip skip lexer keyword classified by local Oracle parser policy +kw_133 JOIN 2113 clause-starter allow allow skip allow skip skip engine-checked: Oracle accepts it as a schema-qualified table name kw_134 JSON 2332 context allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_135 JSON_ARRAY 2333 function allow allow skip allow allow skip lexer keyword classified by local Oracle parser policy kw_136 JSON_EXISTS 2334 function allow allow skip allow allow skip lexer keyword classified by local Oracle parser policy @@ -225,10 +225,10 @@ kw_223 PIVOT 2184 nonreserved allow allow skip allow skip skip lexer keyword cla kw_224 PLAN 2185 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_225 PRAGMA 2186 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_226 PRECEDING 2292 context allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_227 PRIMARY 2187 nonreserved allow reject skip allow skip skip lexer keyword classified by local Oracle parser policy +kw_227 PRIMARY 2187 nonreserved allow allow skip allow skip skip engine-checked: not reserved in V$RESERVED_WORDS; Oracle accepts it as an identifier kw_228 PRIOR 2188 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy kw_229 PRIVATE 2189 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_230 PRIVILEGES 2190 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy +kw_230 PRIVILEGES 2190 nonreserved allow allow skip allow skip skip engine-checked: not reserved in V$RESERVED_WORDS; Oracle accepts it as an identifier kw_231 PROCEDURE 2191 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_232 PROFILE 2329 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_233 PUBLIC 2192 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy diff --git a/oracle/parser/testdata/coverage/plsql_dictionary_known_failures.tsv b/oracle/parser/testdata/coverage/plsql_dictionary_known_failures.tsv new file mode 100644 index 00000000..abc0b824 --- /dev/null +++ b/oracle/parser/testdata/coverage/plsql_dictionary_known_failures.tsv @@ -0,0 +1,103 @@ +unit category position error +GSMADMIN_INTERNAL.DBMS_GSM_COMMON:PACKAGE ACCESSIBLE BY clause 567:26 syntax error at or near "dbms_gsm_dbadmin" +GSMADMIN_INTERNAL.DBMS_GSM_DBADMIN:PACKAGE ACCESSIBLE BY clause 71:24 syntax error at or near "sys" +GSMADMIN_INTERNAL.DBMS_GSM_UTILITY:PACKAGE ACCESSIBLE BY clause 938:1 syntax error at or near "ACCESSIBLE" +GSMADMIN_INTERNAL.DBMS_SHARDING_DIRECTORY:PACKAGE Oracle-internal syntax (PRAGMA INTERFACE, ... parameters) 42:34 syntax error at or near ".." +GSMADMIN_INTERNAL.CAT_ROLLBACK_TRIGGER:TRIGGER trigger CALL with arguments 3:40 syntax error at or near "(" +GSMADMIN_INTERNAL.DONE_TRIGGER:TRIGGER trigger CALL with arguments 3:40 syntax error at or near "(" +GSMADMIN_INTERNAL.REQUEST_DELETE_TRIGGER:TRIGGER trigger CALL with arguments 3:36 syntax error at or near "(" +SYS.AVTUNE_LOG_LISTAGGCLOB$:FUNCTION PIPELINED / AGGREGATE USING implementation type 4:43 syntax error at or near ";" +SYS.JSON_DATAGUIDE:FUNCTION PIPELINED / AGGREGATE USING implementation type 6:38 syntax error at or near ";" +SYS.JSON_HIERDATAGUIDE:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:39 syntax error at or near ";" +SYS.MVAGGRAWBITOR:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:34 syntax error at or near ";" +SYS.SCHEDULER$_BATCHERR_PIPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:50 syntax error at or near "USING" +SYS.STRAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:32 syntax error at or near ";" +SYS.SYS_DGAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 6:40 syntax error at or near ";" +SYS.SYS_HIERDGAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:32 syntax error at or near ";" +SYS.SYS_IXMLAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:26 syntax error at or near ";" +SYS.SYS_IXQAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:25 syntax error at or near ";" +SYS.SYS_IXQAGGAVG:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:28 syntax error at or near ";" +SYS.SYS_IXQAGGSUM:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:28 syntax error at or near ";" +SYS.SYS_NT_COLLECT:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:35 syntax error at or near ";" +SYS.V_LISTBACKUPPIPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:34 syntax error at or near "using" +SYS.V_LISTRSRANGEPIPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:37 syntax error at or near "using" +SYS.XMLSEQUENCEFROMREFCURSOR:FUNCTION PIPELINED / AGGREGATE USING implementation type 4:11 syntax error at or near "using" +SYS.XMLSEQUENCEFROMREFCURSOR2:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:11 syntax error at or near "using" +SYS.XMLSEQUENCEFROMXMLTYPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:34 syntax error at or near "using" +SYS.XQSEQUENCEFROMXMLTYPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:34 syntax error at or near "using" +SYS.XQWINDOWSEQUENCEFROMXMLTYPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 9:12 syntax error at or near "using" +SYS.CDBVIEW_INTERNAL:PACKAGE ACCESSIBLE BY clause 2:26 syntax error at or near "sys" +SYS.DBMS_ASSERT:PACKAGE CHARACTER SET ANY_CS / %CHARSET 21:30 syntax error at or near "CHARACTER" +SYS.DBMS_AW_EXP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 119:54 syntax error at or near "CHARACTER" +SYS.DBMS_CRYPTO:PACKAGE CHARACTER SET ANY_CS / %CHARSET 292:55 syntax error at or near "CHARACTER" +SYS.DBMS_DATA_MINING:PACKAGE PIPELINED / AGGREGATE USING implementation type 1267:13 syntax error at or near "USING" +SYS.DBMS_FREQUENT_ITEMSET:PACKAGE PIPELINED / AGGREGATE USING implementation type 13:49 syntax error at or near "using" +SYS.DBMS_GSM_FIXED:PACKAGE ACCESSIBLE BY clause 86:1 syntax error at or near "ACCESSIBLE" +SYS.DBMS_GSM_GSMUSER:PACKAGE ACCESSIBLE BY clause 13:1 syntax error at or near "ACCESSIBLE" +SYS.DBMS_HADOOP_INTERNAL:PACKAGE PIPELINED / AGGREGATE USING implementation type 408:30 syntax error at or near "USING" +SYS.DBMS_HASH:PACKAGE CHARACTER SET ANY_CS / %CHARSET 17:32 syntax error at or near "CHARACTER" +SYS.DBMS_HCS_LOG:PACKAGE conditional compilation ($IF/$END) 1:1 split into 23 segments +SYS.DBMS_LOB:PACKAGE conditional compilation ($IF/$END) 1:1 split into 219 segments +SYS.DBMS_LOBUTIL:PACKAGE CHARACTER SET ANY_CS / %CHARSET 10:39 syntax error at or near "CHARACTER" +SYS.DBMS_METADATA_INT:PACKAGE ACCESSIBLE BY clause 298:32 syntax error at or near "sys" +SYS.DBMS_METADATA_UTIL:PACKAGE ACCESSIBLE BY clause 818:20 syntax error at or near "ACCESSIBLE" +SYS.DBMS_MLE:PACKAGE conditional compilation ($IF/$END) 1:1 split into 96 segments +SYS.DBMS_PIPE:PACKAGE CHARACTER SET ANY_CS / %CHARSET 246:43 syntax error at or near "character" +SYS.DBMS_PLUGTS:PACKAGE ACCESSIBLE BY clause 121:28 syntax error at or near "sys" +SYS.DBMS_REPORT:PACKAGE Oracle-internal syntax (PRAGMA INTERFACE, ... parameters) 175:22 syntax error at or near ".." +SYS.DBMS_RULE_INTERNAL:PACKAGE ACCESSIBLE BY clause 46:3 syntax error at or near "ACCESSIBLE" +SYS.DBMS_SCHEDULER:PACKAGE CHARACTER SET ANY_CS / %CHARSET 1076:40 syntax error at or near "CHARACTER" +SYS.DBMS_SODA_UTIL:PACKAGE ACCESSIBLE BY clause 1:61 syntax error at or near "accessible" +SYS.DBMS_SQL:PACKAGE conditional compilation ($IF/$END) 1:1 split into 600 segments +SYS.DBMS_SQLTUNE_UTIL0:PACKAGE ACCESSIBLE BY clause 449:26 syntax error at or near "SYS" +SYS.DBMS_SQLTUNE_UTIL1:PACKAGE ACCESSIBLE BY clause 702:3 syntax error at or near "ACCESSIBLE" +SYS.DBMS_SQLTUNE_UTIL2:PACKAGE ACCESSIBLE BY clause 552:3 syntax error at or near "ACCESSIBLE" +SYS.DBMS_STREAMS:PACKAGE CHARACTER SET ANY_CS / %CHARSET 88:59 syntax error at or near "CHARACTER" +SYS.DBMS_TTS:PACKAGE ACCESSIBLE BY clause 289:6 syntax error at or near "ACCESSIBLE" +SYS.DBMS_UPG_JSON0:PACKAGE ACCESSIBLE BY clause 2:1 syntax error at or near "ACCESSIBLE" +SYS.DBMS_UPG_JSON_INT:PACKAGE ACCESSIBLE BY clause 2:1 syntax error at or near "ACCESSIBLE" +SYS.DBMS_UTILITY:PACKAGE conditional compilation ($IF/$END) 1:1 split into 99 segments +SYS.DBMS_VECTOR:PACKAGE PARALLEL_ENABLE (PARTITION ...) 243:55 syntax error at or near "(" +SYS.DBMS_VECTOR_DATABASE_ADMIN:PACKAGE ACCESSIBLE BY clause 2:18 syntax error at or near "accessible" +SYS.HTF:PACKAGE CHARACTER SET ANY_CS / %CHARSET 43:35 syntax error at or near "character" +SYS.HTP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 17:38 syntax error at or near "character" +SYS.OWA_UTIL:PACKAGE CHARACTER SET ANY_CS / %CHARSET 46:42 syntax error at or near "character" +SYS.PLITBLM:PACKAGE Oracle-internal syntax (PRAGMA INTERFACE, ... parameters) 20:29 syntax error at or near ")" +SYS.STANDARD:PACKAGE CHARACTER SET ANY_CS / %CHARSET 255:39 syntax error at or near "CHARACTER" +SYS.UTL_ENCODE:PACKAGE CHARACTER SET ANY_CS / %CHARSET 55:51 syntax error at or near "character" +SYS.UTL_HTTP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 1456:52 syntax error at or near "CHARACTER" +SYS.UTL_I18N:PACKAGE CHARACTER SET ANY_CS / %CHARSET 109:43 syntax error at or near "CHARACTER" +SYS.UTL_LMS:PACKAGE CHARACTER SET ANY_CS / %CHARSET 37:54 syntax error at or near "CHARACTER" +SYS.UTL_RAW:PACKAGE CHARACTER SET ANY_CS / %CHARSET 80:38 syntax error at or near "CHARACTER" +SYS.UTL_RECOMP:PACKAGE conditional compilation ($IF/$END) 1:1 split into 18 segments +SYS.UTL_SMTP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 452:46 syntax error at or near "CHARACTER" +SYS.UTL_TCP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 340:50 syntax error at or near "CHARACTER" +SYS.UTL_URL:PACKAGE CHARACTER SET ANY_CS / %CHARSET 147:53 syntax error at or near "CHARACTER" +SYS.UTL_XML:PACKAGE ACCESSIBLE BY clause 129:3 syntax error at or near "ACCESSIBLE" +SYS.CDBVIEW_INTERNAL:PACKAGE BODY qualified expression (=>) 12:57 syntax error at or near "=>" +SYS.DBMS_HADOOP:PACKAGE BODY type length expression 98:40 syntax error at or near "+" +SYS.DBMS_HADOOP_INTERNAL:PACKAGE BODY type length expression 62:37 syntax error at or near "+" +SYS.HTF:PACKAGE BODY CHARACTER SET ANY_CS / %CHARSET 40:32 syntax error at or near "CHARACTER" +SYS.HTP:PACKAGE BODY CHARACTER SET ANY_CS / %CHARSET 68:39 syntax error at or near "character" +SYS.OWA_COOKIE:PACKAGE BODY package initialization section 221:1 syntax error at or near "begin" +SYS.OWA_CUSTOM:PACKAGE BODY package initialization section 15:1 syntax error at or near "begin" +SYS.OWA_IMAGE:PACKAGE BODY package initialization section 14:1 syntax error at or near "begin" +SYS.OWA_UTIL:PACKAGE BODY CHARACTER SET ANY_CS / %CHARSET 24:26 syntax error at or near "character" +SYS.STANDARD:PACKAGE BODY CHARACTER SET ANY_CS / %CHARSET 16:31 syntax error at or near "CHARACTER" +SYS.DBMS_FEATURE_ADBDCAT:PROCEDURE JSON function clause 181:49 syntax error at or near ")" +SYS.DBMS_FEATURE_ADBXTTAB:PROCEDURE JSON function clause 320:49 syntax error at or near ")" +SYS.DBMS_FEATURE_JOB_CLOUDSQL:PROCEDURE JSON function clause 54:13 syntax error at or near "'type'" +SYS.DBMS_FEATURE_MLE:PROCEDURE JSON function clause 29:49 syntax error at or near ")" +SYS.DBMS_FEATURE_SQL_FIREWALL:PROCEDURE JSON function clause 56:19 syntax error at or near ")" +SYS.OBJNUM_REUSE_HOLES:PROCEDURE SQL%BULK_EXCEPTIONS 107:30 syntax error at or near "%" +SYS.DBMS_KAFKA_DROP_USER:TRIGGER conditional compilation ($IF/$END) 57:7 syntax error at or near "$" +SYS.JDOM_T:TYPE BODY CHARACTER SET ANY_CS / %CHARSET 9:41 syntax error at or near "character" +SYS.JSON_ARRAY_T:TYPE BODY CHARACTER SET ANY_CS / %CHARSET 111:39 syntax error at or near "character" +SYS.JSON_ELEMENT_T:TYPE BODY CHARACTER SET ANY_CS / %CHARSET 2:38 syntax error at or near "character" +SYS.JSON_OBJECT_T:TYPE BODY CHARACTER SET ANY_CS / %CHARSET 110:39 syntax error at or near "character" +SYS.WRR$_ASH_TIME_PERIOD:TYPE BODY type body method modifiers 3:3 syntax error at or near "FINAL" +XDB.XMLINDEXINSFUNC:FUNCTION PIPELINED / AGGREGATE USING implementation type 4:11 syntax error at or near "using" +XDB.XMLINDEXLOADFUNC:FUNCTION PARALLEL_ENABLE (PARTITION ...) 5:17 syntax error at or near "(" +XDB.DBMS_XDB:PACKAGE REF type 234:37 syntax error at or near "SYS" +XDB.DBMS_XDBRESOURCE:PACKAGE REF type 488:56 syntax error at or near "SYS" +XDB.DBMS_XDB_REPOS:PACKAGE REF type 223:37 syntax error at or near "SYS" diff --git a/oracle/parser/testdata/coverage/strictness_v2.tsv b/oracle/parser/testdata/coverage/strictness_v2.tsv index a7800dfb..c3a7f626 100644 --- a/oracle/parser/testdata/coverage/strictness_v2.tsv +++ b/oracle/parser/testdata/coverage/strictness_v2.tsv @@ -36,7 +36,7 @@ expr_010 expression error syntax error SELECT PRIOR expr_011 expression error syntax error SELECT CONNECT_BY_ROOT expr_012 expression error syntax error SELECT (1 expr_013 expression error syntax error SELECT (1 + -expr_014 expression error syntax error SELECT CASE +expr_014 expression error syntax error SELECT CASE 1 expr_015 expression error syntax error SELECT CASE WHEN expr_016 expression error syntax error SELECT CASE WHEN 1 = 1 THEN expr_017 expression error syntax error SELECT CASE WHEN 1 = 1 THEN 2 ELSE From 9c61863b7e320abc54a80c4fc00e30f346e1c6f6 Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 03:16:39 +0000 Subject: [PATCH 02/11] fix(oracle): keep Boolean CASE selectors and outer-joined keyword columns (BYT-10301) Codex review on #446: - `CASE NOT TRUE WHEN TRUE THEN 1 ELSE 0 END` regressed: CASE followed by NOT was taken for a column. Oracle 23ai accepts a Boolean selector there, and also accepts a column named CASE in NOT IN, NOT BETWEEN, and NOT LIKE, so caseNamesColumn now looks past NOT (peekAhead lexes a copy of the lexer). - `WHERE cast(+) = u.a` failed because CAST before '(' always opened CAST(...). A non-reserved keyword followed by (+) is now a column for every word Oracle accepts that way. Asked word by word, Oracle rejects word(+) for JSON, JSON_ARRAY, JSON_EXISTS, JSON_MERGEPATCH, JSON_OBJECT, JSON_QUERY, JSON_TABLE, JSON_VALUE, TREAT, XMLELEMENT, XMLFOREST, and XMLROOT; those now fail at the '+' (JSON and JSON_TABLE were accepted before). The keyword column audit gains the outer-join marker and NOT IN contexts (232 words, 2,088 checks). NOT IN exposed one pre-existing gap, recorded in oracle_keyword_column_gaps.tsv: Oracle never lets CONNECT_BY_ROOT name a referenced column, while omni accepts `CONNECT_BY_ROOT NOT IN (1, 2)` because a reserved word in operand position (IN) parses as a function name. Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 45 ++++++++++++++++++- oracle/parser/keyword_column_oracle_test.go | 7 ++- oracle/parser/parser.go | 16 +++++++ oracle/parser/select_test.go | 43 ++++++++++++++++++ .../coverage/oracle_keyword_column_gaps.tsv | 1 + .../testdata/coverage/reference_oracle.tsv | 1 + 6 files changed, 110 insertions(+), 3 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index 6385d87a..855b6ddc 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -532,6 +532,19 @@ func (p *Parser) parsePrefix() (nodes.ExprNode, error) { func (p *Parser) parsePrimary() (nodes.ExprNode, error) { start := p.pos() + // A keyword Oracle does not reserve, followed by the legacy outer-join + // marker (+), is a column, whatever construct the word would otherwise + // open: WHERE cast(+) = u.a. A few words Oracle still reads as their + // construct, and it rejects the + inside it. + if p.cur.Type >= 2000 && !isOracleSQLReservedKeyword(p.cur) && p.nextIsOuterJoinMarker() { + if keepsConstructBeforeOuterJoin(p.cur.Type) { + p.advance() // consume the keyword + p.advance() // consume ( + return nil, p.syntaxErrorAtCur() + } + return p.parseIdentExpr() + } + switch p.cur.Type { case tokICONST: tok := p.advance() @@ -676,18 +689,48 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { // reference rather than a CASE expression. Oracle does not reserve CASE, and // a CASE expression always continues with WHEN or an operand, never with a // token that ends an operand or joins two of them. +// +// NOT needs one more token: a simple CASE may test a Boolean selector +// (CASE NOT flag WHEN TRUE ..., Oracle 23ai), while a column named CASE is +// followed by NOT only in NOT IN, NOT BETWEEN, and NOT LIKE. func (p *Parser) caseNamesColumn() bool { switch p.peekNext().Type { case tokEOF, ';', ',', ')', '.', '=', '<', '>', '*', '/', tokLESSEQ, tokGREATEQ, tokNOTEQ, tokCONCAT, kwFROM, kwWHERE, kwGROUP, kwORDER, kwHAVING, kwINTO, kwAS, - kwIS, kwIN, kwNOT, kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4, kwBETWEEN, + kwIS, kwIN, kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4, kwBETWEEN, kwAND, kwOR, kwASC, kwDESC, kwNULLS, kwUNION, kwINTERSECT, kwMINUS: return true + case kwNOT: + switch p.peekAhead(2)[1].Type { + case kwIN, kwBETWEEN, kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4: + return true + } } return false } +// keepsConstructBeforeOuterJoin reports the non-reserved keywords Oracle +// parses as their own construct even when (+) follows, rejecting word(+) +// (ORA-00931, ORA-00936), so they cannot name an outer-joined column +// unquoted. Engine-checked by TestOracleNonReservedKeywordsAsColumns. +func keepsConstructBeforeOuterJoin(tokenType int) bool { + switch tokenType { + case kwJSON, kwJSON_ARRAY, kwJSON_EXISTS, kwJSON_MERGEPATCH, kwJSON_OBJECT, + kwJSON_QUERY, kwJSON_TABLE, kwJSON_VALUE, kwTREAT, + kwXMLELEMENT, kwXMLFOREST, kwXMLROOT: + return true + } + return false +} + +// nextIsOuterJoinMarker reports whether the tokens after the current one are +// the legacy outer-join marker (+). +func (p *Parser) nextIsOuterJoinMarker() bool { + toks := p.peekAhead(3) + return toks[0].Type == '(' && toks[1].Type == '+' && toks[2].Type == ')' +} + // parseDateTimeLiteral parses ANSI datetime literals such as DATE '2020-01-01'. func (p *Parser) parseDateTimeLiteral() (nodes.ExprNode, error) { start := p.pos() diff --git a/oracle/parser/keyword_column_oracle_test.go b/oracle/parser/keyword_column_oracle_test.go index 69905c03..d82e102d 100644 --- a/oracle/parser/keyword_column_oracle_test.go +++ b/oracle/parser/keyword_column_oracle_test.go @@ -20,16 +20,19 @@ var keywordColumnContexts = []struct { }{ {"select list", "SELECT %[2]s FROM %[1]s"}, {"where", "SELECT a FROM %[1]s WHERE %[2]s IS NULL"}, + {"not in", "SELECT a FROM %[1]s WHERE %[2]s NOT IN (1, 2)"}, {"order by", "SELECT a FROM %[1]s ORDER BY %[2]s"}, {"update", "UPDATE %[1]s SET %[2]s = NULL WHERE %[2]s IS NULL"}, {"insert columns", "INSERT INTO %[1]s (%[2]s) VALUES (1)"}, {"index", "CREATE INDEX %[1]s_i ON %[1]s (%[2]s)"}, + {"outer join marker", "SELECT d.dummy FROM dual d, %[1]s WHERE %[2]s(+) = LENGTH(d.dummy)"}, } // TestOracleNonReservedKeywordsAsColumns checks every word omni lexes as a // keyword that Oracle does not reserve (V$RESERVED_WORDS) as a column name: -// defined by CREATE TABLE, then referenced from a select list, a WHERE -// predicate, ORDER BY, UPDATE, an INSERT column list, and an index. omni must +// defined by CREATE TABLE, then referenced from a select list, WHERE +// predicates (IS NULL, NOT IN), ORDER BY, UPDATE, an INSERT column list, an +// index, and with the legacy outer-join marker (+). omni must // accept or reject each statement exactly as Oracle does. The keyword // manifest's column_name context covers only the definition; OFFSET, FETCH, // JOIN, MODEL, and USING failed in the references alone. diff --git a/oracle/parser/parser.go b/oracle/parser/parser.go index cfa4971c..f33595e7 100644 --- a/oracle/parser/parser.go +++ b/oracle/parser/parser.go @@ -340,6 +340,22 @@ func (p *Parser) peekNext() Token { return p.nextBuf } +// peekAhead returns the n tokens after cur without consuming them; tokens +// past the end of input are tokEOF. It lexes a copy of the lexer, so the +// parser's position is untouched. +func (p *Parser) peekAhead(n int) []Token { + toks := make([]Token, 0, n) + if n == 0 { + return toks + } + toks = append(toks, p.peekNext()) + lexer := *p.lexer + for len(toks) < n { + toks = append(toks, lexer.NextToken()) + } + return toks +} + // peek returns the current token without consuming it. func (p *Parser) peek() Token { return p.cur diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index a8acf914..6ad94cff 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -412,3 +412,46 @@ func TestParseOffsetColumnNameAST(t *testing.T) { t.Fatalf("row limiting clause missing: %s", ast.NodeToString(sel)) } } + +// TestParseCaseBooleanSelector: a simple CASE may test a Boolean selector +// that starts with NOT (Oracle 23ai), while a column named CASE is followed +// by NOT only in NOT IN, NOT BETWEEN, and NOT LIKE. Oracle 23ai accepts all. +func TestParseCaseBooleanSelector(t *testing.T) { + result := ParseAndCheck(t, "SELECT CASE NOT TRUE WHEN TRUE THEN 1 ELSE 0 END FROM t") + target := result.Items[0].(*ast.RawStmt).Stmt.(*ast.SelectStmt).TargetList.Items[0].(*ast.ResTarget) + if _, ok := target.Expr.(*ast.CaseExpr); !ok { + t.Fatalf("target = %T, want *ast.CaseExpr", target.Expr) + } + for _, sql := range []string{ + "SELECT a FROM t WHERE case NOT IN (1, 2)", + "SELECT a FROM t WHERE case NOT BETWEEN 1 AND 2", + "SELECT a FROM t WHERE case NOT LIKE 'x%'", + } { + t.Run(sql, func(t *testing.T) { + ParseAndCheck(t, sql) + }) + } +} + +// TestParseKeywordColumnOuterJoin: a non-reserved keyword followed by the +// legacy outer-join marker (+) is a column, except for the words Oracle still +// reads as their construct (it rejects JSON_VALUE(+) and XMLELEMENT(+)). +func TestParseKeywordColumnOuterJoin(t *testing.T) { + for _, word := range []string{"cast", "decode", "case", "interval", "xmlagg"} { + sql := "SELECT 1 FROM t, u WHERE " + word + "(+) = u.a" + t.Run(sql, func(t *testing.T) { + result := ParseAndCheck(t, sql) + where := result.Items[0].(*ast.RawStmt).Stmt.(*ast.SelectStmt).WhereClause.(*ast.BinaryExpr) + col, ok := where.Left.(*ast.ColumnRef) + if !ok || !col.OuterJoin { + t.Fatalf("left = %s, want an outer-joined column", ast.NodeToString(where.Left)) + } + if violations := CheckLocations(t, sql); len(violations) > 0 { + t.Fatalf("Loc violations: %v", violations) + } + }) + } + for _, word := range []string{"json_value", "xmlelement", "treat", "json"} { + ParseShouldFail(t, "SELECT 1 FROM t, u WHERE "+word+"(+) = u.a") + } +} diff --git a/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv b/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv index 3df089c5..3489b3aa 100644 --- a/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv +++ b/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv @@ -1,2 +1,3 @@ case oracle_accepts omni_accepts sql +CONNECT_BY_ROOT:not in false true SELECT a FROM t WHERE CONNECT_BY_ROOT NOT IN (1, 2) XMLELEMENT:index false true CREATE INDEX t_i ON t (XMLELEMENT) diff --git a/oracle/parser/testdata/coverage/reference_oracle.tsv b/oracle/parser/testdata/coverage/reference_oracle.tsv index e2e856f9..96039cea 100644 --- a/oracle/parser/testdata/coverage/reference_oracle.tsv +++ b/oracle/parser/testdata/coverage/reference_oracle.tsv @@ -93,3 +93,4 @@ ref_091 plsql_body accept CREATE OR REPLACE FUNCTION omni_cs_names_{run} (langua ref_092 plsql_body reject CREATE OR REPLACE TYPE omni_bad_novr_{run} AS OBJECT (a NUMBER, NOT OVERRIDING MEMBER FUNCTION f RETURN NUMBER) NOT FINAL NOT OVERRIDING in a method spec ref_093 plsql_body reject CREATE OR REPLACE TYPE omni_bad_fovr_{run} AS OBJECT (a NUMBER, FINAL OVERRIDING MEMBER FUNCTION f RETURN NUMBER) OVERRIDING after another inheritance clause ref_094 plsql_body accept CREATE OR REPLACE TYPE omni_ninf_{run} AS OBJECT (a NUMBER, NOT INSTANTIABLE NOT FINAL MEMBER FUNCTION f RETURN NUMBER) NOT FINAL NOT INSTANTIABLE NOT INSTANTIABLE NOT FINAL method spec +ref_095 select accept SELECT CASE NOT TRUE WHEN TRUE THEN 1 ELSE 0 END FROM dual simple CASE with a Boolean NOT selector From c04206251a42c61cd798a10203db78b3256c2edc Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 03:29:49 +0000 Subject: [PATCH 03/11] fix(oracle): keep pseudo-columns out of the outer-join column shortcut (BYT-10301) Codex review on #446: SYSTIMESTAMP(+) became an outer-joined column because SYSTIMESTAMP is a non-reserved keyword. Oracle 23ai reads it as SYSTIMESTAMP(precision) and rejects it with ORA-30088, even when the table has a column of that name; only a qualified t.systimestamp(+) is a column. The shortcut now skips pseudo-column keywords (SYSTIMESTAMP is the only non-reserved one), as main did. Pinned by TestParseKeywordColumnOuterJoin and reference row ref_096. Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 7 +++++-- oracle/parser/select_test.go | 7 +++++-- oracle/parser/testdata/coverage/reference_oracle.tsv | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index 855b6ddc..482b4c5c 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -535,8 +535,11 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { // A keyword Oracle does not reserve, followed by the legacy outer-join // marker (+), is a column, whatever construct the word would otherwise // open: WHERE cast(+) = u.a. A few words Oracle still reads as their - // construct, and it rejects the + inside it. - if p.cur.Type >= 2000 && !isOracleSQLReservedKeyword(p.cur) && p.nextIsOuterJoinMarker() { + // construct, and it rejects the + inside it. Pseudo-columns stay + // pseudo-columns: Oracle reads SYSTIMESTAMP(+) as SYSTIMESTAMP(precision) + // and rejects it (ORA-30088) even when the table has such a column. + if p.cur.Type >= 2000 && !isOracleSQLReservedKeyword(p.cur) && + !isOraclePseudoColumnKeyword(p.cur.Type) && p.nextIsOuterJoinMarker() { if keepsConstructBeforeOuterJoin(p.cur.Type) { p.advance() // consume the keyword p.advance() // consume ( diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index 6ad94cff..3ae72d59 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -435,7 +435,8 @@ func TestParseCaseBooleanSelector(t *testing.T) { // TestParseKeywordColumnOuterJoin: a non-reserved keyword followed by the // legacy outer-join marker (+) is a column, except for the words Oracle still -// reads as their construct (it rejects JSON_VALUE(+) and XMLELEMENT(+)). +// reads as their construct (it rejects JSON_VALUE(+) and XMLELEMENT(+)) and +// for pseudo-columns (SYSTIMESTAMP(+) is ORA-30088 even with such a column). func TestParseKeywordColumnOuterJoin(t *testing.T) { for _, word := range []string{"cast", "decode", "case", "interval", "xmlagg"} { sql := "SELECT 1 FROM t, u WHERE " + word + "(+) = u.a" @@ -451,7 +452,9 @@ func TestParseKeywordColumnOuterJoin(t *testing.T) { } }) } - for _, word := range []string{"json_value", "xmlelement", "treat", "json"} { + for _, word := range []string{"json_value", "xmlelement", "treat", "json", "systimestamp"} { ParseShouldFail(t, "SELECT 1 FROM t, u WHERE "+word+"(+) = u.a") } + // A qualified reference is a column even for a pseudo-column word. + ParseAndCheck(t, "SELECT 1 FROM t, u WHERE t.systimestamp(+) = u.a") } diff --git a/oracle/parser/testdata/coverage/reference_oracle.tsv b/oracle/parser/testdata/coverage/reference_oracle.tsv index 96039cea..690c2745 100644 --- a/oracle/parser/testdata/coverage/reference_oracle.tsv +++ b/oracle/parser/testdata/coverage/reference_oracle.tsv @@ -94,3 +94,4 @@ ref_092 plsql_body reject CREATE OR REPLACE TYPE omni_bad_novr_{run} AS OBJECT ( ref_093 plsql_body reject CREATE OR REPLACE TYPE omni_bad_fovr_{run} AS OBJECT (a NUMBER, FINAL OVERRIDING MEMBER FUNCTION f RETURN NUMBER) OVERRIDING after another inheritance clause ref_094 plsql_body accept CREATE OR REPLACE TYPE omni_ninf_{run} AS OBJECT (a NUMBER, NOT INSTANTIABLE NOT FINAL MEMBER FUNCTION f RETURN NUMBER) NOT FINAL NOT INSTANTIABLE NOT INSTANTIABLE NOT FINAL method spec ref_095 select accept SELECT CASE NOT TRUE WHEN TRUE THEN 1 ELSE 0 END FROM dual simple CASE with a Boolean NOT selector +ref_096 select reject SELECT 1 FROM t, s WHERE systimestamp(+) = s.x pseudo-column with the outer-join marker (ORA-30088) From 9906b13ffb6ffae55010896a5942f88c90d0961b Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 03:44:00 +0000 Subject: [PATCH 04/11] fix(oracle): look past NULLS and LIKEC after CASE, check dictionary failure signatures (BYT-10301) Codex review on #446: - `CASE nulls WHEN 1 THEN 2 END` regressed: NULLS after CASE was taken for a column named CASE ordered NULLS FIRST/LAST. Oracle 23ai accepts both readings, and LIKEC, LIKE2, and LIKE4 (also not reserved) are ambiguous the same way. caseNamesColumn now reads one more token for each: NULLS orders the column only before FIRST or LAST; LIKEC/LIKE2/ LIKE4 open a selector before WHEN or an operator and compare the column otherwise. The keyword column audit gains a `case selector` context (every non-reserved word as CASE word WHEN ...; 232 words, 2,320 checks), and reference rows ref_097..ref_101 pin the column side. - TestOraclePLSQLDictionary compared only which units fail. It now also compares each known unit's recorded line:column and message, so a regression that makes it fail earlier is reported (and a later failure asks for the manifest to be re-recorded). Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 24 ++++++++-- oracle/parser/keyword_column_oracle_test.go | 4 +- oracle/parser/oracle_helper_test.go | 2 + oracle/parser/plsql_dictionary_oracle_test.go | 46 +++++++++++++++---- oracle/parser/select_test.go | 24 ++++++++-- .../testdata/coverage/reference_oracle.tsv | 5 ++ 6 files changed, 88 insertions(+), 17 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index 482b4c5c..4e9b5920 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -693,22 +693,36 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { // a CASE expression always continues with WHEN or an operand, never with a // token that ends an operand or joins two of them. // -// NOT needs one more token: a simple CASE may test a Boolean selector -// (CASE NOT flag WHEN TRUE ..., Oracle 23ai), while a column named CASE is -// followed by NOT only in NOT IN, NOT BETWEEN, and NOT LIKE. +// The words below that Oracle does not reserve need one more token, because +// they can also open a simple CASE selector (Oracle 23ai accepts each form): +// NOT starts a Boolean selector unless NOT IN / NOT BETWEEN / NOT LIKE +// follows; NULLS is a column named NULLS unless NULLS FIRST / NULLS LAST +// orders the column CASE; LIKEC, LIKE2, and LIKE4 are a selector when WHEN +// or an operator follows, and compare the column CASE otherwise. func (p *Parser) caseNamesColumn() bool { switch p.peekNext().Type { case tokEOF, ';', ',', ')', '.', '=', '<', '>', '*', '/', tokLESSEQ, tokGREATEQ, tokNOTEQ, tokCONCAT, kwFROM, kwWHERE, kwGROUP, kwORDER, kwHAVING, kwINTO, kwAS, - kwIS, kwIN, kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4, kwBETWEEN, - kwAND, kwOR, kwASC, kwDESC, kwNULLS, kwUNION, kwINTERSECT, kwMINUS: + kwIS, kwIN, kwLIKE, kwBETWEEN, + kwAND, kwOR, kwASC, kwDESC, kwUNION, kwINTERSECT, kwMINUS: return true case kwNOT: switch p.peekAhead(2)[1].Type { case kwIN, kwBETWEEN, kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4: return true } + case kwNULLS: + switch p.peekAhead(2)[1].Type { + case kwFIRST, kwLAST: + return true + } + case kwLIKEC, kwLIKE2, kwLIKE4: + switch p.peekAhead(2)[1].Type { + case kwWHEN, '+', '-', '*', '/', '.', tokCONCAT, tokEXPON: + return false + } + return true } return false } diff --git a/oracle/parser/keyword_column_oracle_test.go b/oracle/parser/keyword_column_oracle_test.go index d82e102d..5fd82723 100644 --- a/oracle/parser/keyword_column_oracle_test.go +++ b/oracle/parser/keyword_column_oracle_test.go @@ -26,13 +26,15 @@ var keywordColumnContexts = []struct { {"insert columns", "INSERT INTO %[1]s (%[2]s) VALUES (1)"}, {"index", "CREATE INDEX %[1]s_i ON %[1]s (%[2]s)"}, {"outer join marker", "SELECT d.dummy FROM dual d, %[1]s WHERE %[2]s(+) = LENGTH(d.dummy)"}, + {"case selector", "SELECT CASE %[2]s WHEN 1 THEN 2 END FROM %[1]s"}, } // TestOracleNonReservedKeywordsAsColumns checks every word omni lexes as a // keyword that Oracle does not reserve (V$RESERVED_WORDS) as a column name: // defined by CREATE TABLE, then referenced from a select list, WHERE // predicates (IS NULL, NOT IN), ORDER BY, UPDATE, an INSERT column list, an -// index, and with the legacy outer-join marker (+). omni must +// index, with the legacy outer-join marker (+), and as a simple CASE +// selector. omni must // accept or reject each statement exactly as Oracle does. The keyword // manifest's column_name context covers only the definition; OFFSET, FETCH, // JOIN, MODEL, and USING failed in the references alone. diff --git a/oracle/parser/oracle_helper_test.go b/oracle/parser/oracle_helper_test.go index f00f9695..53a7f512 100644 --- a/oracle/parser/oracle_helper_test.go +++ b/oracle/parser/oracle_helper_test.go @@ -224,6 +224,8 @@ func setupOracleSchema(o *oracleDB) { `CREATE TABLE j_purchaseorder (po_document CLOB)`, // OFFSET and PARTITION are not reserved and name columns in practice. `CREATE TABLE t_offset (a NUMBER, offset NUMBER, partition NUMBER)`, + // CASE, NULLS, and LIKEC are not reserved either. + `CREATE TABLE t_kwcol (a NUMBER, case NUMBER, nulls NUMBER, likec VARCHAR2(10))`, } for _, ddl := range ddls { diff --git a/oracle/parser/plsql_dictionary_oracle_test.go b/oracle/parser/plsql_dictionary_oracle_test.go index 9b75d9e8..2f15756a 100644 --- a/oracle/parser/plsql_dictionary_oracle_test.go +++ b/oracle/parser/plsql_dictionary_oracle_test.go @@ -33,10 +33,11 @@ type dictionaryUnit struct { // of code the engine compiled. Each unit goes through Split and ParseRange, // as Bytebase runs a script, and must come out as one segment that parses. // -// The units that fail today are recorded with a category in -// plsql_dictionary_known_failures.tsv. A unit that starts failing is a -// regression; a recorded unit that now parses must leave the manifest, so the -// list only shrinks. Regenerate it with -update and review the diff. The +// The units that fail today are recorded with a category, the line:column of +// the first error, and its message in plsql_dictionary_known_failures.tsv. A +// unit that starts failing is a regression, and so is a recorded unit that +// now fails earlier; one that fails later or parses must be re-recorded or +// leave the manifest, so the list only shrinks. Regenerate it with -update and review the diff. The // source text itself is Oracle's and is read at test time, never committed. func TestOraclePLSQLDictionary(t *testing.T) { units := loadOracleDictionaryUnits(t) @@ -60,7 +61,7 @@ func TestOraclePLSQLDictionary(t *testing.T) { } var failures []failure byCategory := map[string]int{} - var newFailures, fixed int + var newFailures, changed, fixed int for _, u := range units { category, where, message, ok := parseDictionaryUnit(u) if ok { @@ -72,15 +73,32 @@ func TestOraclePLSQLDictionary(t *testing.T) { } failures = append(failures, failure{unit: u, category: category, where: where, message: message}) byCategory[category]++ - if _, wasKnown := known[u.key]; !wasKnown && !*update { + if *update { + continue + } + row, wasKnown := known[u.key] + if !wasKnown { newFailures++ t.Errorf("NEW FAILURE %s at %s: %s (%s)", u.key, where, message, category) + continue + } + // A known unit must still fail at the recorded place for the + // recorded reason: failing earlier is a regression hidden inside + // it, failing later is progress the manifest has to record. + if row.Fields["position"] != where || row.Fields["error"] != message { + changed++ + direction := "later (progress)" + if lineColBefore(where, row.Fields["position"]) { + direction = "EARLIER (regression)" + } + t.Errorf("CHANGED %s now fails %s: at %s: %s (%s), recorded at %s: %s. Rerun with -update if intended.", + u.key, direction, where, message, category, row.Fields["position"], row.Fields["error"]) } } - t.Logf("Oracle %s dictionary: %d units, %d fail (%.1f%%), %d new, %d fixed", + t.Logf("Oracle %s dictionary: %d units, %d fail (%.1f%%), %d new, %d changed, %d fixed", oracleDictionaryVersion, len(units), len(failures), - 100*float64(len(failures))/float64(len(units)), newFailures, fixed) + 100*float64(len(failures))/float64(len(units)), newFailures, changed, fixed) var categories []string for c := range byCategory { categories = append(categories, c) @@ -253,3 +271,15 @@ func classifyDictionaryFailure(u dictionaryUnit, line string) string { } return "unclassified" } + +// lineColBefore reports whether position a ("line:col") comes before b. +func lineColBefore(a, b string) bool { + var al, ac, bl, bc int + if _, err := fmt.Sscanf(a, "%d:%d", &al, &ac); err != nil { + return false + } + if _, err := fmt.Sscanf(b, "%d:%d", &bl, &bc); err != nil { + return false + } + return al < bl || (al == bl && ac < bc) +} diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index 3ae72d59..709ce835 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -413,19 +413,37 @@ func TestParseOffsetColumnNameAST(t *testing.T) { } } -// TestParseCaseBooleanSelector: a simple CASE may test a Boolean selector -// that starts with NOT (Oracle 23ai), while a column named CASE is followed -// by NOT only in NOT IN, NOT BETWEEN, and NOT LIKE. Oracle 23ai accepts all. +// TestParseCaseBooleanSelector: a simple CASE selector may start with NOT +// (a Boolean selector), NULLS, or LIKEC, while a column named CASE is +// followed by NOT only in NOT IN / NOT BETWEEN / NOT LIKE and by NULLS only +// in NULLS FIRST / NULLS LAST. Oracle 23ai accepts all of these. func TestParseCaseBooleanSelector(t *testing.T) { result := ParseAndCheck(t, "SELECT CASE NOT TRUE WHEN TRUE THEN 1 ELSE 0 END FROM t") target := result.Items[0].(*ast.RawStmt).Stmt.(*ast.SelectStmt).TargetList.Items[0].(*ast.ResTarget) if _, ok := target.Expr.(*ast.CaseExpr); !ok { t.Fatalf("target = %T, want *ast.CaseExpr", target.Expr) } + for _, sql := range []string{ + "SELECT CASE nulls WHEN 1 THEN 2 END FROM t", + "SELECT CASE nulls + 1 WHEN 1 THEN 2 END FROM t", + "SELECT CASE likec WHEN 'x' THEN 2 END FROM t", + "SELECT CASE like2 || 'a' WHEN 'xa' THEN 2 END FROM t", + } { + t.Run(sql, func(t *testing.T) { + result := ParseAndCheck(t, sql) + target := result.Items[0].(*ast.RawStmt).Stmt.(*ast.SelectStmt).TargetList.Items[0].(*ast.ResTarget) + if _, ok := target.Expr.(*ast.CaseExpr); !ok { + t.Fatalf("target = %T, want *ast.CaseExpr", target.Expr) + } + }) + } for _, sql := range []string{ "SELECT a FROM t WHERE case NOT IN (1, 2)", "SELECT a FROM t WHERE case NOT BETWEEN 1 AND 2", "SELECT a FROM t WHERE case NOT LIKE 'x%'", + "SELECT a FROM t ORDER BY case NULLS FIRST", + "SELECT a FROM t ORDER BY case DESC NULLS LAST", + "SELECT a FROM t WHERE case LIKEC '1%'", } { t.Run(sql, func(t *testing.T) { ParseAndCheck(t, sql) diff --git a/oracle/parser/testdata/coverage/reference_oracle.tsv b/oracle/parser/testdata/coverage/reference_oracle.tsv index 690c2745..be3052a8 100644 --- a/oracle/parser/testdata/coverage/reference_oracle.tsv +++ b/oracle/parser/testdata/coverage/reference_oracle.tsv @@ -95,3 +95,8 @@ ref_093 plsql_body reject CREATE OR REPLACE TYPE omni_bad_fovr_{run} AS OBJECT ( ref_094 plsql_body accept CREATE OR REPLACE TYPE omni_ninf_{run} AS OBJECT (a NUMBER, NOT INSTANTIABLE NOT FINAL MEMBER FUNCTION f RETURN NUMBER) NOT FINAL NOT INSTANTIABLE NOT INSTANTIABLE NOT FINAL method spec ref_095 select accept SELECT CASE NOT TRUE WHEN TRUE THEN 1 ELSE 0 END FROM dual simple CASE with a Boolean NOT selector ref_096 select reject SELECT 1 FROM t, s WHERE systimestamp(+) = s.x pseudo-column with the outer-join marker (ORA-30088) +ref_097 select accept SELECT a FROM t_kwcol ORDER BY case NULLS FIRST column named CASE ordered NULLS FIRST +ref_098 select accept SELECT a FROM t_kwcol ORDER BY case DESC NULLS LAST column named CASE ordered DESC NULLS LAST +ref_099 select accept SELECT a FROM t_kwcol WHERE case LIKEC '1%' column named CASE compared with LIKEC +ref_100 select accept SELECT CASE nulls WHEN 1 THEN 2 END FROM t_kwcol column named NULLS as a simple CASE selector +ref_101 select accept SELECT CASE likec WHEN 'x' THEN 2 END FROM t_kwcol column named LIKEC as a simple CASE selector From d7e3b052cbe1a24822e21c56b035635d9ca9cb98 Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 03:54:27 +0000 Subject: [PATCH 05/11] fix(oracle): decide CASE expression or column by probing for WHEN (BYT-10301) Codex review on #446: `CASE likec = 'x' WHEN TRUE THEN 1 ELSE 0 END` regressed. A simple CASE selector may be any expression, Boolean comparisons included (Oracle 23ai), so judging from the one or two tokens after CASE kept missing forms (NOT, NULLS, LIKEC, then comparisons). caseNamesColumn now applies the rule as Oracle reads it: CASE opens an expression when WHEN follows it, or when an expression follows and is itself followed by WHEN; otherwise it is a column. The selector is parsed on a probe whose parser state (saveState / restoreState) is discarded, so a real CASE expression is parsed again and its errors keep their positions (TestParseCaseExpressionErrorPosition). Verified on Oracle 23ai: Boolean comparison and IN selectors on LIKEC, `CASE case IS NULL WHEN TRUE ...`, and a column named CASE with an alias are all valid. Reference row ref_102 pins the comparison selector; the keyword audit (232 words, 2,320 checks) still agrees with Oracle. Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 48 ++++++------------- oracle/parser/parser.go | 19 ++++++++ oracle/parser/select_test.go | 41 ++++++++++++++-- .../testdata/coverage/reference_oracle.tsv | 1 + 4 files changed, 71 insertions(+), 38 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index 4e9b5920..943f4407 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -689,42 +689,22 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { } // caseNamesColumn reports whether CASE at the current token is a column -// reference rather than a CASE expression. Oracle does not reserve CASE, and -// a CASE expression always continues with WHEN or an operand, never with a -// token that ends an operand or joins two of them. -// -// The words below that Oracle does not reserve need one more token, because -// they can also open a simple CASE selector (Oracle 23ai accepts each form): -// NOT starts a Boolean selector unless NOT IN / NOT BETWEEN / NOT LIKE -// follows; NULLS is a column named NULLS unless NULLS FIRST / NULLS LAST -// orders the column CASE; LIKEC, LIKE2, and LIKE4 are a selector when WHEN -// or an operator follows, and compare the column CASE otherwise. +// reference rather than a CASE expression. Oracle does not reserve CASE: it +// reads a CASE expression when WHEN follows (searched CASE) or when an +// expression follows and is itself followed by WHEN (simple CASE, whose +// selector may be any expression, Boolean ones included in 23ai), and a +// column otherwise (ORDER BY case NULLS FIRST, WHERE case LIKEC 'x%'). The +// selector is parsed on a probe whose state is then discarded, so a CASE +// expression is parsed once more for real and keeps its own errors. func (p *Parser) caseNamesColumn() bool { - switch p.peekNext().Type { - case tokEOF, ';', ',', ')', '.', '=', '<', '>', '*', '/', - tokLESSEQ, tokGREATEQ, tokNOTEQ, tokCONCAT, - kwFROM, kwWHERE, kwGROUP, kwORDER, kwHAVING, kwINTO, kwAS, - kwIS, kwIN, kwLIKE, kwBETWEEN, - kwAND, kwOR, kwASC, kwDESC, kwUNION, kwINTERSECT, kwMINUS: - return true - case kwNOT: - switch p.peekAhead(2)[1].Type { - case kwIN, kwBETWEEN, kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4: - return true - } - case kwNULLS: - switch p.peekAhead(2)[1].Type { - case kwFIRST, kwLAST: - return true - } - case kwLIKEC, kwLIKE2, kwLIKE4: - switch p.peekAhead(2)[1].Type { - case kwWHEN, '+', '-', '*', '/', '.', tokCONCAT, tokEXPON: - return false - } - return true + if p.peekNext().Type == kwWHEN { + return false } - return false + saved := p.saveState() + defer p.restoreState(saved) + p.advance() // consume CASE + selector, err := p.parseExpr() + return err != nil || selector == nil || p.cur.Type != kwWHEN } // keepsConstructBeforeOuterJoin reports the non-reserved keywords Oracle diff --git a/oracle/parser/parser.go b/oracle/parser/parser.go index f33595e7..190d2c85 100644 --- a/oracle/parser/parser.go +++ b/oracle/parser/parser.go @@ -356,6 +356,25 @@ func (p *Parser) peekAhead(n int) []Token { return toks } +// parserState is a snapshot of everything the parser advances, so a probe can +// parse ahead and then put the parser back where it was. +type parserState struct { + lexer Lexer + cur Token + prev Token + nextBuf Token + hasNext bool +} + +func (p *Parser) saveState() parserState { + return parserState{lexer: *p.lexer, cur: p.cur, prev: p.prev, nextBuf: p.nextBuf, hasNext: p.hasNext} +} + +func (p *Parser) restoreState(s parserState) { + *p.lexer = s.lexer + p.cur, p.prev, p.nextBuf, p.hasNext = s.cur, s.prev, s.nextBuf, s.hasNext +} + // peek returns the current token without consuming it. func (p *Parser) peek() Token { return p.cur diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index 709ce835..1c53a819 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -1,6 +1,8 @@ package parser import ( + "errors" + "strings" "testing" "github.com/bytebase/omni/oracle/ast" @@ -413,10 +415,11 @@ func TestParseOffsetColumnNameAST(t *testing.T) { } } -// TestParseCaseBooleanSelector: a simple CASE selector may start with NOT -// (a Boolean selector), NULLS, or LIKEC, while a column named CASE is -// followed by NOT only in NOT IN / NOT BETWEEN / NOT LIKE and by NULLS only -// in NULLS FIRST / NULLS LAST. Oracle 23ai accepts all of these. +// TestParseCaseBooleanSelector: CASE opens an expression when WHEN follows it +// or follows an expression after it; the selector may start with NOT, NULLS, +// or LIKEC and may be a Boolean comparison. Otherwise CASE is a column: +// case NOT IN (...), ORDER BY case NULLS FIRST, case LIKEC '...', and a column +// with an alias. Oracle 23ai accepts all of these. func TestParseCaseBooleanSelector(t *testing.T) { result := ParseAndCheck(t, "SELECT CASE NOT TRUE WHEN TRUE THEN 1 ELSE 0 END FROM t") target := result.Items[0].(*ast.RawStmt).Stmt.(*ast.SelectStmt).TargetList.Items[0].(*ast.ResTarget) @@ -428,6 +431,9 @@ func TestParseCaseBooleanSelector(t *testing.T) { "SELECT CASE nulls + 1 WHEN 1 THEN 2 END FROM t", "SELECT CASE likec WHEN 'x' THEN 2 END FROM t", "SELECT CASE like2 || 'a' WHEN 'xa' THEN 2 END FROM t", + "SELECT CASE likec = 'x' WHEN TRUE THEN 1 ELSE 0 END FROM t", + "SELECT CASE likec IN ('x', 'y') WHEN TRUE THEN 1 ELSE 0 END FROM t", + "SELECT CASE case IS NULL WHEN TRUE THEN 1 ELSE 0 END FROM t", } { t.Run(sql, func(t *testing.T) { result := ParseAndCheck(t, sql) @@ -444,6 +450,7 @@ func TestParseCaseBooleanSelector(t *testing.T) { "SELECT a FROM t ORDER BY case NULLS FIRST", "SELECT a FROM t ORDER BY case DESC NULLS LAST", "SELECT a FROM t WHERE case LIKEC '1%'", + "SELECT case a2 FROM t", } { t.Run(sql, func(t *testing.T) { ParseAndCheck(t, sql) @@ -476,3 +483,29 @@ func TestParseKeywordColumnOuterJoin(t *testing.T) { // A qualified reference is a column even for a pseudo-column word. ParseAndCheck(t, "SELECT 1 FROM t, u WHERE t.systimestamp(+) = u.a") } + +// TestParseCaseExpressionErrorPosition: deciding between a CASE expression +// and a column named CASE probes the selector without consuming it, so an +// error inside a real CASE expression is still reported where it occurs. +func TestParseCaseExpressionErrorPosition(t *testing.T) { + tests := []struct { + sql string + near string + }{ + {"SELECT CASE x WHEN 1 THEN (1 + ) END FROM t", ")"}, + {"SELECT CASE WHEN a = 1 THEN 1 FROM t", "FROM"}, + {"SELECT CASE likec = 'x' WHEN TRUE THEN (1 + ) END FROM t", ")"}, + } + for _, tt := range tests { + t.Run(tt.sql, func(t *testing.T) { + _, err := Parse(tt.sql) + var pe *ParseError + if !errors.As(err, &pe) { + t.Fatalf("Parse error = %v, want a ParseError", err) + } + if want := strings.Index(tt.sql, tt.near); pe.Position != want { + t.Fatalf("error at %d (%q), want %d (%q)", pe.Position, tt.sql[pe.Position:], want, tt.near) + } + }) + } +} diff --git a/oracle/parser/testdata/coverage/reference_oracle.tsv b/oracle/parser/testdata/coverage/reference_oracle.tsv index be3052a8..0ce38f45 100644 --- a/oracle/parser/testdata/coverage/reference_oracle.tsv +++ b/oracle/parser/testdata/coverage/reference_oracle.tsv @@ -100,3 +100,4 @@ ref_098 select accept SELECT a FROM t_kwcol ORDER BY case DESC NULLS LAST column ref_099 select accept SELECT a FROM t_kwcol WHERE case LIKEC '1%' column named CASE compared with LIKEC ref_100 select accept SELECT CASE nulls WHEN 1 THEN 2 END FROM t_kwcol column named NULLS as a simple CASE selector ref_101 select accept SELECT CASE likec WHEN 'x' THEN 2 END FROM t_kwcol column named LIKEC as a simple CASE selector +ref_102 select accept SELECT CASE likec = 'x' WHEN TRUE THEN 1 ELSE 0 END FROM t_kwcol simple CASE with a Boolean comparison selector From ab841ae3a452dbf5711574b792b3f11dabd99bc8 Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 04:08:54 +0000 Subject: [PATCH 06/11] fix(oracle): parse a CASE selector once, so nested CASE stays linear (BYT-10301) Codex review on #446: probing the selector and then reparsing it made nested simple CASE selectors exponential, doubling per level (20 levels, 478 bytes, took 1 s; each further level doubled it), which a short statement could use to exhaust the parser in review or completion. parseCaseOrColumn replaces the probe-then-parse pair. When the selector is followed by WHEN, the CASE expression continues from the selector already parsed (parseCaseWhens). When it is not, the parser state is restored and the CASE offset recorded in caseColumns, so reparsing what follows does not probe the same CASE again; without that record a chain of columns named CASE (case LIKEC (case LIKEC (...))) is exponential too (1.5 s at 20 levels). Measured after the change: 400 nested selectors (9 KB) parse in under 1 ms, 400 nested CASE columns in about 60 ms. TestParseNestedCaseStaysFast pins both shapes at 60 levels. Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 66 +++++++++++++++++++++++------------- oracle/parser/parser.go | 5 +++ oracle/parser/select_test.go | 26 ++++++++++++++ 3 files changed, 74 insertions(+), 23 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index 943f4407..b10668f3 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -599,10 +599,7 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { return p.parseParenExpr() case kwCASE: - if p.caseNamesColumn() { - return p.parseIdentExpr() - } - return p.parseCaseExpr() + return p.parseCaseOrColumn() // CAST, DECODE, and INTERVAL are not reserved; without the '(' or the // string literal that opens the function or literal, they name a column. @@ -688,23 +685,38 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { } } -// caseNamesColumn reports whether CASE at the current token is a column -// reference rather than a CASE expression. Oracle does not reserve CASE: it -// reads a CASE expression when WHEN follows (searched CASE) or when an -// expression follows and is itself followed by WHEN (simple CASE, whose -// selector may be any expression, Boolean ones included in 23ai), and a -// column otherwise (ORDER BY case NULLS FIRST, WHERE case LIKEC 'x%'). The -// selector is parsed on a probe whose state is then discarded, so a CASE -// expression is parsed once more for real and keeps its own errors. -func (p *Parser) caseNamesColumn() bool { +// parseCaseOrColumn parses CASE as Oracle reads it, since Oracle does not +// reserve the word: a CASE expression when WHEN follows it (searched CASE) or +// when an expression follows and is itself followed by WHEN (simple CASE, +// whose selector may be any expression, Boolean ones included in 23ai), and +// a column named CASE otherwise (ORDER BY case NULLS FIRST, +// WHERE case LIKEC 'x%'). +// +// The selector is parsed once. When WHEN follows, it is kept and the CASE +// expression continues from there, so nested CASE selectors stay linear and +// errors inside the expression keep their positions. Otherwise the parser +// state is restored and the offset recorded in caseColumns, so the reparse +// of what follows does not probe this CASE again. +func (p *Parser) parseCaseOrColumn() (nodes.ExprNode, error) { + start := p.pos() if p.peekNext().Type == kwWHEN { - return false + return p.parseCaseExpr() + } + if p.caseColumns[start] { + return p.parseIdentExpr() } saved := p.saveState() - defer p.restoreState(saved) p.advance() // consume CASE selector, err := p.parseExpr() - return err != nil || selector == nil || p.cur.Type != kwWHEN + if err == nil && selector != nil && p.cur.Type == kwWHEN { + return p.parseCaseWhens(start, selector) + } + p.restoreState(saved) + if p.caseColumns == nil { + p.caseColumns = make(map[int]bool) + } + p.caseColumns[start] = true + return p.parseIdentExpr() } // keepsConstructBeforeOuterJoin reports the non-reserved keywords Oracle @@ -1687,23 +1699,31 @@ func (p *Parser) parseCaseExpr() (nodes.ExprNode, error) { start := p.pos() p.advance() // consume CASE - ce := &nodes.CaseExpr{ - Whens: &nodes.List{}, - Loc: nodes.Loc{Start: start}, - } - // Simple CASE: CASE expr WHEN ... // Searched CASE: CASE WHEN ... + var arg nodes.ExprNode if p.cur.Type != kwWHEN { var parseErr730 error - ce.Arg, parseErr730 = p.parseExpr() + arg, parseErr730 = p.parseExpr() if parseErr730 != nil { return nil, parseErr730 } - if ce.Arg == nil { + if arg == nil { return nil, p.syntaxErrorAtCur() } } + return p.parseCaseWhens(start, arg) +} + +// parseCaseWhens parses the WHEN ... [ELSE ...] END of a CASE expression that +// starts at start, after CASE and the selector arg (nil for a searched CASE) +// have been consumed. +func (p *Parser) parseCaseWhens(start int, arg nodes.ExprNode) (nodes.ExprNode, error) { + ce := &nodes.CaseExpr{ + Arg: arg, + Whens: &nodes.List{}, + Loc: nodes.Loc{Start: start}, + } for p.cur.Type == kwWHEN { whenStart := p.pos() diff --git a/oracle/parser/parser.go b/oracle/parser/parser.go index 190d2c85..3aea8547 100644 --- a/oracle/parser/parser.go +++ b/oracle/parser/parser.go @@ -22,6 +22,11 @@ type Parser struct { prev Token // previous token (for error reporting) nextBuf Token // buffered next token for 2-token lookahead hasNext bool // whether nextBuf is valid + + // caseColumns holds the offsets of CASE tokens parseCaseOrColumn has + // found to name a column, so a reparse after a failed probe does not + // probe the same CASE again. + caseColumns map[int]bool } // Parse parses a SQL string into an AST list. diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index 1c53a819..6ec2bb70 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -4,6 +4,7 @@ import ( "errors" "strings" "testing" + "time" "github.com/bytebase/omni/oracle/ast" ) @@ -509,3 +510,28 @@ func TestParseCaseExpressionErrorPosition(t *testing.T) { }) } } + +// TestParseNestedCaseStaysFast guards parseCaseOrColumn against reparsing: +// a CASE whose selector is another simple CASE, and a column named CASE +// compared with an operand holding another such column, each nested 60 +// deep. Parsing either twice per level is exponential (seconds by 20 levels); +// both take milliseconds. +func TestParseNestedCaseStaysFast(t *testing.T) { + const depth = 60 + selector := "1" + column := "'1%'" + for i := 0; i < depth; i++ { + selector = "CASE " + selector + " WHEN 1 THEN 1 END" + column = "case LIKEC (" + column + ")" + } + for _, sql := range []string{ + "SELECT " + selector + " FROM dual", + "SELECT a FROM t WHERE " + column, + } { + start := time.Now() + _, _ = Parse(sql) + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Fatalf("parsing %d nested CASE levels took %s: %.60q...", depth, elapsed, sql) + } + } +} From e272144cfd8b45541619922dbbaceeab230b6918 Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 04:26:58 +0000 Subject: [PATCH 07/11] fix(oracle): decide every CASE in one linear pre-pass (BYT-10301) Codex review on #446: probing the expression after each CASE kept the work quadratic for long sums of a column named CASE (case + case + ...; 1,600 terms took 0.68 s, doubling the input quadrupled it), because each failed probe parsed the whole remaining expression before the CASE was recorded as a column. The probe is gone. The first time parsePrimary meets CASE, decideCaseTokens lexes the rest of the range once and settles every CASE right to left with a jump table: CASE opens an expression when the first token after it that can end an operand at its depth is WHEN, skipping parenthesized groups and the CASE expressions after it. The token right after CASE must be able to start a selector (case IS NULL is a column), and END there is a column named END (Oracle accepts CASE end WHEN ..., which the keyword audit caught). Each token is visited a constant number of times. Measured: 20,000 CASE column terms (140 KB) in 24 ms, 4,000 nested CASE selectors in 11 ms, 4,000 nested CASE columns in 11 ms. TestParseCaseDecisionStaysLinear pins all three; the keyword audit (2,320 checks) and the CASE tests still agree with Oracle. Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 180 ++++++++++++++++++++++++++++++----- oracle/parser/parser.go | 27 +----- oracle/parser/select_test.go | 27 +++--- 3 files changed, 175 insertions(+), 59 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index b10668f3..a47dc6b6 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -687,36 +687,168 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { // parseCaseOrColumn parses CASE as Oracle reads it, since Oracle does not // reserve the word: a CASE expression when WHEN follows it (searched CASE) or -// when an expression follows and is itself followed by WHEN (simple CASE, -// whose selector may be any expression, Boolean ones included in 23ai), and -// a column named CASE otherwise (ORDER BY case NULLS FIRST, -// WHERE case LIKEC 'x%'). -// -// The selector is parsed once. When WHEN follows, it is kept and the CASE -// expression continues from there, so nested CASE selectors stay linear and -// errors inside the expression keep their positions. Otherwise the parser -// state is restored and the offset recorded in caseColumns, so the reparse -// of what follows does not probe this CASE again. +// follows the selector expression after it (simple CASE, whose selector may +// be any expression, Boolean ones included in 23ai), and a column named CASE +// otherwise (ORDER BY case NULLS FIRST, WHERE case LIKEC 'x%', case + 1). +// decideCaseTokens settles every CASE of the range in one linear pass, so no +// expression is parsed twice to decide. func (p *Parser) parseCaseOrColumn() (nodes.ExprNode, error) { - start := p.pos() - if p.peekNext().Type == kwWHEN { - return p.parseCaseExpr() + if p.caseKinds == nil { + p.caseKinds = decideCaseTokens(p.source, p.cur.Loc, p.lexer.end) } - if p.caseColumns[start] { + opensExpression, decided := p.caseKinds[p.cur.Loc] + if decided && !opensExpression { return p.parseIdentExpr() } - saved := p.saveState() - p.advance() // consume CASE - selector, err := p.parseExpr() - if err == nil && selector != nil && p.cur.Type == kwWHEN { - return p.parseCaseWhens(start, selector) + return p.parseCaseExpr() +} + +// decideCaseTokens lexes source[start:end] and reports, for each CASE token, +// whether it opens a CASE expression. A CASE opens one when the first token +// after it that can end an operand at its own depth is WHEN: the scan skips +// parenthesized groups and the CASE expressions that start after it, and +// stops at WHEN, THEN, ELSE, END, ')', ',', ';', and the reserved words that +// begin a clause. Deciding right to left with a jump table (stop) visits each +// token a constant number of times. +func decideCaseTokens(source string, start, end int) map[int]bool { + lexer := NewLexerRange(source, start, end) + var toks []Token + for { + tok := lexer.NextToken() + if tok.Type == tokEOF || lexer.Err != nil { + break + } + toks = append(toks, tok) + } + n := len(toks) + + // closing[i] is the index of the ')' that closes the '(' at i, or n. + closing := make([]int, n) + var open []int + for i, tok := range toks { + switch tok.Type { + case '(': + closing[i] = n + open = append(open, i) + case ')': + if len(open) > 0 { + closing[open[len(open)-1]] = i + open = open[:len(open)-1] + } + } } - p.restoreState(saved) - if p.caseColumns == nil { - p.caseColumns = make(map[int]bool) + + // stop[i] is the index of the first token from i on, at i's depth, that + // ends an operand; n stands for the end of the range. + stop := make([]int, n+1) + stop[n] = n + after := func(i int) int { + if i+1 >= n { + return n + } + return stop[i+1] + } + kinds := make(map[int]bool) + for i := n - 1; i >= 0; i-- { + tok := toks[i] + switch { + case isCaseScanStop(tok): + stop[i] = i + case tok.Type == '(': + stop[i] = after(closing[i]) + case tok.Type == kwCASE: + // s is where the scan for WHEN stops; -1 when the token after + // CASE cannot start a selector, so CASE is a column (case IS NULL). + s := -1 + switch { + case i+1 >= n: + case toks[i+1].Type == kwWHEN: + s = i + 1 + case !canStartCaseSelector(toks[i+1]): + case toks[i+1].Type == kwEND: + s = after(i + 1) // a column named END as the selector + default: + s = after(i) + } + if s >= 0 && s < n && toks[s].Type == kwWHEN { + kinds[tok.Loc] = true + if e := caseExprEnd(toks, stop, s); e >= 0 { + stop[i] = after(e) + } else { + stop[i] = s + } + } else { + kinds[tok.Loc] = false + stop[i] = after(i) + } + default: + stop[i] = after(i) + } } - p.caseColumns[start] = true - return p.parseIdentExpr() + return kinds +} + +// caseExprEnd follows WHEN ... THEN ... [WHEN ... THEN ...]... [ELSE ...] END +// from the WHEN at w through the stop table and returns the index of END, or +// -1 when the CASE expression is malformed. +func caseExprEnd(toks []Token, stop []int, w int) int { + n := len(toks) + next := func(i int) int { + if i+1 >= n { + return n + } + return stop[i+1] + } + for { + then := next(w) + if then >= n || toks[then].Type != kwTHEN { + return -1 + } + r := next(then) + if r >= n { + return -1 + } + switch toks[r].Type { + case kwWHEN: + w = r + case kwELSE: + if e := next(r); e < n && toks[e].Type == kwEND { + return e + } + return -1 + case kwEND: + return r + default: + return -1 + } + } +} + +// canStartCaseSelector reports whether tok can begin the selector of a simple +// CASE. Operators and keywords that only follow an operand (=, IS, IN, AND, +// ...) and the clause words that end one cannot; END can, as a column named +// END, which Oracle accepts there. +func canStartCaseSelector(tok Token) bool { + switch tok.Type { + case '=', '<', '>', '*', '/', '.', tokLESSEQ, tokGREATEQ, tokNOTEQ, + tokCONCAT, tokEXPON, kwIS, kwIN, kwLIKE, kwBETWEEN, kwAND, kwOR, + kwASC, kwDESC: + return false + case kwEND: + return true + } + return !isCaseScanStop(tok) +} + +// isCaseScanStop reports the tokens that end an operand for decideCaseTokens. +func isCaseScanStop(tok Token) bool { + switch tok.Type { + case kwWHEN, kwTHEN, kwELSE, kwEND, ')', ',', ';', + kwFROM, kwWHERE, kwGROUP, kwHAVING, kwORDER, kwUNION, kwINTERSECT, + kwMINUS, kwINTO, kwAS, kwCONNECT, kwSTART, kwSET, kwVALUES, kwFOR: + return true + } + return false } // keepsConstructBeforeOuterJoin reports the non-reserved keywords Oracle diff --git a/oracle/parser/parser.go b/oracle/parser/parser.go index 3aea8547..ecc2e154 100644 --- a/oracle/parser/parser.go +++ b/oracle/parser/parser.go @@ -23,10 +23,10 @@ type Parser struct { nextBuf Token // buffered next token for 2-token lookahead hasNext bool // whether nextBuf is valid - // caseColumns holds the offsets of CASE tokens parseCaseOrColumn has - // found to name a column, so a reparse after a failed probe does not - // probe the same CASE again. - caseColumns map[int]bool + // caseKinds tells, for each CASE token from the first one parsePrimary + // met to the end of the range, whether it opens a CASE expression (true) + // or names a column (false). decideCaseTokens fills it in one pass. + caseKinds map[int]bool } // Parse parses a SQL string into an AST list. @@ -361,25 +361,6 @@ func (p *Parser) peekAhead(n int) []Token { return toks } -// parserState is a snapshot of everything the parser advances, so a probe can -// parse ahead and then put the parser back where it was. -type parserState struct { - lexer Lexer - cur Token - prev Token - nextBuf Token - hasNext bool -} - -func (p *Parser) saveState() parserState { - return parserState{lexer: *p.lexer, cur: p.cur, prev: p.prev, nextBuf: p.nextBuf, hasNext: p.hasNext} -} - -func (p *Parser) restoreState(s parserState) { - *p.lexer = s.lexer - p.cur, p.prev, p.nextBuf, p.hasNext = s.cur, s.prev, s.nextBuf, s.hasNext -} - // peek returns the current token without consuming it. func (p *Parser) peek() Token { return p.cur diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index 6ec2bb70..61030e72 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -511,27 +511,30 @@ func TestParseCaseExpressionErrorPosition(t *testing.T) { } } -// TestParseNestedCaseStaysFast guards parseCaseOrColumn against reparsing: -// a CASE whose selector is another simple CASE, and a column named CASE -// compared with an operand holding another such column, each nested 60 -// deep. Parsing either twice per level is exponential (seconds by 20 levels); -// both take milliseconds. -func TestParseNestedCaseStaysFast(t *testing.T) { - const depth = 60 +// TestParseCaseDecisionStaysLinear guards how CASE is told apart from a column +// named CASE: deciding by parsing what follows each CASE made nested simple +// CASE selectors exponential and long sums of CASE columns quadratic. Each +// shape below takes tens of milliseconds when linear, and seconds to minutes +// otherwise. +func TestParseCaseDecisionStaysLinear(t *testing.T) { + const depth = 4000 selector := "1" column := "'1%'" for i := 0; i < depth; i++ { selector = "CASE " + selector + " WHEN 1 THEN 1 END" column = "case LIKEC (" + column + ")" } - for _, sql := range []string{ - "SELECT " + selector + " FROM dual", - "SELECT a FROM t WHERE " + column, + for name, sql := range map[string]string{ + "nested CASE selectors": "SELECT " + selector + " FROM dual", + "nested CASE columns": "SELECT a FROM t WHERE " + column, + "sum of 20000 CASE column terms": "SELECT " + strings.Repeat("case + ", 20000) + "1 FROM t", } { start := time.Now() - _, _ = Parse(sql) + if _, err := Parse(sql); err != nil { + t.Fatalf("%s: %v", name, err) + } if elapsed := time.Since(start); elapsed > 5*time.Second { - t.Fatalf("parsing %d nested CASE levels took %s: %.60q...", depth, elapsed, sql) + t.Fatalf("%s took %s", name, elapsed) } } } From 8cafe4927a53b9f04976c409aa6548b9b6c3ba88 Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 04:38:10 +0000 Subject: [PATCH 08/11] fix(oracle): treat t.case, t.end, and operand END as names in the CASE pre-pass (BYT-10301) Codex review on #446: the pre-pass took every END for the end of a CASE and every CASE for an opener, so `CASE t.end WHEN 1 THEN 2 END`, `CASE 1 + end WHEN 2 THEN 3 END`, and `CASE t.case WHEN 1 THEN 2 END` were read with a column named CASE and failed. Oracle 23ai accepts all three, and `THEN end ELSE 0 END` too. A word after '.' is now a name component the scan ignores, and END right where an operand must start (after an operator, '(', ',', CASE, WHEN, THEN, ELSE, AND, OR, NOT, ...) is a column named END; only END after an operand closes a CASE. This also covers END as the first selector token, which had its own branch. The keyword audit gains qualified and operand CASE selector contexts (232 words, 2,784 checks), so every non-reserved word is checked in those shapes against Oracle; it still agrees apart from the two recorded gaps. Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 23 +++++++++++++++++++-- oracle/parser/keyword_column_oracle_test.go | 6 ++++-- oracle/parser/select_test.go | 5 +++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index a47dc6b6..cc366a13 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -751,7 +751,14 @@ func decideCaseTokens(source string, start, end int) map[int]bool { kinds := make(map[int]bool) for i := n - 1; i >= 0; i-- { tok := toks[i] + // A word after '.' is a name component (t.case, t.end), and END where + // an operand must start is a column named END (1 + end, THEN end): + // neither opens nor closes anything for the scan. + plain := i > 0 && (toks[i-1].Type == '.' || + tok.Type == kwEND && expectsOperandAfter(toks[i-1])) switch { + case plain: + stop[i] = after(i) case isCaseScanStop(tok): stop[i] = i case tok.Type == '(': @@ -765,8 +772,6 @@ func decideCaseTokens(source string, start, end int) map[int]bool { case toks[i+1].Type == kwWHEN: s = i + 1 case !canStartCaseSelector(toks[i+1]): - case toks[i+1].Type == kwEND: - s = after(i + 1) // a column named END as the selector default: s = after(i) } @@ -840,6 +845,20 @@ func canStartCaseSelector(tok Token) bool { return !isCaseScanStop(tok) } +// expectsOperandAfter reports whether an operand must follow tok, so an END +// right after it is a column named END rather than the end of a CASE. +func expectsOperandAfter(tok Token) bool { + switch tok.Type { + case '+', '-', '*', '/', '=', '<', '>', '(', ',', '.', + tokCONCAT, tokEXPON, tokLESSEQ, tokGREATEQ, tokNOTEQ, + kwCASE, kwWHEN, kwTHEN, kwELSE, kwAND, kwOR, kwNOT, kwPRIOR, + kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4, kwBETWEEN, + kwSELECT, kwWHERE, kwBY, kwHAVING, kwON, kwSET, kwRETURN: + return true + } + return false +} + // isCaseScanStop reports the tokens that end an operand for decideCaseTokens. func isCaseScanStop(tok Token) bool { switch tok.Type { diff --git a/oracle/parser/keyword_column_oracle_test.go b/oracle/parser/keyword_column_oracle_test.go index 5fd82723..eb9bd047 100644 --- a/oracle/parser/keyword_column_oracle_test.go +++ b/oracle/parser/keyword_column_oracle_test.go @@ -27,14 +27,16 @@ var keywordColumnContexts = []struct { {"index", "CREATE INDEX %[1]s_i ON %[1]s (%[2]s)"}, {"outer join marker", "SELECT d.dummy FROM dual d, %[1]s WHERE %[2]s(+) = LENGTH(d.dummy)"}, {"case selector", "SELECT CASE %[2]s WHEN 1 THEN 2 END FROM %[1]s"}, + {"qualified case selector", "SELECT CASE k.%[2]s WHEN 1 THEN 2 END FROM %[1]s k"}, + {"case selector operand", "SELECT CASE 1 + %[2]s WHEN 2 THEN 3 END FROM %[1]s"}, } // TestOracleNonReservedKeywordsAsColumns checks every word omni lexes as a // keyword that Oracle does not reserve (V$RESERVED_WORDS) as a column name: // defined by CREATE TABLE, then referenced from a select list, WHERE // predicates (IS NULL, NOT IN), ORDER BY, UPDATE, an INSERT column list, an -// index, with the legacy outer-join marker (+), and as a simple CASE -// selector. omni must +// index, with the legacy outer-join marker (+), and in a simple CASE +// selector (alone, qualified, and as an operand). omni must // accept or reject each statement exactly as Oracle does. The keyword // manifest's column_name context covers only the definition; OFFSET, FETCH, // JOIN, MODEL, and USING failed in the references alone. diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index 61030e72..d14fe49c 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -435,6 +435,11 @@ func TestParseCaseBooleanSelector(t *testing.T) { "SELECT CASE likec = 'x' WHEN TRUE THEN 1 ELSE 0 END FROM t", "SELECT CASE likec IN ('x', 'y') WHEN TRUE THEN 1 ELSE 0 END FROM t", "SELECT CASE case IS NULL WHEN TRUE THEN 1 ELSE 0 END FROM t", + "SELECT CASE end WHEN 1 THEN 2 END FROM t", + "SELECT CASE t.end WHEN 1 THEN 2 END FROM t", + "SELECT CASE 1 + end WHEN 2 THEN 3 END FROM t", + "SELECT CASE t.case WHEN 1 THEN 2 END FROM t", + "SELECT CASE a WHEN 1 THEN end ELSE 0 END FROM t", } { t.Run(sql, func(t *testing.T) { result := ParseAndCheck(t, sql) From f44b6635bf7f0503f903e04478aa0a8c87f42ad7 Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 04:48:10 +0000 Subject: [PATCH 09/11] fix(oracle): END after CONNECT_BY_ROOT, ESCAPE, OF, or ZONE is a column in the CASE pre-pass (BYT-10301) Codex review on #446: `SELECT CASE CONNECT_BY_ROOT end WHEN 1 THEN 2 END FROM t CONNECT BY ...` failed because expectsOperandAfter knew the unary PRIOR but not CONNECT_BY_ROOT, so the pre-pass took the column END for the end of the CASE. Auditing the other expression-level keywords that take an operand found the same gap after ESCAPE (`CASE a LIKE 'x' ESCAPE end WHEN TRUE ...`), OF (MEMBER OF, SUBMULTISET OF), and ZONE (AT TIME ZONE); Oracle 23ai accepts the CONNECT_BY_ROOT and ESCAPE forms. Tokens inside parentheses do not matter: the scan skips the group. Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 8 ++++++-- oracle/parser/select_test.go | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index cc366a13..83c4522b 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -846,12 +846,16 @@ func canStartCaseSelector(tok Token) bool { } // expectsOperandAfter reports whether an operand must follow tok, so an END -// right after it is a column named END rather than the end of a CASE. +// right after it is a column named END rather than the end of a CASE. Only +// tokens that work at expression level matter: an END inside parentheses is +// skipped with its group. The unary PRIOR and CONNECT_BY_ROOT, ESCAPE, OF +// (MEMBER OF, SUBMULTISET OF), and ZONE (AT TIME ZONE) all take an operand. func expectsOperandAfter(tok Token) bool { switch tok.Type { case '+', '-', '*', '/', '=', '<', '>', '(', ',', '.', tokCONCAT, tokEXPON, tokLESSEQ, tokGREATEQ, tokNOTEQ, - kwCASE, kwWHEN, kwTHEN, kwELSE, kwAND, kwOR, kwNOT, kwPRIOR, + kwCASE, kwWHEN, kwTHEN, kwELSE, kwAND, kwOR, kwNOT, + kwPRIOR, kwCONNECT_BY_ROOT, kwESCAPE, kwOF, kwZONE, kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4, kwBETWEEN, kwSELECT, kwWHERE, kwBY, kwHAVING, kwON, kwSET, kwRETURN: return true diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index d14fe49c..a52ff7ee 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -440,6 +440,9 @@ func TestParseCaseBooleanSelector(t *testing.T) { "SELECT CASE 1 + end WHEN 2 THEN 3 END FROM t", "SELECT CASE t.case WHEN 1 THEN 2 END FROM t", "SELECT CASE a WHEN 1 THEN end ELSE 0 END FROM t", + "SELECT CASE CONNECT_BY_ROOT end WHEN 1 THEN 2 END FROM t CONNECT BY PRIOR id = pid", + "SELECT CASE PRIOR end WHEN 1 THEN 2 END FROM t CONNECT BY PRIOR id = pid", + "SELECT CASE a LIKE 'x' ESCAPE end WHEN TRUE THEN 1 END FROM t", } { t.Run(sql, func(t *testing.T) { result := ParseAndCheck(t, sql) From ff32eea9ce4975a6d14d9962e4e254dadf88dc3a Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 04:57:09 +0000 Subject: [PATCH 10/11] fix(oracle): skip bracketed MODEL cell references in the CASE pre-pass (BYT-10301) Codex review on #446: `CASE sales['Mouse Pad', 1998] WHEN 1 THEN 2 END` regressed because the pre-pass matched only parentheses, so the comma inside the MODEL cell reference ended the scan and the CASE was taken for a column. Brackets are now matched and skipped like parentheses, and ']' ends an operand like ')'. Verified on Oracle 23ai with a two-dimensional MODEL rule; reference row ref_103 pins it. Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 17 +++++++++++------ oracle/parser/select_test.go | 1 + .../testdata/coverage/reference_oracle.tsv | 1 + 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index 83c4522b..e16112ce 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -722,16 +722,21 @@ func decideCaseTokens(source string, start, end int) map[int]bool { } n := len(toks) - // closing[i] is the index of the ')' that closes the '(' at i, or n. + // closing[i] is the index of the ')' or ']' that closes the '(' or '[' + // at i (a MODEL cell reference such as s['Mouse Pad', 1998]), or n. closing := make([]int, n) var open []int for i, tok := range toks { switch tok.Type { - case '(': + case '(', '[': closing[i] = n open = append(open, i) - case ')': - if len(open) > 0 { + case ')', ']': + want := '(' + if tok.Type == ']' { + want = '[' + } + if len(open) > 0 && toks[open[len(open)-1]].Type == int(want) { closing[open[len(open)-1]] = i open = open[:len(open)-1] } @@ -761,7 +766,7 @@ func decideCaseTokens(source string, start, end int) map[int]bool { stop[i] = after(i) case isCaseScanStop(tok): stop[i] = i - case tok.Type == '(': + case tok.Type == '(' || tok.Type == '[': stop[i] = after(closing[i]) case tok.Type == kwCASE: // s is where the scan for WHEN stops; -1 when the token after @@ -866,7 +871,7 @@ func expectsOperandAfter(tok Token) bool { // isCaseScanStop reports the tokens that end an operand for decideCaseTokens. func isCaseScanStop(tok Token) bool { switch tok.Type { - case kwWHEN, kwTHEN, kwELSE, kwEND, ')', ',', ';', + case kwWHEN, kwTHEN, kwELSE, kwEND, ')', ']', ',', ';', kwFROM, kwWHERE, kwGROUP, kwHAVING, kwORDER, kwUNION, kwINTERSECT, kwMINUS, kwINTO, kwAS, kwCONNECT, kwSTART, kwSET, kwVALUES, kwFOR: return true diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index a52ff7ee..f408320d 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -443,6 +443,7 @@ func TestParseCaseBooleanSelector(t *testing.T) { "SELECT CASE CONNECT_BY_ROOT end WHEN 1 THEN 2 END FROM t CONNECT BY PRIOR id = pid", "SELECT CASE PRIOR end WHEN 1 THEN 2 END FROM t CONNECT BY PRIOR id = pid", "SELECT CASE a LIKE 'x' ESCAPE end WHEN TRUE THEN 1 END FROM t", + "SELECT CASE sales['Mouse Pad', 1998] WHEN 1 THEN 2 END FROM t", } { t.Run(sql, func(t *testing.T) { result := ParseAndCheck(t, sql) diff --git a/oracle/parser/testdata/coverage/reference_oracle.tsv b/oracle/parser/testdata/coverage/reference_oracle.tsv index 0ce38f45..1376667f 100644 --- a/oracle/parser/testdata/coverage/reference_oracle.tsv +++ b/oracle/parser/testdata/coverage/reference_oracle.tsv @@ -101,3 +101,4 @@ ref_099 select accept SELECT a FROM t_kwcol WHERE case LIKEC '1%' column named C ref_100 select accept SELECT CASE nulls WHEN 1 THEN 2 END FROM t_kwcol column named NULLS as a simple CASE selector ref_101 select accept SELECT CASE likec WHEN 'x' THEN 2 END FROM t_kwcol column named LIKEC as a simple CASE selector ref_102 select accept SELECT CASE likec = 'x' WHEN TRUE THEN 1 ELSE 0 END FROM t_kwcol simple CASE with a Boolean comparison selector +ref_103 select accept SELECT k, j, v FROM (SELECT 1 k, 1 j, 10 v FROM dual) MODEL DIMENSION BY (k, j) MEASURES (v) RULES (v[2, 1] = CASE v[1, 1] WHEN 10 THEN 20 END) MODEL cell reference as a simple CASE selector From e596e9bd77e17c128531d599b252cbc9f94cef1f Mon Sep 17 00:00:00 2001 From: rebelice Date: Sat, 10 Oct 2026 05:12:09 +0000 Subject: [PATCH 11/11] fix(oracle): read case[...] as a MODEL cell of a measure named CASE (BYT-10301) Codex review on #446: with a MODEL measure named CASE, `RULES (case[2] = CASE case[1] WHEN 10 THEN 20 END)` failed because the pre-pass let '[' start the selector of the inner CASE, which then consumed the outer WHEN ... END. '[' cannot begin a selector, so CASE followed by '[' is now the cell-reference identifier. Oracle 23ai runs the full query (main rejected it too, at the select-list column CASE); reference row ref_104 pins it. Co-Authored-By: Claude Opus 5.5 --- oracle/parser/expr.go | 7 ++++--- oracle/parser/select_test.go | 1 + oracle/parser/testdata/coverage/reference_oracle.tsv | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index e16112ce..58ba0401 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -836,11 +836,12 @@ func caseExprEnd(toks []Token, stop []int, w int) int { // canStartCaseSelector reports whether tok can begin the selector of a simple // CASE. Operators and keywords that only follow an operand (=, IS, IN, AND, -// ...) and the clause words that end one cannot; END can, as a column named -// END, which Oracle accepts there. +// ...), '[' (case[1] is a MODEL cell of a measure named CASE), and the clause +// words that end one cannot; END can, as a column named END, which Oracle +// accepts there. func canStartCaseSelector(tok Token) bool { switch tok.Type { - case '=', '<', '>', '*', '/', '.', tokLESSEQ, tokGREATEQ, tokNOTEQ, + case '=', '<', '>', '*', '/', '.', '[', tokLESSEQ, tokGREATEQ, tokNOTEQ, tokCONCAT, tokEXPON, kwIS, kwIN, kwLIKE, kwBETWEEN, kwAND, kwOR, kwASC, kwDESC: return false diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index f408320d..e99a68eb 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -444,6 +444,7 @@ func TestParseCaseBooleanSelector(t *testing.T) { "SELECT CASE PRIOR end WHEN 1 THEN 2 END FROM t CONNECT BY PRIOR id = pid", "SELECT CASE a LIKE 'x' ESCAPE end WHEN TRUE THEN 1 END FROM t", "SELECT CASE sales['Mouse Pad', 1998] WHEN 1 THEN 2 END FROM t", + "SELECT CASE case[1] WHEN 10 THEN 20 END FROM t", } { t.Run(sql, func(t *testing.T) { result := ParseAndCheck(t, sql) diff --git a/oracle/parser/testdata/coverage/reference_oracle.tsv b/oracle/parser/testdata/coverage/reference_oracle.tsv index 1376667f..46e1df91 100644 --- a/oracle/parser/testdata/coverage/reference_oracle.tsv +++ b/oracle/parser/testdata/coverage/reference_oracle.tsv @@ -102,3 +102,4 @@ ref_100 select accept SELECT CASE nulls WHEN 1 THEN 2 END FROM t_kwcol column na ref_101 select accept SELECT CASE likec WHEN 'x' THEN 2 END FROM t_kwcol column named LIKEC as a simple CASE selector ref_102 select accept SELECT CASE likec = 'x' WHEN TRUE THEN 1 ELSE 0 END FROM t_kwcol simple CASE with a Boolean comparison selector ref_103 select accept SELECT k, j, v FROM (SELECT 1 k, 1 j, 10 v FROM dual) MODEL DIMENSION BY (k, j) MEASURES (v) RULES (v[2, 1] = CASE v[1, 1] WHEN 10 THEN 20 END) MODEL cell reference as a simple CASE selector +ref_104 select accept SELECT k, case FROM (SELECT 1 k, 10 v FROM dual) MODEL DIMENSION BY (k) MEASURES (v case) RULES (case[2] = CASE case[1] WHEN 10 THEN 20 END) MODEL measure named CASE