diff --git a/AGENTS.md b/AGENTS.md index 6e59ad774..f8aacae5e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ The rules above are the target. These places do not meet them yet; do not copy t - Engine dialect and DDL fidelity tests live here, beside the package they cover. Bytebase tests its API and workflow against PostgreSQL only and trusts omni's result, so a behavior Bytebase depends on is pinned by an omni test. - Test against the engine, not against memory. Parser acceptance, catalog results, and generated DDL are verified against a real engine wherever one can run: testcontainers for pg, mysql, mariadb, tidb, mssql, doris, and starrocks; Trino and the Spanner emulator as CI services; Oracle and Redshift through `ORACLE_PARSER_REF_DSN` and `REDSHIFT_COMPAT_DSN`. Gate a test on an environment variable only when it needs a server CI cannot start. - Container tests fail when Docker is missing; they do not skip. CI runs every test on every PR with no `-short` and no nightly. Never check `testing.Short()`, and never park a known gap behind a skip. -- Pinned images are part of the recorded expectations: `trinodb/trino:482` and the Spanner emulator `1.5.54` in `.github/workflows/ci.yml`, `postgres:17-alpine` in `pg/parser`. Upgrading one means rerunning its oracle tests and reading the drift, not editing expectations to match. +- Pinned images are part of the recorded expectations: `trinodb/trino:482` and the Spanner emulator `1.5.54` in `.github/workflows/ci.yml`, `postgres:17-alpine` in `pg/parser`, `gvenzl/oracle-free:23.26.3-slim-faststart` in `oracle/parser`. Upgrading one means rerunning its oracle tests and reading the drift, not editing expectations to match. - CI runs the sweeps at `PAREN_FUZZ_SIZE=10000`, `SPLITTEST_N=1000000`, and `S3DIFF_N=2000`; local defaults are smaller. Trino packages run one at a time because they share the service and run stateful DDL against the same names. - A PR runs the packages `scripts/affected-packages.sh` derives from the diff. A change to `go.mod`, `proto/`, `scripts/`, `harness/`, `docs/`, or any root file runs everything. diff --git a/oracle/parser/create_table.go b/oracle/parser/create_table.go index 2b4ed83dd..baeffcf9e 100644 --- a/oracle/parser/create_table.go +++ b/oracle/parser/create_table.go @@ -564,10 +564,10 @@ func (p *Parser) isTableConstraintStart() bool { case kwCONSTRAINT: next := p.peekNext() return !isOracleTypeToken(next.Type) - case kwPRIMARY: - return true - case kwFOREIGN: - return true + case kwPRIMARY, kwFOREIGN: + // PRIMARY and FOREIGN are not reserved: without KEY they start a + // column definition (engine-checked by TestOracleNonReservedKeywordsAsColumns). + return p.peekNext().Type == kwKEY case kwUNIQUE: // UNIQUE could be a column constraint when it follows a type, // but at the top level of the column list it's a table constraint diff --git a/oracle/parser/expr.go b/oracle/parser/expr.go index 54f001937..58ba04015 100644 --- a/oracle/parser/expr.go +++ b/oracle/parser/expr.go @@ -532,6 +532,22 @@ func (p *Parser) parsePrefix() (nodes.ExprNode, error) { func (p *Parser) parsePrimary() (nodes.ExprNode, error) { start := p.pos() + // A keyword Oracle does not reserve, followed by the legacy outer-join + // marker (+), is a column, whatever construct the word would otherwise + // open: WHERE cast(+) = u.a. A few words Oracle still reads as their + // construct, and it rejects the + inside it. Pseudo-columns stay + // pseudo-columns: Oracle reads SYSTIMESTAMP(+) as SYSTIMESTAMP(precision) + // and rejects it (ORA-30088) even when the table has such a column. + if p.cur.Type >= 2000 && !isOracleSQLReservedKeyword(p.cur) && + !isOraclePseudoColumnKeyword(p.cur.Type) && p.nextIsOuterJoinMarker() { + if keepsConstructBeforeOuterJoin(p.cur.Type) { + p.advance() // consume the keyword + p.advance() // consume ( + return nil, p.syntaxErrorAtCur() + } + return p.parseIdentExpr() + } + switch p.cur.Type { case tokICONST: tok := p.advance() @@ -583,9 +599,14 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { return p.parseParenExpr() case kwCASE: - return p.parseCaseExpr() + return p.parseCaseOrColumn() + // CAST, DECODE, and INTERVAL are not reserved; without the '(' or the + // string literal that opens the function or literal, they name a column. case kwCAST: + if p.peekNext().Type != '(' { + return p.parseIdentExpr() + } return p.parseCastExpr() case kwDATE, kwTIMESTAMP: @@ -595,6 +616,9 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { return p.parseIdentExpr() case kwDECODE: + if p.peekNext().Type != '(' { + return p.parseIdentExpr() + } return p.parseDecodeExpr() case kwEXISTS: @@ -633,6 +657,9 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { return p.parseJsonPathFunc("JSON_MERGEPATCH") case kwINTERVAL: + if p.peekNext().Type != tokSCONST { + return p.parseIdentExpr() + } return p.parseIntervalExpr() default: @@ -642,10 +669,12 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { } // Identifier — could be column ref, function call, or keyword-as-identifier. - // OFFSET is not reserved in Oracle and names columns in practice; it - // starts the row_limiting_clause only after a complete expression or - // table reference, never where an operand begins. - if isOracleClauseStarterKeyword(p.cur.Type) && p.cur.Type != kwOFFSET { + // A clause keyword Oracle reserves (FROM, WHERE, ...) ends the operand. + // The ones it does not reserve (FETCH, JOIN, MODEL, OFFSET, USING) name + // columns in practice and start their clauses only after a complete + // expression or table reference, never where an operand begins. + // TestOracleNonReservedKeywordsAsColumns checks them on the engine. + if isOracleClauseStarterKeyword(p.cur.Type) && isOracleSQLReservedKeyword(p.cur) { return nil, nil } if p.isIdentLike() { @@ -656,6 +685,222 @@ func (p *Parser) parsePrimary() (nodes.ExprNode, error) { } } +// parseCaseOrColumn parses CASE as Oracle reads it, since Oracle does not +// reserve the word: a CASE expression when WHEN follows it (searched CASE) or +// follows the selector expression after it (simple CASE, whose selector may +// be any expression, Boolean ones included in 23ai), and a column named CASE +// otherwise (ORDER BY case NULLS FIRST, WHERE case LIKEC 'x%', case + 1). +// decideCaseTokens settles every CASE of the range in one linear pass, so no +// expression is parsed twice to decide. +func (p *Parser) parseCaseOrColumn() (nodes.ExprNode, error) { + if p.caseKinds == nil { + p.caseKinds = decideCaseTokens(p.source, p.cur.Loc, p.lexer.end) + } + opensExpression, decided := p.caseKinds[p.cur.Loc] + if decided && !opensExpression { + return p.parseIdentExpr() + } + return p.parseCaseExpr() +} + +// decideCaseTokens lexes source[start:end] and reports, for each CASE token, +// whether it opens a CASE expression. A CASE opens one when the first token +// after it that can end an operand at its own depth is WHEN: the scan skips +// parenthesized groups and the CASE expressions that start after it, and +// stops at WHEN, THEN, ELSE, END, ')', ',', ';', and the reserved words that +// begin a clause. Deciding right to left with a jump table (stop) visits each +// token a constant number of times. +func decideCaseTokens(source string, start, end int) map[int]bool { + lexer := NewLexerRange(source, start, end) + var toks []Token + for { + tok := lexer.NextToken() + if tok.Type == tokEOF || lexer.Err != nil { + break + } + toks = append(toks, tok) + } + n := len(toks) + + // closing[i] is the index of the ')' or ']' that closes the '(' or '[' + // at i (a MODEL cell reference such as s['Mouse Pad', 1998]), or n. + closing := make([]int, n) + var open []int + for i, tok := range toks { + switch tok.Type { + case '(', '[': + closing[i] = n + open = append(open, i) + case ')', ']': + want := '(' + if tok.Type == ']' { + want = '[' + } + if len(open) > 0 && toks[open[len(open)-1]].Type == int(want) { + closing[open[len(open)-1]] = i + open = open[:len(open)-1] + } + } + } + + // stop[i] is the index of the first token from i on, at i's depth, that + // ends an operand; n stands for the end of the range. + stop := make([]int, n+1) + stop[n] = n + after := func(i int) int { + if i+1 >= n { + return n + } + return stop[i+1] + } + kinds := make(map[int]bool) + for i := n - 1; i >= 0; i-- { + tok := toks[i] + // A word after '.' is a name component (t.case, t.end), and END where + // an operand must start is a column named END (1 + end, THEN end): + // neither opens nor closes anything for the scan. + plain := i > 0 && (toks[i-1].Type == '.' || + tok.Type == kwEND && expectsOperandAfter(toks[i-1])) + switch { + case plain: + stop[i] = after(i) + case isCaseScanStop(tok): + stop[i] = i + case tok.Type == '(' || tok.Type == '[': + stop[i] = after(closing[i]) + case tok.Type == kwCASE: + // s is where the scan for WHEN stops; -1 when the token after + // CASE cannot start a selector, so CASE is a column (case IS NULL). + s := -1 + switch { + case i+1 >= n: + case toks[i+1].Type == kwWHEN: + s = i + 1 + case !canStartCaseSelector(toks[i+1]): + default: + s = after(i) + } + if s >= 0 && s < n && toks[s].Type == kwWHEN { + kinds[tok.Loc] = true + if e := caseExprEnd(toks, stop, s); e >= 0 { + stop[i] = after(e) + } else { + stop[i] = s + } + } else { + kinds[tok.Loc] = false + stop[i] = after(i) + } + default: + stop[i] = after(i) + } + } + return kinds +} + +// caseExprEnd follows WHEN ... THEN ... [WHEN ... THEN ...]... [ELSE ...] END +// from the WHEN at w through the stop table and returns the index of END, or +// -1 when the CASE expression is malformed. +func caseExprEnd(toks []Token, stop []int, w int) int { + n := len(toks) + next := func(i int) int { + if i+1 >= n { + return n + } + return stop[i+1] + } + for { + then := next(w) + if then >= n || toks[then].Type != kwTHEN { + return -1 + } + r := next(then) + if r >= n { + return -1 + } + switch toks[r].Type { + case kwWHEN: + w = r + case kwELSE: + if e := next(r); e < n && toks[e].Type == kwEND { + return e + } + return -1 + case kwEND: + return r + default: + return -1 + } + } +} + +// canStartCaseSelector reports whether tok can begin the selector of a simple +// CASE. Operators and keywords that only follow an operand (=, IS, IN, AND, +// ...), '[' (case[1] is a MODEL cell of a measure named CASE), and the clause +// words that end one cannot; END can, as a column named END, which Oracle +// accepts there. +func canStartCaseSelector(tok Token) bool { + switch tok.Type { + case '=', '<', '>', '*', '/', '.', '[', tokLESSEQ, tokGREATEQ, tokNOTEQ, + tokCONCAT, tokEXPON, kwIS, kwIN, kwLIKE, kwBETWEEN, kwAND, kwOR, + kwASC, kwDESC: + return false + case kwEND: + return true + } + return !isCaseScanStop(tok) +} + +// expectsOperandAfter reports whether an operand must follow tok, so an END +// right after it is a column named END rather than the end of a CASE. Only +// tokens that work at expression level matter: an END inside parentheses is +// skipped with its group. The unary PRIOR and CONNECT_BY_ROOT, ESCAPE, OF +// (MEMBER OF, SUBMULTISET OF), and ZONE (AT TIME ZONE) all take an operand. +func expectsOperandAfter(tok Token) bool { + switch tok.Type { + case '+', '-', '*', '/', '=', '<', '>', '(', ',', '.', + tokCONCAT, tokEXPON, tokLESSEQ, tokGREATEQ, tokNOTEQ, + kwCASE, kwWHEN, kwTHEN, kwELSE, kwAND, kwOR, kwNOT, + kwPRIOR, kwCONNECT_BY_ROOT, kwESCAPE, kwOF, kwZONE, + kwLIKE, kwLIKEC, kwLIKE2, kwLIKE4, kwBETWEEN, + kwSELECT, kwWHERE, kwBY, kwHAVING, kwON, kwSET, kwRETURN: + return true + } + return false +} + +// isCaseScanStop reports the tokens that end an operand for decideCaseTokens. +func isCaseScanStop(tok Token) bool { + switch tok.Type { + case kwWHEN, kwTHEN, kwELSE, kwEND, ')', ']', ',', ';', + kwFROM, kwWHERE, kwGROUP, kwHAVING, kwORDER, kwUNION, kwINTERSECT, + kwMINUS, kwINTO, kwAS, kwCONNECT, kwSTART, kwSET, kwVALUES, kwFOR: + return true + } + return false +} + +// keepsConstructBeforeOuterJoin reports the non-reserved keywords Oracle +// parses as their own construct even when (+) follows, rejecting word(+) +// (ORA-00931, ORA-00936), so they cannot name an outer-joined column +// unquoted. Engine-checked by TestOracleNonReservedKeywordsAsColumns. +func keepsConstructBeforeOuterJoin(tokenType int) bool { + switch tokenType { + case kwJSON, kwJSON_ARRAY, kwJSON_EXISTS, kwJSON_MERGEPATCH, kwJSON_OBJECT, + kwJSON_QUERY, kwJSON_TABLE, kwJSON_VALUE, kwTREAT, + kwXMLELEMENT, kwXMLFOREST, kwXMLROOT: + return true + } + return false +} + +// nextIsOuterJoinMarker reports whether the tokens after the current one are +// the legacy outer-join marker (+). +func (p *Parser) nextIsOuterJoinMarker() bool { + toks := p.peekAhead(3) + return toks[0].Type == '(' && toks[1].Type == '+' && toks[2].Type == ')' +} + // parseDateTimeLiteral parses ANSI datetime literals such as DATE '2020-01-01'. func (p *Parser) parseDateTimeLiteral() (nodes.ExprNode, error) { start := p.pos() @@ -1615,23 +1860,31 @@ func (p *Parser) parseCaseExpr() (nodes.ExprNode, error) { start := p.pos() p.advance() // consume CASE - ce := &nodes.CaseExpr{ - Whens: &nodes.List{}, - Loc: nodes.Loc{Start: start}, - } - // Simple CASE: CASE expr WHEN ... // Searched CASE: CASE WHEN ... + var arg nodes.ExprNode if p.cur.Type != kwWHEN { var parseErr730 error - ce.Arg, parseErr730 = p.parseExpr() + arg, parseErr730 = p.parseExpr() if parseErr730 != nil { return nil, parseErr730 } - if ce.Arg == nil { + if arg == nil { return nil, p.syntaxErrorAtCur() } } + return p.parseCaseWhens(start, arg) +} + +// parseCaseWhens parses the WHEN ... [ELSE ...] END of a CASE expression that +// starts at start, after CASE and the selector arg (nil for a searched CASE) +// have been consumed. +func (p *Parser) parseCaseWhens(start int, arg nodes.ExprNode) (nodes.ExprNode, error) { + ce := &nodes.CaseExpr{ + Arg: arg, + Whens: &nodes.List{}, + Loc: nodes.Loc{Start: start}, + } for p.cur.Type == kwWHEN { whenStart := p.pos() diff --git a/oracle/parser/keyword_column_oracle_test.go b/oracle/parser/keyword_column_oracle_test.go new file mode 100644 index 000000000..eb9bd047a --- /dev/null +++ b/oracle/parser/keyword_column_oracle_test.go @@ -0,0 +1,145 @@ +package parser + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +var oracleKeywordColumnGaps = filepath.Join("testdata", "coverage", "oracle_keyword_column_gaps.tsv") + +// keywordColumnContexts reference a column named by the keyword after it is +// defined; %[1]s is the table and %[2]s the keyword. +var keywordColumnContexts = []struct { + name string + sql string +}{ + {"select list", "SELECT %[2]s FROM %[1]s"}, + {"where", "SELECT a FROM %[1]s WHERE %[2]s IS NULL"}, + {"not in", "SELECT a FROM %[1]s WHERE %[2]s NOT IN (1, 2)"}, + {"order by", "SELECT a FROM %[1]s ORDER BY %[2]s"}, + {"update", "UPDATE %[1]s SET %[2]s = NULL WHERE %[2]s IS NULL"}, + {"insert columns", "INSERT INTO %[1]s (%[2]s) VALUES (1)"}, + {"index", "CREATE INDEX %[1]s_i ON %[1]s (%[2]s)"}, + {"outer join marker", "SELECT d.dummy FROM dual d, %[1]s WHERE %[2]s(+) = LENGTH(d.dummy)"}, + {"case selector", "SELECT CASE %[2]s WHEN 1 THEN 2 END FROM %[1]s"}, + {"qualified case selector", "SELECT CASE k.%[2]s WHEN 1 THEN 2 END FROM %[1]s k"}, + {"case selector operand", "SELECT CASE 1 + %[2]s WHEN 2 THEN 3 END FROM %[1]s"}, +} + +// TestOracleNonReservedKeywordsAsColumns checks every word omni lexes as a +// keyword that Oracle does not reserve (V$RESERVED_WORDS) as a column name: +// defined by CREATE TABLE, then referenced from a select list, WHERE +// predicates (IS NULL, NOT IN), ORDER BY, UPDATE, an INSERT column list, an +// index, with the legacy outer-join marker (+), and in a simple CASE +// selector (alone, qualified, and as an operand). omni must +// accept or reject each statement exactly as Oracle does. The keyword +// manifest's column_name context covers only the definition; OFFSET, FETCH, +// JOIN, MODEL, and USING failed in the references alone. +// +// Pseudo-column keywords are left out: a reference resolves to the +// pseudo-column (SYSTIMESTAMP compares as a TIMESTAMP), not to the column. +// Disagreements still open are recorded in oracle_keyword_column_gaps.tsv; +// the list only shrinks, and -update regenerates it. +func TestOracleNonReservedKeywordsAsColumns(t *testing.T) { + o := startOracleDB(t) + + reserved := map[string]bool{} + rows, err := o.adminDB.QueryContext(o.ctx, ` +SELECT keyword FROM v$reserved_words + WHERE keyword IS NOT NULL + AND (reserved = 'Y' OR res_type = 'Y' OR res_attr = 'Y' OR res_semi = 'Y')`) + if err != nil { + t.Fatalf("query v$reserved_words: %v", err) + } + for rows.Next() { + var keyword string + if err := rows.Scan(&keyword); err != nil { + t.Fatalf("scan v$reserved_words: %v", err) + } + reserved[keyword] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate v$reserved_words: %v", err) + } + rows.Close() + + var words []string + for word, tokenType := range oracleKeywords { + if reserved[word] || isOraclePseudoColumnKeyword(tokenType) { + continue + } + words = append(words, word) + } + sort.Strings(words) + if len(words) < 200 { + t.Fatalf("checked %d non-reserved keywords, want at least 200", len(words)) + } + + known := map[string]coverageRow{} + if !*update { + gaps, _ := readCoverageTSV(t, oracleKeywordColumnGaps) + for _, row := range gaps { + known[row.Key] = row + } + } + + run := fmt.Sprintf("%d", time.Now().UnixNano()%1_000_000_000) + var gaps []string + var newGaps, fixed, checks int + // check records a disagreement with the table renamed to t, so the + // manifest does not change from run to run. + check := func(word, context, table, sqlText string, oracleAccepts bool) { + checks++ + _, omniErr := Parse(sqlText) + key := word + ":" + context + _, wasKnown := known[key] + if oracleAccepts == (omniErr == nil) { + if wasKnown { + fixed++ + t.Errorf("FIXED %s now matches Oracle: remove it from %s", key, oracleKeywordColumnGaps) + } + return + } + gaps = append(gaps, fmt.Sprintf("%s\t%v\t%v\t%s", key, oracleAccepts, omniErr == nil, strings.ReplaceAll(sqlText, table, "t"))) + if !wasKnown && !*update { + newGaps++ + t.Errorf("NEW GAP %s: oracle_accepts=%v omni_err=%v sql=%q", key, oracleAccepts, omniErr, sqlText) + } + } + + for i, word := range words { + table := fmt.Sprintf("omni_kw_%s_%d", run, i) + create := fmt.Sprintf("CREATE TABLE %s (a NUMBER, %s NUMBER)", table, word) + _, createErr := o.db.ExecContext(o.ctx, create) + check(word, "column definition", table, create, createErr == nil) + if createErr != nil { + continue + } + for _, c := range keywordColumnContexts { + sqlText := fmt.Sprintf(c.sql, table, word) + check(word, c.name, table, sqlText, oracleParseOnly(o.ctx, o.db, sqlText) == nil) + } + if _, err := o.db.ExecContext(o.ctx, "DROP TABLE "+table+" PURGE"); err != nil { + t.Logf("drop %s: %v", table, err) + } + } + + t.Logf("non-reserved keywords as columns: %d words, %d checks, %d disagree with Oracle (%d new, %d fixed)", + len(words), checks, len(gaps), newGaps, fixed) + + if *update { + content := "case\toracle_accepts\tomni_accepts\tsql\n" + strings.Join(gaps, "\n") + if len(gaps) > 0 { + content += "\n" + } + if err := os.WriteFile(oracleKeywordColumnGaps, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", oracleKeywordColumnGaps, err) + } + t.Logf("wrote %s (%d gaps)", oracleKeywordColumnGaps, len(gaps)) + } +} diff --git a/oracle/parser/keywords.go b/oracle/parser/keywords.go index 85bd6634a..1ab4bbfe8 100644 --- a/oracle/parser/keywords.go +++ b/oracle/parser/keywords.go @@ -140,7 +140,7 @@ func isOracleSQLReservedKeyword(tok Token) bool { kwINTO, kwIS, kwLEVEL, kwLIKE, kwLOCK, kwLONG, kwMAXEXTENTS, kwMINUS, kwMLSLABEL, kwMODE, kwMODIFY, kwNESTED_TABLE_ID, kwNOAUDIT, kwNOCOMPRESS, kwNOT, kwNOWAIT, kwNULL, kwNUMBER, kwOF, kwOFFLINE, kwON, kwONLINE, - kwOPTION, kwOR, kwORDER, kwPCTFREE, kwPRIOR, kwPRIVILEGES, + kwOPTION, kwOR, kwORDER, kwPCTFREE, kwPRIOR, kwPUBLIC, kwRAW, kwRENAME, kwRESOURCE, kwREVOKE, kwROW, kwROWID, kwROWNUM, kwROWS, kwSELECT, kwSESSION, kwSET, kwSHARE, kwSIZE, kwSMALLINT, kwSTART, kwSUCCESSFUL, kwSYNONYM, kwSYSDATE, kwTABLE, kwTHEN, diff --git a/oracle/parser/oracle_helper_test.go b/oracle/parser/oracle_helper_test.go index 3ad830d59..53a7f5124 100644 --- a/oracle/parser/oracle_helper_test.go +++ b/oracle/parser/oracle_helper_test.go @@ -37,7 +37,7 @@ func startOracleDB(t *testing.T) *oracleDB { ctx := context.Background() req := testcontainers.ContainerRequest{ - Image: "gvenzl/oracle-free:23-slim-faststart", + Image: "gvenzl/oracle-free:23.26.3-slim-faststart", ExposedPorts: []string{"1521/tcp"}, Env: map[string]string{ "ORACLE_PASSWORD": "testpass", @@ -224,6 +224,8 @@ func setupOracleSchema(o *oracleDB) { `CREATE TABLE j_purchaseorder (po_document CLOB)`, // OFFSET and PARTITION are not reserved and name columns in practice. `CREATE TABLE t_offset (a NUMBER, offset NUMBER, partition NUMBER)`, + // CASE, NULLS, and LIKEC are not reserved either. + `CREATE TABLE t_kwcol (a NUMBER, case NUMBER, nulls NUMBER, likec VARCHAR2(10))`, } for _, ddl := range ddls { diff --git a/oracle/parser/parser.go b/oracle/parser/parser.go index 1eed0648f..ecc2e1540 100644 --- a/oracle/parser/parser.go +++ b/oracle/parser/parser.go @@ -22,6 +22,11 @@ type Parser struct { prev Token // previous token (for error reporting) nextBuf Token // buffered next token for 2-token lookahead hasNext bool // whether nextBuf is valid + + // caseKinds tells, for each CASE token from the first one parsePrimary + // met to the end of the range, whether it opens a CASE expression (true) + // or names a column (false). decideCaseTokens fills it in one pass. + caseKinds map[int]bool } // Parse parses a SQL string into an AST list. @@ -138,11 +143,14 @@ func (p *Parser) isIncompleteStatementEnd(stmt nodes.StmtNode) bool { return true } + // Keywords Oracle does not reserve (CONTENT, JOIN, USING) are left out: a + // statement can end with a column of that name, and a clause they open + // without its operand ("FROM t JOIN") fails in the clause's own parser. switch stmt.(type) { case *nodes.SelectStmt: switch p.prev.Type { - case kwAND, kwAS, kwBY, kwCONTENT, kwFROM, kwGROUP, kwHAVING, - kwIS, kwJOIN, kwNOT, kwON, kwOR, kwORDER, kwTHEN, kwUNION, + case kwAND, kwAS, kwBY, kwFROM, kwGROUP, kwHAVING, + kwIS, kwNOT, kwON, kwOR, kwORDER, kwTHEN, kwUNION, kwWHERE: return true } @@ -154,7 +162,7 @@ func (p *Parser) isIncompleteStatementEnd(stmt nodes.StmtNode) bool { return p.prev.Type == kwFROM || p.prev.Type == kwWHERE case *nodes.MergeStmt: switch p.prev.Type { - case kwINTO, kwON, kwSET, kwTHEN, kwUSING, kwWHERE: + case kwINTO, kwON, kwSET, kwTHEN, kwWHERE: return true } case *nodes.CreateTableStmt: @@ -337,6 +345,22 @@ func (p *Parser) peekNext() Token { return p.nextBuf } +// peekAhead returns the n tokens after cur without consuming them; tokens +// past the end of input are tokEOF. It lexes a copy of the lexer, so the +// parser's position is untouched. +func (p *Parser) peekAhead(n int) []Token { + toks := make([]Token, 0, n) + if n == 0 { + return toks + } + toks = append(toks, p.peekNext()) + lexer := *p.lexer + for len(toks) < n { + toks = append(toks, lexer.NextToken()) + } + return toks +} + // peek returns the current token without consuming it. func (p *Parser) peek() Token { return p.cur diff --git a/oracle/parser/plsql_dictionary_oracle_test.go b/oracle/parser/plsql_dictionary_oracle_test.go new file mode 100644 index 000000000..2f15756ad --- /dev/null +++ b/oracle/parser/plsql_dictionary_oracle_test.go @@ -0,0 +1,285 @@ +package parser + +import ( + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +var update = flag.Bool("update", false, "rewrite the known-gap manifests of the Oracle dictionary tests from the current results") + +// oracleDictionaryVersion is the release of the pinned Oracle image whose +// dictionary the known-gap manifests were recorded on. +const oracleDictionaryVersion = "23.26.3.0.0" + +var plsqlDictionaryManifest = filepath.Join("testdata", "coverage", "plsql_dictionary_known_failures.tsv") + +// dictionaryUnit is one PL/SQL unit read from DBA_SOURCE. +type dictionaryUnit struct { + key string // OWNER.NAME:TYPE + kind string // DBA_SOURCE type + text string // CREATE OR REPLACE followed by the stored source +} + +// TestOraclePLSQLDictionary parses the PL/SQL that ships with Oracle: every +// unwrapped, VALID procedure, function, package, package body, type body, and +// trigger of an Oracle-maintained schema in the pinned image, about 875 units +// of code the engine compiled. Each unit goes through Split and ParseRange, +// as Bytebase runs a script, and must come out as one segment that parses. +// +// The units that fail today are recorded with a category, the line:column of +// the first error, and its message in plsql_dictionary_known_failures.tsv. A +// unit that starts failing is a regression, and so is a recorded unit that +// now fails earlier; one that fails later or parses must be re-recorded or +// leave the manifest, so the list only shrinks. Regenerate it with -update and review the diff. The +// source text itself is Oracle's and is read at test time, never committed. +func TestOraclePLSQLDictionary(t *testing.T) { + units := loadOracleDictionaryUnits(t) + if len(units) < 800 { + t.Fatalf("read %d dictionary units, want at least 800: the DBA_SOURCE query no longer matches the image", len(units)) + } + + known := map[string]coverageRow{} + if !*update { + rows, _ := readCoverageTSV(t, plsqlDictionaryManifest) + for _, row := range rows { + known[row.Key] = row + } + } + + type failure struct { + unit dictionaryUnit + category string + where string + message string + } + var failures []failure + byCategory := map[string]int{} + var newFailures, changed, fixed int + for _, u := range units { + category, where, message, ok := parseDictionaryUnit(u) + if ok { + if _, wasKnown := known[u.key]; wasKnown { + fixed++ + t.Errorf("FIXED %s now parses: remove it from %s", u.key, plsqlDictionaryManifest) + } + continue + } + failures = append(failures, failure{unit: u, category: category, where: where, message: message}) + byCategory[category]++ + if *update { + continue + } + row, wasKnown := known[u.key] + if !wasKnown { + newFailures++ + t.Errorf("NEW FAILURE %s at %s: %s (%s)", u.key, where, message, category) + continue + } + // A known unit must still fail at the recorded place for the + // recorded reason: failing earlier is a regression hidden inside + // it, failing later is progress the manifest has to record. + if row.Fields["position"] != where || row.Fields["error"] != message { + changed++ + direction := "later (progress)" + if lineColBefore(where, row.Fields["position"]) { + direction = "EARLIER (regression)" + } + t.Errorf("CHANGED %s now fails %s: at %s: %s (%s), recorded at %s: %s. Rerun with -update if intended.", + u.key, direction, where, message, category, row.Fields["position"], row.Fields["error"]) + } + } + + t.Logf("Oracle %s dictionary: %d units, %d fail (%.1f%%), %d new, %d changed, %d fixed", + oracleDictionaryVersion, len(units), len(failures), + 100*float64(len(failures))/float64(len(units)), newFailures, changed, fixed) + var categories []string + for c := range byCategory { + categories = append(categories, c) + } + sort.Slice(categories, func(i, j int) bool { + if byCategory[categories[i]] != byCategory[categories[j]] { + return byCategory[categories[i]] > byCategory[categories[j]] + } + return categories[i] < categories[j] + }) + for _, c := range categories { + t.Logf(" %4d %s", byCategory[c], c) + } + + if *update { + var b strings.Builder + b.WriteString("unit\tcategory\tposition\terror\n") + for _, f := range failures { + fmt.Fprintf(&b, "%s\t%s\t%s\t%s\n", f.unit.key, f.category, f.where, f.message) + } + if err := os.WriteFile(plsqlDictionaryManifest, []byte(b.String()), 0o644); err != nil { + t.Fatalf("write %s: %v", plsqlDictionaryManifest, err) + } + t.Logf("wrote %s (%d units)", plsqlDictionaryManifest, len(failures)) + } +} + +// loadOracleDictionaryUnits reads the units from the container started by +// startOracleDB, after checking that it runs the release the manifest was +// recorded on. It does not honor ORACLE_PARSER_REF_* DSNs: the manifest +// describes the pinned image's dictionary, not another server's. +func loadOracleDictionaryUnits(t *testing.T) []dictionaryUnit { + t.Helper() + o := startOracleDB(t) + + var version string + if err := o.adminDB.QueryRowContext(o.ctx, + `SELECT version_full FROM product_component_version WHERE product LIKE 'Oracle%'`).Scan(&version); err != nil { + t.Fatalf("read Oracle version: %v", err) + } + if version != oracleDictionaryVersion { + t.Fatalf("Oracle %s is running; the dictionary manifests were recorded on %s. Rerun with -update and review the drift.", + version, oracleDictionaryVersion) + } + + // The dba_objects and dba_users filters are semi-joins: a join repeats + // rows for objects listed in more than one container. + rows, err := o.adminDB.QueryContext(o.ctx, ` +SELECT s.owner, s.name, s.type, s.line, s.text + FROM dba_source s + WHERE s.type IN ('PROCEDURE', 'FUNCTION', 'PACKAGE', 'PACKAGE BODY', 'TYPE BODY', 'TRIGGER') + AND s.owner IN (SELECT username FROM dba_users WHERE oracle_maintained = 'Y') + AND (s.owner, s.name, s.type) IN + (SELECT owner, object_name, object_type FROM dba_objects WHERE status = 'VALID') + AND (s.owner, s.name, s.type) NOT IN + (SELECT owner, name, type FROM dba_source WHERE line = 1 AND LOWER(text) LIKE '%wrapped%') + ORDER BY s.owner, s.type, s.name, s.line`) + if err != nil { + t.Fatalf("query DBA_SOURCE: %v", err) + } + defer rows.Close() + + var units []dictionaryUnit + var cur *dictionaryUnit + var body strings.Builder + lastLine := 0 + flush := func() { + if cur != nil { + cur.text = "CREATE OR REPLACE " + body.String() + units = append(units, *cur) + } + body.Reset() + } + for rows.Next() { + var owner, name, kind, text string + var line int + if err := rows.Scan(&owner, &name, &kind, &line, &text); err != nil { + t.Fatalf("scan DBA_SOURCE: %v", err) + } + key := owner + "." + name + ":" + kind + if cur == nil || cur.key != key { + flush() + cur = &dictionaryUnit{key: key, kind: kind} + lastLine = 0 + } + if line == lastLine { + continue + } + lastLine = line + body.WriteString(text) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate DBA_SOURCE: %v", err) + } + flush() + return units +} + +// parseDictionaryUnit splits and parses one unit. On failure it returns the +// category of the construct at the error, the error's line:column within the +// unit, and the message. +func parseDictionaryUnit(u dictionaryUnit) (category, where, message string, ok bool) { + var segments []Segment + for _, seg := range Split(u.text) { + if !seg.Empty() { + segments = append(segments, seg) + } + } + if len(segments) != 1 { + return classifyDictionaryFailure(u, ""), "1:1", fmt.Sprintf("split into %d segments", len(segments)), false + } + _, err := ParseRange(u.text, segments[0].ByteStart, segments[0].ByteEnd) + if err == nil { + return "", "", "", true + } + var pe *ParseError + if !errors.As(err, &pe) { + return "unclassified", "1:1", err.Error(), false + } + pos := pe.Position + if pos < 0 || pos > len(u.text) { + pos = 0 + } + lineStart := strings.LastIndexByte(u.text[:pos], '\n') + 1 + lineEnd := strings.IndexByte(u.text[pos:], '\n') + if lineEnd < 0 { + lineEnd = len(u.text) + } else { + lineEnd += pos + } + line := strings.Count(u.text[:pos], "\n") + 1 + where = fmt.Sprintf("%d:%d", line, len([]rune(u.text[lineStart:pos]))+1) + return classifyDictionaryFailure(u, u.text[lineStart:lineEnd]), where, pe.Message, false +} + +// dictionaryCategories name the construct behind a failure from the source +// line the parser stopped on. First match wins. +var dictionaryCategories = []struct { + name string + kinds string // DBA_SOURCE types the rule applies to; empty for all + pattern *regexp.Regexp +}{ + {"conditional compilation ($IF/$END)", "", regexp.MustCompile(`(?i)^\s*\$|\$(if|then|elsif|else|end|error)\b|\$\$`)}, + {"ACCESSIBLE BY clause", "", regexp.MustCompile(`(?i)accessible\s+by`)}, + {"CHARACTER SET ANY_CS / %CHARSET", "", regexp.MustCompile(`(?i)character\s+set`)}, + {"PIPELINED / AGGREGATE USING implementation type", "", regexp.MustCompile(`(?i)(pipelined|aggregate|parallel_enable)\s+using|^\s*using\b`)}, + {"PARALLEL_ENABLE (PARTITION ...)", "", regexp.MustCompile(`(?i)parallel_enable\s*\(`)}, + {"package initialization section", "PACKAGE BODY", regexp.MustCompile(`(?i)^\s*begin\b`)}, + {"REF type", "", regexp.MustCompile(`(?i)\bref\s`)}, + {"Oracle-internal syntax (PRAGMA INTERFACE, ... parameters)", "", regexp.MustCompile(`(?i)pragma\s+interface|\.\.\.`)}, + {"JSON function clause", "", regexp.MustCompile(`(?i)\breturning\b|json_|^\s*key\b`)}, + {"trigger CALL with arguments", "TRIGGER", regexp.MustCompile(`(?i)^\s*call\b|\(\s*:(new|old)\.`)}, + {"qualified expression (=>)", "", regexp.MustCompile(`=>`)}, + {"type length expression", "", regexp.MustCompile(`(?i)\(\s*[a-z_0-9]*\s*[*+-]`)}, + {"SQL%BULK_EXCEPTIONS", "", regexp.MustCompile(`(?i)sql%bulk`)}, + {"type body method modifiers", "TYPE BODY", regexp.MustCompile(`(?i)\b(final|instantiable|overriding)\b`)}, +} + +func classifyDictionaryFailure(u dictionaryUnit, line string) string { + if line == "" && strings.Contains(strings.ToUpper(u.text), "$END") { + return "conditional compilation ($IF/$END)" + } + for _, c := range dictionaryCategories { + if c.kinds != "" && c.kinds != u.kind { + continue + } + if c.pattern.MatchString(line) { + return c.name + } + } + return "unclassified" +} + +// lineColBefore reports whether position a ("line:col") comes before b. +func lineColBefore(a, b string) bool { + var al, ac, bl, bc int + if _, err := fmt.Sscanf(a, "%d:%d", &al, &ac); err != nil { + return false + } + if _, err := fmt.Sscanf(b, "%d:%d", &bl, &bc); err != nil { + return false + } + return al < bl || (al == bl && ac < bc) +} diff --git a/oracle/parser/select.go b/oracle/parser/select.go index a32ba0950..3d8bccab8 100644 --- a/oracle/parser/select.go +++ b/oracle/parser/select.go @@ -387,12 +387,13 @@ func (p *Parser) parseSelectList() (*nodes.List, error) { } // isSelectListTerminator reports whether the current token ends the select -// list where an item would start. OFFSET is absent: an item can be a column -// named OFFSET, and the row_limiting_clause never follows an empty list. +// list where an item would start. FETCH and OFFSET are absent: an item can be +// a column of that name, and the row_limiting_clause never follows an empty +// list. func (p *Parser) isSelectListTerminator() bool { switch p.cur.Type { case tokEOF, kwFROM, kwWHERE, kwGROUP, kwHAVING, kwORDER, kwUNION, - kwINTERSECT, kwMINUS, kwFOR, kwCONNECT, kwSTART, kwFETCH: + kwINTERSECT, kwMINUS, kwFOR, kwCONNECT, kwSTART: return true default: return false diff --git a/oracle/parser/select_test.go b/oracle/parser/select_test.go index a8acf914c..e99a68eb9 100644 --- a/oracle/parser/select_test.go +++ b/oracle/parser/select_test.go @@ -1,7 +1,10 @@ package parser import ( + "errors" + "strings" "testing" + "time" "github.com/bytebase/omni/oracle/ast" ) @@ -412,3 +415,136 @@ func TestParseOffsetColumnNameAST(t *testing.T) { t.Fatalf("row limiting clause missing: %s", ast.NodeToString(sel)) } } + +// TestParseCaseBooleanSelector: CASE opens an expression when WHEN follows it +// or follows an expression after it; the selector may start with NOT, NULLS, +// or LIKEC and may be a Boolean comparison. Otherwise CASE is a column: +// case NOT IN (...), ORDER BY case NULLS FIRST, case LIKEC '...', and a column +// with an alias. Oracle 23ai accepts all of these. +func TestParseCaseBooleanSelector(t *testing.T) { + result := ParseAndCheck(t, "SELECT CASE NOT TRUE WHEN TRUE THEN 1 ELSE 0 END FROM t") + target := result.Items[0].(*ast.RawStmt).Stmt.(*ast.SelectStmt).TargetList.Items[0].(*ast.ResTarget) + if _, ok := target.Expr.(*ast.CaseExpr); !ok { + t.Fatalf("target = %T, want *ast.CaseExpr", target.Expr) + } + for _, sql := range []string{ + "SELECT CASE nulls WHEN 1 THEN 2 END FROM t", + "SELECT CASE nulls + 1 WHEN 1 THEN 2 END FROM t", + "SELECT CASE likec WHEN 'x' THEN 2 END FROM t", + "SELECT CASE like2 || 'a' WHEN 'xa' THEN 2 END FROM t", + "SELECT CASE likec = 'x' WHEN TRUE THEN 1 ELSE 0 END FROM t", + "SELECT CASE likec IN ('x', 'y') WHEN TRUE THEN 1 ELSE 0 END FROM t", + "SELECT CASE case IS NULL WHEN TRUE THEN 1 ELSE 0 END FROM t", + "SELECT CASE end WHEN 1 THEN 2 END FROM t", + "SELECT CASE t.end WHEN 1 THEN 2 END FROM t", + "SELECT CASE 1 + end WHEN 2 THEN 3 END FROM t", + "SELECT CASE t.case WHEN 1 THEN 2 END FROM t", + "SELECT CASE a WHEN 1 THEN end ELSE 0 END FROM t", + "SELECT CASE CONNECT_BY_ROOT end WHEN 1 THEN 2 END FROM t CONNECT BY PRIOR id = pid", + "SELECT CASE PRIOR end WHEN 1 THEN 2 END FROM t CONNECT BY PRIOR id = pid", + "SELECT CASE a LIKE 'x' ESCAPE end WHEN TRUE THEN 1 END FROM t", + "SELECT CASE sales['Mouse Pad', 1998] WHEN 1 THEN 2 END FROM t", + "SELECT CASE case[1] WHEN 10 THEN 20 END FROM t", + } { + t.Run(sql, func(t *testing.T) { + result := ParseAndCheck(t, sql) + target := result.Items[0].(*ast.RawStmt).Stmt.(*ast.SelectStmt).TargetList.Items[0].(*ast.ResTarget) + if _, ok := target.Expr.(*ast.CaseExpr); !ok { + t.Fatalf("target = %T, want *ast.CaseExpr", target.Expr) + } + }) + } + for _, sql := range []string{ + "SELECT a FROM t WHERE case NOT IN (1, 2)", + "SELECT a FROM t WHERE case NOT BETWEEN 1 AND 2", + "SELECT a FROM t WHERE case NOT LIKE 'x%'", + "SELECT a FROM t ORDER BY case NULLS FIRST", + "SELECT a FROM t ORDER BY case DESC NULLS LAST", + "SELECT a FROM t WHERE case LIKEC '1%'", + "SELECT case a2 FROM t", + } { + t.Run(sql, func(t *testing.T) { + ParseAndCheck(t, sql) + }) + } +} + +// TestParseKeywordColumnOuterJoin: a non-reserved keyword followed by the +// legacy outer-join marker (+) is a column, except for the words Oracle still +// reads as their construct (it rejects JSON_VALUE(+) and XMLELEMENT(+)) and +// for pseudo-columns (SYSTIMESTAMP(+) is ORA-30088 even with such a column). +func TestParseKeywordColumnOuterJoin(t *testing.T) { + for _, word := range []string{"cast", "decode", "case", "interval", "xmlagg"} { + sql := "SELECT 1 FROM t, u WHERE " + word + "(+) = u.a" + t.Run(sql, func(t *testing.T) { + result := ParseAndCheck(t, sql) + where := result.Items[0].(*ast.RawStmt).Stmt.(*ast.SelectStmt).WhereClause.(*ast.BinaryExpr) + col, ok := where.Left.(*ast.ColumnRef) + if !ok || !col.OuterJoin { + t.Fatalf("left = %s, want an outer-joined column", ast.NodeToString(where.Left)) + } + if violations := CheckLocations(t, sql); len(violations) > 0 { + t.Fatalf("Loc violations: %v", violations) + } + }) + } + for _, word := range []string{"json_value", "xmlelement", "treat", "json", "systimestamp"} { + ParseShouldFail(t, "SELECT 1 FROM t, u WHERE "+word+"(+) = u.a") + } + // A qualified reference is a column even for a pseudo-column word. + ParseAndCheck(t, "SELECT 1 FROM t, u WHERE t.systimestamp(+) = u.a") +} + +// TestParseCaseExpressionErrorPosition: deciding between a CASE expression +// and a column named CASE probes the selector without consuming it, so an +// error inside a real CASE expression is still reported where it occurs. +func TestParseCaseExpressionErrorPosition(t *testing.T) { + tests := []struct { + sql string + near string + }{ + {"SELECT CASE x WHEN 1 THEN (1 + ) END FROM t", ")"}, + {"SELECT CASE WHEN a = 1 THEN 1 FROM t", "FROM"}, + {"SELECT CASE likec = 'x' WHEN TRUE THEN (1 + ) END FROM t", ")"}, + } + for _, tt := range tests { + t.Run(tt.sql, func(t *testing.T) { + _, err := Parse(tt.sql) + var pe *ParseError + if !errors.As(err, &pe) { + t.Fatalf("Parse error = %v, want a ParseError", err) + } + if want := strings.Index(tt.sql, tt.near); pe.Position != want { + t.Fatalf("error at %d (%q), want %d (%q)", pe.Position, tt.sql[pe.Position:], want, tt.near) + } + }) + } +} + +// TestParseCaseDecisionStaysLinear guards how CASE is told apart from a column +// named CASE: deciding by parsing what follows each CASE made nested simple +// CASE selectors exponential and long sums of CASE columns quadratic. Each +// shape below takes tens of milliseconds when linear, and seconds to minutes +// otherwise. +func TestParseCaseDecisionStaysLinear(t *testing.T) { + const depth = 4000 + selector := "1" + column := "'1%'" + for i := 0; i < depth; i++ { + selector = "CASE " + selector + " WHEN 1 THEN 1 END" + column = "case LIKEC (" + column + ")" + } + for name, sql := range map[string]string{ + "nested CASE selectors": "SELECT " + selector + " FROM dual", + "nested CASE columns": "SELECT a FROM t WHERE " + column, + "sum of 20000 CASE column terms": "SELECT " + strings.Repeat("case + ", 20000) + "1 FROM t", + } { + start := time.Now() + if _, err := Parse(sql); err != nil { + t.Fatalf("%s: %v", name, err) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Fatalf("%s took %s", name, elapsed) + } + } +} diff --git a/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv b/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv new file mode 100644 index 000000000..3489b3aa0 --- /dev/null +++ b/oracle/parser/testdata/coverage/oracle_keyword_column_gaps.tsv @@ -0,0 +1,3 @@ +case oracle_accepts omni_accepts sql +CONNECT_BY_ROOT:not in false true SELECT a FROM t WHERE CONNECT_BY_ROOT NOT IN (1, 2) +XMLELEMENT:index false true CREATE INDEX t_i ON t (XMLELEMENT) diff --git a/oracle/parser/testdata/coverage/oracle_keywords.tsv b/oracle/parser/testdata/coverage/oracle_keywords.tsv index 6c7033f17..c7713e03e 100644 --- a/oracle/parser/testdata/coverage/oracle_keywords.tsv +++ b/oracle/parser/testdata/coverage/oracle_keywords.tsv @@ -44,7 +44,7 @@ kw_042 CONNECT_BY_ROOT 2036 nonreserved allow allow skip allow skip skip lexer k kw_043 CONSTANT 2037 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_044 CONSTRAINT 2038 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_045 CONSTRAINTS 2039 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_046 CONTENT 2309 context allow allow skip reject skip skip lexer keyword classified by local Oracle parser policy +kw_046 CONTENT 2309 context allow allow skip allow skip skip engine-checked: Oracle accepts it as a schema-qualified table name kw_047 CONTEXT 2325 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_048 CONTINUE 2040 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_049 CREATE 2041 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy @@ -95,7 +95,7 @@ kw_093 FOLLOWING 2290 context allow allow skip allow skip skip lexer keyword cla kw_094 FOR 2079 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy kw_095 FORALL 2080 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_096 FORCE 2081 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_097 FOREIGN 2082 nonreserved allow reject skip allow skip skip lexer keyword classified by local Oracle parser policy +kw_097 FOREIGN 2082 nonreserved allow allow skip allow skip skip engine-checked: not reserved in V$RESERVED_WORDS; Oracle accepts it as an identifier kw_098 FORMAT 2331 context allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_099 FROM 2083 clause-starter reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy kw_100 FULL 2084 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy @@ -131,7 +131,7 @@ kw_129 IS 2111 reserved reject reject skip reject skip skip lexer keyword classi kw_130 ISOLATION 2112 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_131 ITERATE 2280 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_132 JAVA 2327 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_133 JOIN 2113 clause-starter allow allow skip reject skip skip lexer keyword classified by local Oracle parser policy +kw_133 JOIN 2113 clause-starter allow allow skip allow skip skip engine-checked: Oracle accepts it as a schema-qualified table name kw_134 JSON 2332 context allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_135 JSON_ARRAY 2333 function allow allow skip allow allow skip lexer keyword classified by local Oracle parser policy kw_136 JSON_EXISTS 2334 function allow allow skip allow allow skip lexer keyword classified by local Oracle parser policy @@ -225,10 +225,10 @@ kw_223 PIVOT 2184 nonreserved allow allow skip allow skip skip lexer keyword cla kw_224 PLAN 2185 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_225 PRAGMA 2186 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_226 PRECEDING 2292 context allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_227 PRIMARY 2187 nonreserved allow reject skip allow skip skip lexer keyword classified by local Oracle parser policy +kw_227 PRIMARY 2187 nonreserved allow allow skip allow skip skip engine-checked: not reserved in V$RESERVED_WORDS; Oracle accepts it as an identifier kw_228 PRIOR 2188 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy kw_229 PRIVATE 2189 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy -kw_230 PRIVILEGES 2190 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy +kw_230 PRIVILEGES 2190 nonreserved allow allow skip allow skip skip engine-checked: not reserved in V$RESERVED_WORDS; Oracle accepts it as an identifier kw_231 PROCEDURE 2191 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_232 PROFILE 2329 nonreserved allow allow skip allow skip skip lexer keyword classified by local Oracle parser policy kw_233 PUBLIC 2192 reserved reject reject skip reject skip skip lexer keyword classified by local Oracle parser policy diff --git a/oracle/parser/testdata/coverage/plsql_dictionary_known_failures.tsv b/oracle/parser/testdata/coverage/plsql_dictionary_known_failures.tsv new file mode 100644 index 000000000..abc0b824d --- /dev/null +++ b/oracle/parser/testdata/coverage/plsql_dictionary_known_failures.tsv @@ -0,0 +1,103 @@ +unit category position error +GSMADMIN_INTERNAL.DBMS_GSM_COMMON:PACKAGE ACCESSIBLE BY clause 567:26 syntax error at or near "dbms_gsm_dbadmin" +GSMADMIN_INTERNAL.DBMS_GSM_DBADMIN:PACKAGE ACCESSIBLE BY clause 71:24 syntax error at or near "sys" +GSMADMIN_INTERNAL.DBMS_GSM_UTILITY:PACKAGE ACCESSIBLE BY clause 938:1 syntax error at or near "ACCESSIBLE" +GSMADMIN_INTERNAL.DBMS_SHARDING_DIRECTORY:PACKAGE Oracle-internal syntax (PRAGMA INTERFACE, ... parameters) 42:34 syntax error at or near ".." +GSMADMIN_INTERNAL.CAT_ROLLBACK_TRIGGER:TRIGGER trigger CALL with arguments 3:40 syntax error at or near "(" +GSMADMIN_INTERNAL.DONE_TRIGGER:TRIGGER trigger CALL with arguments 3:40 syntax error at or near "(" +GSMADMIN_INTERNAL.REQUEST_DELETE_TRIGGER:TRIGGER trigger CALL with arguments 3:36 syntax error at or near "(" +SYS.AVTUNE_LOG_LISTAGGCLOB$:FUNCTION PIPELINED / AGGREGATE USING implementation type 4:43 syntax error at or near ";" +SYS.JSON_DATAGUIDE:FUNCTION PIPELINED / AGGREGATE USING implementation type 6:38 syntax error at or near ";" +SYS.JSON_HIERDATAGUIDE:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:39 syntax error at or near ";" +SYS.MVAGGRAWBITOR:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:34 syntax error at or near ";" +SYS.SCHEDULER$_BATCHERR_PIPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:50 syntax error at or near "USING" +SYS.STRAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:32 syntax error at or near ";" +SYS.SYS_DGAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 6:40 syntax error at or near ";" +SYS.SYS_HIERDGAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:32 syntax error at or near ";" +SYS.SYS_IXMLAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:26 syntax error at or near ";" +SYS.SYS_IXQAGG:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:25 syntax error at or near ";" +SYS.SYS_IXQAGGAVG:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:28 syntax error at or near ";" +SYS.SYS_IXQAGGSUM:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:28 syntax error at or near ";" +SYS.SYS_NT_COLLECT:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:35 syntax error at or near ";" +SYS.V_LISTBACKUPPIPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:34 syntax error at or near "using" +SYS.V_LISTRSRANGEPIPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 2:37 syntax error at or near "using" +SYS.XMLSEQUENCEFROMREFCURSOR:FUNCTION PIPELINED / AGGREGATE USING implementation type 4:11 syntax error at or near "using" +SYS.XMLSEQUENCEFROMREFCURSOR2:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:11 syntax error at or near "using" +SYS.XMLSEQUENCEFROMXMLTYPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:34 syntax error at or near "using" +SYS.XQSEQUENCEFROMXMLTYPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 3:34 syntax error at or near "using" +SYS.XQWINDOWSEQUENCEFROMXMLTYPE:FUNCTION PIPELINED / AGGREGATE USING implementation type 9:12 syntax error at or near "using" +SYS.CDBVIEW_INTERNAL:PACKAGE ACCESSIBLE BY clause 2:26 syntax error at or near "sys" +SYS.DBMS_ASSERT:PACKAGE CHARACTER SET ANY_CS / %CHARSET 21:30 syntax error at or near "CHARACTER" +SYS.DBMS_AW_EXP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 119:54 syntax error at or near "CHARACTER" +SYS.DBMS_CRYPTO:PACKAGE CHARACTER SET ANY_CS / %CHARSET 292:55 syntax error at or near "CHARACTER" +SYS.DBMS_DATA_MINING:PACKAGE PIPELINED / AGGREGATE USING implementation type 1267:13 syntax error at or near "USING" +SYS.DBMS_FREQUENT_ITEMSET:PACKAGE PIPELINED / AGGREGATE USING implementation type 13:49 syntax error at or near "using" +SYS.DBMS_GSM_FIXED:PACKAGE ACCESSIBLE BY clause 86:1 syntax error at or near "ACCESSIBLE" +SYS.DBMS_GSM_GSMUSER:PACKAGE ACCESSIBLE BY clause 13:1 syntax error at or near "ACCESSIBLE" +SYS.DBMS_HADOOP_INTERNAL:PACKAGE PIPELINED / AGGREGATE USING implementation type 408:30 syntax error at or near "USING" +SYS.DBMS_HASH:PACKAGE CHARACTER SET ANY_CS / %CHARSET 17:32 syntax error at or near "CHARACTER" +SYS.DBMS_HCS_LOG:PACKAGE conditional compilation ($IF/$END) 1:1 split into 23 segments +SYS.DBMS_LOB:PACKAGE conditional compilation ($IF/$END) 1:1 split into 219 segments +SYS.DBMS_LOBUTIL:PACKAGE CHARACTER SET ANY_CS / %CHARSET 10:39 syntax error at or near "CHARACTER" +SYS.DBMS_METADATA_INT:PACKAGE ACCESSIBLE BY clause 298:32 syntax error at or near "sys" +SYS.DBMS_METADATA_UTIL:PACKAGE ACCESSIBLE BY clause 818:20 syntax error at or near "ACCESSIBLE" +SYS.DBMS_MLE:PACKAGE conditional compilation ($IF/$END) 1:1 split into 96 segments +SYS.DBMS_PIPE:PACKAGE CHARACTER SET ANY_CS / %CHARSET 246:43 syntax error at or near "character" +SYS.DBMS_PLUGTS:PACKAGE ACCESSIBLE BY clause 121:28 syntax error at or near "sys" +SYS.DBMS_REPORT:PACKAGE Oracle-internal syntax (PRAGMA INTERFACE, ... parameters) 175:22 syntax error at or near ".." +SYS.DBMS_RULE_INTERNAL:PACKAGE ACCESSIBLE BY clause 46:3 syntax error at or near "ACCESSIBLE" +SYS.DBMS_SCHEDULER:PACKAGE CHARACTER SET ANY_CS / %CHARSET 1076:40 syntax error at or near "CHARACTER" +SYS.DBMS_SODA_UTIL:PACKAGE ACCESSIBLE BY clause 1:61 syntax error at or near "accessible" +SYS.DBMS_SQL:PACKAGE conditional compilation ($IF/$END) 1:1 split into 600 segments +SYS.DBMS_SQLTUNE_UTIL0:PACKAGE ACCESSIBLE BY clause 449:26 syntax error at or near "SYS" +SYS.DBMS_SQLTUNE_UTIL1:PACKAGE ACCESSIBLE BY clause 702:3 syntax error at or near "ACCESSIBLE" +SYS.DBMS_SQLTUNE_UTIL2:PACKAGE ACCESSIBLE BY clause 552:3 syntax error at or near "ACCESSIBLE" +SYS.DBMS_STREAMS:PACKAGE CHARACTER SET ANY_CS / %CHARSET 88:59 syntax error at or near "CHARACTER" +SYS.DBMS_TTS:PACKAGE ACCESSIBLE BY clause 289:6 syntax error at or near "ACCESSIBLE" +SYS.DBMS_UPG_JSON0:PACKAGE ACCESSIBLE BY clause 2:1 syntax error at or near "ACCESSIBLE" +SYS.DBMS_UPG_JSON_INT:PACKAGE ACCESSIBLE BY clause 2:1 syntax error at or near "ACCESSIBLE" +SYS.DBMS_UTILITY:PACKAGE conditional compilation ($IF/$END) 1:1 split into 99 segments +SYS.DBMS_VECTOR:PACKAGE PARALLEL_ENABLE (PARTITION ...) 243:55 syntax error at or near "(" +SYS.DBMS_VECTOR_DATABASE_ADMIN:PACKAGE ACCESSIBLE BY clause 2:18 syntax error at or near "accessible" +SYS.HTF:PACKAGE CHARACTER SET ANY_CS / %CHARSET 43:35 syntax error at or near "character" +SYS.HTP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 17:38 syntax error at or near "character" +SYS.OWA_UTIL:PACKAGE CHARACTER SET ANY_CS / %CHARSET 46:42 syntax error at or near "character" +SYS.PLITBLM:PACKAGE Oracle-internal syntax (PRAGMA INTERFACE, ... parameters) 20:29 syntax error at or near ")" +SYS.STANDARD:PACKAGE CHARACTER SET ANY_CS / %CHARSET 255:39 syntax error at or near "CHARACTER" +SYS.UTL_ENCODE:PACKAGE CHARACTER SET ANY_CS / %CHARSET 55:51 syntax error at or near "character" +SYS.UTL_HTTP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 1456:52 syntax error at or near "CHARACTER" +SYS.UTL_I18N:PACKAGE CHARACTER SET ANY_CS / %CHARSET 109:43 syntax error at or near "CHARACTER" +SYS.UTL_LMS:PACKAGE CHARACTER SET ANY_CS / %CHARSET 37:54 syntax error at or near "CHARACTER" +SYS.UTL_RAW:PACKAGE CHARACTER SET ANY_CS / %CHARSET 80:38 syntax error at or near "CHARACTER" +SYS.UTL_RECOMP:PACKAGE conditional compilation ($IF/$END) 1:1 split into 18 segments +SYS.UTL_SMTP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 452:46 syntax error at or near "CHARACTER" +SYS.UTL_TCP:PACKAGE CHARACTER SET ANY_CS / %CHARSET 340:50 syntax error at or near "CHARACTER" +SYS.UTL_URL:PACKAGE CHARACTER SET ANY_CS / %CHARSET 147:53 syntax error at or near "CHARACTER" +SYS.UTL_XML:PACKAGE ACCESSIBLE BY clause 129:3 syntax error at or near "ACCESSIBLE" +SYS.CDBVIEW_INTERNAL:PACKAGE BODY qualified expression (=>) 12:57 syntax error at or near "=>" +SYS.DBMS_HADOOP:PACKAGE BODY type length expression 98:40 syntax error at or near "+" +SYS.DBMS_HADOOP_INTERNAL:PACKAGE BODY type length expression 62:37 syntax error at or near "+" +SYS.HTF:PACKAGE BODY CHARACTER SET ANY_CS / %CHARSET 40:32 syntax error at or near "CHARACTER" +SYS.HTP:PACKAGE BODY CHARACTER SET ANY_CS / %CHARSET 68:39 syntax error at or near "character" +SYS.OWA_COOKIE:PACKAGE BODY package initialization section 221:1 syntax error at or near "begin" +SYS.OWA_CUSTOM:PACKAGE BODY package initialization section 15:1 syntax error at or near "begin" +SYS.OWA_IMAGE:PACKAGE BODY package initialization section 14:1 syntax error at or near "begin" +SYS.OWA_UTIL:PACKAGE BODY CHARACTER SET ANY_CS / %CHARSET 24:26 syntax error at or near "character" +SYS.STANDARD:PACKAGE BODY CHARACTER SET ANY_CS / %CHARSET 16:31 syntax error at or near "CHARACTER" +SYS.DBMS_FEATURE_ADBDCAT:PROCEDURE JSON function clause 181:49 syntax error at or near ")" +SYS.DBMS_FEATURE_ADBXTTAB:PROCEDURE JSON function clause 320:49 syntax error at or near ")" +SYS.DBMS_FEATURE_JOB_CLOUDSQL:PROCEDURE JSON function clause 54:13 syntax error at or near "'type'" +SYS.DBMS_FEATURE_MLE:PROCEDURE JSON function clause 29:49 syntax error at or near ")" +SYS.DBMS_FEATURE_SQL_FIREWALL:PROCEDURE JSON function clause 56:19 syntax error at or near ")" +SYS.OBJNUM_REUSE_HOLES:PROCEDURE SQL%BULK_EXCEPTIONS 107:30 syntax error at or near "%" +SYS.DBMS_KAFKA_DROP_USER:TRIGGER conditional compilation ($IF/$END) 57:7 syntax error at or near "$" +SYS.JDOM_T:TYPE BODY CHARACTER SET ANY_CS / %CHARSET 9:41 syntax error at or near "character" +SYS.JSON_ARRAY_T:TYPE BODY CHARACTER SET ANY_CS / %CHARSET 111:39 syntax error at or near "character" +SYS.JSON_ELEMENT_T:TYPE BODY CHARACTER SET ANY_CS / %CHARSET 2:38 syntax error at or near "character" +SYS.JSON_OBJECT_T:TYPE BODY CHARACTER SET ANY_CS / %CHARSET 110:39 syntax error at or near "character" +SYS.WRR$_ASH_TIME_PERIOD:TYPE BODY type body method modifiers 3:3 syntax error at or near "FINAL" +XDB.XMLINDEXINSFUNC:FUNCTION PIPELINED / AGGREGATE USING implementation type 4:11 syntax error at or near "using" +XDB.XMLINDEXLOADFUNC:FUNCTION PARALLEL_ENABLE (PARTITION ...) 5:17 syntax error at or near "(" +XDB.DBMS_XDB:PACKAGE REF type 234:37 syntax error at or near "SYS" +XDB.DBMS_XDBRESOURCE:PACKAGE REF type 488:56 syntax error at or near "SYS" +XDB.DBMS_XDB_REPOS:PACKAGE REF type 223:37 syntax error at or near "SYS" diff --git a/oracle/parser/testdata/coverage/reference_oracle.tsv b/oracle/parser/testdata/coverage/reference_oracle.tsv index e2e856f92..46e1df916 100644 --- a/oracle/parser/testdata/coverage/reference_oracle.tsv +++ b/oracle/parser/testdata/coverage/reference_oracle.tsv @@ -93,3 +93,13 @@ ref_091 plsql_body accept CREATE OR REPLACE FUNCTION omni_cs_names_{run} (langua ref_092 plsql_body reject CREATE OR REPLACE TYPE omni_bad_novr_{run} AS OBJECT (a NUMBER, NOT OVERRIDING MEMBER FUNCTION f RETURN NUMBER) NOT FINAL NOT OVERRIDING in a method spec ref_093 plsql_body reject CREATE OR REPLACE TYPE omni_bad_fovr_{run} AS OBJECT (a NUMBER, FINAL OVERRIDING MEMBER FUNCTION f RETURN NUMBER) OVERRIDING after another inheritance clause ref_094 plsql_body accept CREATE OR REPLACE TYPE omni_ninf_{run} AS OBJECT (a NUMBER, NOT INSTANTIABLE NOT FINAL MEMBER FUNCTION f RETURN NUMBER) NOT FINAL NOT INSTANTIABLE NOT INSTANTIABLE NOT FINAL method spec +ref_095 select accept SELECT CASE NOT TRUE WHEN TRUE THEN 1 ELSE 0 END FROM dual simple CASE with a Boolean NOT selector +ref_096 select reject SELECT 1 FROM t, s WHERE systimestamp(+) = s.x pseudo-column with the outer-join marker (ORA-30088) +ref_097 select accept SELECT a FROM t_kwcol ORDER BY case NULLS FIRST column named CASE ordered NULLS FIRST +ref_098 select accept SELECT a FROM t_kwcol ORDER BY case DESC NULLS LAST column named CASE ordered DESC NULLS LAST +ref_099 select accept SELECT a FROM t_kwcol WHERE case LIKEC '1%' column named CASE compared with LIKEC +ref_100 select accept SELECT CASE nulls WHEN 1 THEN 2 END FROM t_kwcol column named NULLS as a simple CASE selector +ref_101 select accept SELECT CASE likec WHEN 'x' THEN 2 END FROM t_kwcol column named LIKEC as a simple CASE selector +ref_102 select accept SELECT CASE likec = 'x' WHEN TRUE THEN 1 ELSE 0 END FROM t_kwcol simple CASE with a Boolean comparison selector +ref_103 select accept SELECT k, j, v FROM (SELECT 1 k, 1 j, 10 v FROM dual) MODEL DIMENSION BY (k, j) MEASURES (v) RULES (v[2, 1] = CASE v[1, 1] WHEN 10 THEN 20 END) MODEL cell reference as a simple CASE selector +ref_104 select accept SELECT k, case FROM (SELECT 1 k, 10 v FROM dual) MODEL DIMENSION BY (k) MEASURES (v case) RULES (case[2] = CASE case[1] WHEN 10 THEN 20 END) MODEL measure named CASE diff --git a/oracle/parser/testdata/coverage/strictness_v2.tsv b/oracle/parser/testdata/coverage/strictness_v2.tsv index a7800dfb0..c3a7f6264 100644 --- a/oracle/parser/testdata/coverage/strictness_v2.tsv +++ b/oracle/parser/testdata/coverage/strictness_v2.tsv @@ -36,7 +36,7 @@ expr_010 expression error syntax error SELECT PRIOR expr_011 expression error syntax error SELECT CONNECT_BY_ROOT expr_012 expression error syntax error SELECT (1 expr_013 expression error syntax error SELECT (1 + -expr_014 expression error syntax error SELECT CASE +expr_014 expression error syntax error SELECT CASE 1 expr_015 expression error syntax error SELECT CASE WHEN expr_016 expression error syntax error SELECT CASE WHEN 1 = 1 THEN expr_017 expression error syntax error SELECT CASE WHEN 1 = 1 THEN 2 ELSE