=== modified file 'Lib/re.py' --- Lib/re.py 2008-05-24 16:05:21 +0000 +++ Lib/re.py 2008-05-29 18:49:52 +0000 @@ -222,13 +222,21 @@ _pattern_type = type(sre_compile.compile("", 0)) -_MAXCACHE = 100 +_MAXCACHE = 256 +_CACHE_CLEAR = _MAXCACHE >> 1 + +_CACHE_IDX = -1 +_CACHE_REPL_IDX = -1 def _compile(*key): # internal: compile pattern + global _CACHE_IDX + _CACHE_IDX += 1 cachekey = (type(key[0]),) + key - p = _cache.get(cachekey) - if p is not None: + entry = _cache.get(cachekey) + if entry is not None: + p = entry[1] + _cache[cachekey] = (_CACHE_IDX, p) return p pattern, flags = key if isinstance(pattern, _pattern_type): @@ -242,14 +250,26 @@ except error, v: raise error, v # invalid expression if len(_cache) >= _MAXCACHE: + cache_items = _cache.items() _cache.clear() - _cache[cachekey] = p + + # Sort by index + cache_items.sort(key = lambda x: x[1][0]) + + # Reinsert the last + _cache.update(cache_items[-_CACHE_CLEAR:]) + + _cache[cachekey] = (_CACHE_IDX, p) return p def _compile_repl(*key): # internal: compile replacement pattern - p = _cache_repl.get(key) - if p is not None: + global _CACHE_REPL_IDX + _CACHE_REPL_IDX += 1 + entry = _cache_repl.get(key) + if entry is not None: + p = entry[1] + _cache_repl[key] = (_CACHE_REPL_IDX, p) return p repl, pattern = key try: @@ -257,8 +277,16 @@ except error, v: raise error, v # invalid expression if len(_cache_repl) >= _MAXCACHE: + cache_items = _cache_repl.items() _cache_repl.clear() - _cache_repl[key] = p + + # Sort by index + cache_items.sort(key = lambda x: x[1][0]) + + # Reinsert the last + _cache_repl.update(cache_items[-_CACHE_CLEAR:]) + + _cache_repl[key] = (_CACHE_REPL_IDX, p) return p def _expand(pattern, match, template): === modified file 'Lib/test/test_re.py' --- Lib/test/test_re.py 2008-05-29 18:47:54 +0000 +++ Lib/test/test_re.py 2008-05-29 18:50:07 +0000 @@ -684,6 +684,111 @@ self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a#\nb#\nc#') self.assertEqual(pattern.sub('#', '\n'), '#\n#') + def test_caching_mechinism(self): + """Testing Caches + Verifies the following aspects of the Regular Expression Cache: + 1) The Cache invalidates half its entries when full + 2) The most recent half of the cache entries are kept on invalidation + 3) Verify that initially the cache entry does not exist + 4) Verify that after execution, a cache entry now exists + 5) Perform test on both compile and sub templates + 6) Test purging of the caches + """ + def as_expr(n): + return '(Regexp%03d)' % n + def as_strn(n): + return 'Regexp%03d' % n + def as_tmpl(n): + return r'\1-%03d-\1' % n + def as_resp(n): + return as_strn(n) + ('-%03d-' % n) + as_strn(n) + + # Test purging + # Force at least 1 entry in each cache + self.assertEqual(re.sub('(\w)', r'\1\1', 'a'), 'aa') + + # Verify there is at least one entry in each cache + self.assertNotEqual(len(re._cache), 0) + self.assertNotEqual(len(re._cache_repl), 0) + + # Purge and verify + re.purge() + self.assertEqual(len(re._cache), 0) + self.assertEqual(len(re._cache_repl), 0) + + # Used for testing when the cache overflows + expressions = [ ] + templates = [ ] + compiled_expr = [ ] + compiled_tmpl = [ ] + + # Check that items are cached + for x in range(re._MAXCACHE): + # Get the Corresponding Regular Expression + expr = as_expr(x) + expressions.append((str, expr, 0)) + + # Verify the cache is empty + self.assertFalse(expressions[-1] in re._cache) + + # Create a new expression + pattern = re.compile(expr) + + # Verify the expression is now in the cache + self.assertTrue(expressions[-1] in re._cache) + compiled_expr.append(re._cache.get(expressions[-1])) + + # Get the template expression, tmpl + tmpl = as_tmpl(x) + templates.append((tmpl, pattern)) + + # Verify the replacement cache is empty + self.assertFalse(templates[-1] in re._cache_repl) + + # Perform a new substitution, wherein the template will be + # compiled + pattern.sub(tmpl, "") + + # Verify the expression is now in the cache + self.assertTrue(templates[-1] in re._cache_repl) + compiled_tmpl.append(re._cache_repl.get(templates[-1])) + + # Get the reference source Strings for testing + strn = as_strn(x) + resp = as_resp(x) + + # Verify the expressions were generated correctly + self.assertEqual(pattern.sub(tmpl, strn), resp) + + # Force one more element to be added, clearing half the cache + self.assertEqual(re.sub('(\w)', r'\1\1', 'a'), 'aa') + + # Verify the oldest re._MAXCACHE - re._CACHE_CLEAR items have + # been removed + for idx in range(re._MAXCACHE - re._CACHE_CLEAR): + # Verify + self.assertEqual(re._cache.get(expressions[idx]), None) + self.assertEqual(re._cache_repl.get(templates[idx]), None) + + # Verify the newest re._CACHE_CLEAR items still exist + for idx in range(re._MAXCACHE - re._CACHE_CLEAR, re._MAXCACHE): + # Get the expected value for the Compiled Expression + if compiled_expr[idx] == None: + self.fail() + e = None + else: + e = compiled_expr[idx][1] + + # Get the expected value for the Compiled Template + if compiled_tmpl[idx] == None: + self.fail() + t = None + else: + t = compiled_tmpl[idx][1] + + # Verify + self.assertEqual(re._cache.get(expressions[idx])[1], e) + self.assertEqual(re._cache_repl.get(templates[idx])[1], t) def run_re_tests(): from test.re_tests import benchmarks, tests, SUCCEED, FAIL, SYNTAX_ERROR