changeset: 83155:d17d10c84d27 branch: 2.7 parent: 83150:52eecab8ab03 user: Serhiy Storchaka date: Sat Apr 06 22:04:10 2013 +0300 files: Lib/test/test_itertools.py Misc/NEWS Modules/itertoolsmodule.c Objects/abstract.c description: Issue #14010: Fix a crash when iterating or deleting deeply nested filters in itertools module (i.e. itertools.izip(), itertools.chain(), etc). diff -r 52eecab8ab03 -r d17d10c84d27 Lib/test/test_itertools.py --- a/Lib/test/test_itertools.py Sat Apr 06 18:54:19 2013 +0300 +++ b/Lib/test/test_itertools.py Sat Apr 06 22:04:10 2013 +0300 @@ -1413,6 +1413,139 @@ self.assertNotIn("does not take keyword arguments", err.args[0]) +class TestRecursionLimit(unittest.TestCase): + # Issue #14010 + recursionlimit = sys.getrecursionlimit() + + def test_chain(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = chain(it, ()) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_compress(self): + data = (0, 1) + selectors = (True, True) + it = data + for _ in xrange(self.recursionlimit): + it = compress(it, selectors) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + it = selectors + for _ in xrange(self.recursionlimit): + it = compress(data, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_cycle(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = cycle(it) + with self.assertRaises(RuntimeError): + for _ in range(3): + next(it) + del it + + def test_dropwhile(self): + it = (0, 1, 0) + for _ in xrange(self.recursionlimit): + it = dropwhile(bool, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_ifilter(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = ifilter(bool, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_ifilterfalse(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = ifilterfalse(bool, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_groupby(self): + key = operator.itemgetter(0) + it = ((0, []), (1, [])) + for _ in xrange(self.recursionlimit): + it = groupby(it, key) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_imap(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = imap(int, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_islice(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = islice(it, 2) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_starmap(self): + it = 'ab' + for _ in xrange(self.recursionlimit): + it = starmap(tuple, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_takewhile(self): + it = (1, 0) + for _ in xrange(self.recursionlimit): + it = takewhile(bool, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_izip(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = izip(it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_izip_longest(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = izip_longest(it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + libreftest = """ Doctest for examples in the library reference: libitertools.tex @@ -1645,7 +1778,7 @@ def test_main(verbose=None): test_classes = (TestBasicOps, TestVariousIteratorArgs, TestGC, RegressionTests, LengthTransparency, - SubclassWithKwargsTest, TestExamples) + SubclassWithKwargsTest, TestExamples, TestRecursionLimit) test_support.run_unittest(*test_classes) # verify reference counting diff -r 52eecab8ab03 -r d17d10c84d27 Misc/NEWS --- a/Misc/NEWS Sat Apr 06 18:54:19 2013 +0300 +++ b/Misc/NEWS Sat Apr 06 22:04:10 2013 +0300 @@ -11,6 +11,8 @@ Library ------- +- Issue #14010: Fix a crash when iterating or deleting deeply nested filters + in itertools module (i.e. itertools.izip(), itertools.chain(), etc). - Issue #14254: IDLE now handles readline correctly across shell restarts. @@ -285,7 +287,7 @@ - Issue #12718: Fix interaction with winpdb overriding __import__ by setting importer attribute on BaseConfigurator instance. - + - Issue #17521: Corrected non-enabling of logger following two calls to fileConfig(). diff -r 52eecab8ab03 -r d17d10c84d27 Modules/itertoolsmodule.c --- a/Modules/itertoolsmodule.c Sat Apr 06 18:54:19 2013 +0300 +++ b/Modules/itertoolsmodule.c Sat Apr 06 22:04:10 2013 +0300 @@ -54,12 +54,14 @@ groupby_dealloc(groupbyobject *gbo) { PyObject_GC_UnTrack(gbo); + Py_TRASHCAN_SAFE_BEGIN(gbo) Py_XDECREF(gbo->it); Py_XDECREF(gbo->keyfunc); Py_XDECREF(gbo->tgtkey); Py_XDECREF(gbo->currkey); Py_XDECREF(gbo->currvalue); Py_TYPE(gbo)->tp_free(gbo); + Py_TRASHCAN_SAFE_END(gbo) } static int @@ -741,9 +743,11 @@ cycle_dealloc(cycleobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->saved); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -886,9 +890,11 @@ dropwhile_dealloc(dropwhileobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -909,7 +915,10 @@ iternext = *Py_TYPE(it)->tp_iternext; for (;;) { + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; if (lz->start == 1) @@ -1030,9 +1039,11 @@ takewhile_dealloc(takewhileobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -1053,7 +1064,10 @@ if (lz->stop == 1) return NULL; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = (*Py_TYPE(it)->tp_iternext)(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; @@ -1221,8 +1235,10 @@ islice_dealloc(isliceobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -1243,7 +1259,10 @@ iternext = *Py_TYPE(it)->tp_iternext; while (lz->cnt < lz->next) { + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; Py_DECREF(item); @@ -1251,7 +1270,10 @@ } if (stop != -1 && lz->cnt >= stop) return NULL; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; lz->cnt++; @@ -1364,9 +1386,11 @@ starmap_dealloc(starmapobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -1384,7 +1408,10 @@ PyObject *result; PyObject *it = lz->it; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; args = (*Py_TYPE(it)->tp_iternext)(it); + Py_LeaveRecursiveCall(); if (args == NULL) return NULL; if (!PyTuple_CheckExact(args)) { @@ -1509,9 +1536,11 @@ imap_dealloc(imapobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->iters); Py_XDECREF(lz->func); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -1686,9 +1715,11 @@ chain_dealloc(chainobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->active); Py_XDECREF(lz->source); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -2837,9 +2868,11 @@ compress_dealloc(compressobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->data); Py_XDECREF(lz->selectors); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -2866,11 +2899,16 @@ exception first). */ + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; datum = datanext(data); - if (datum == NULL) + if (datum == NULL) { + Py_LeaveRecursiveCall(); return NULL; + } selector = selectornext(selectors); + Py_LeaveRecursiveCall(); if (selector == NULL) { Py_DECREF(datum); return NULL; @@ -2983,9 +3021,11 @@ ifilter_dealloc(ifilterobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -3006,7 +3046,10 @@ iternext = *Py_TYPE(it)->tp_iternext; for (;;) { + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; @@ -3128,9 +3171,11 @@ ifilterfalse_dealloc(ifilterfalseobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -3151,7 +3196,10 @@ iternext = *Py_TYPE(it)->tp_iternext; for (;;) { + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; @@ -3551,9 +3599,11 @@ izip_dealloc(izipobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->ittuple); Py_XDECREF(lz->result); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -3576,15 +3626,15 @@ if (tuplesize == 0) return NULL; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; if (Py_REFCNT(result) == 1) { Py_INCREF(result); for (i=0 ; i < tuplesize ; i++) { it = PyTuple_GET_ITEM(lz->ittuple, i); item = (*Py_TYPE(it)->tp_iternext)(it); - if (item == NULL) { - Py_DECREF(result); - return NULL; - } + if (item == NULL) + goto error; olditem = PyTuple_GET_ITEM(result, i); PyTuple_SET_ITEM(result, i, item); Py_DECREF(olditem); @@ -3592,18 +3642,21 @@ } else { result = PyTuple_New(tuplesize); if (result == NULL) - return NULL; + goto error; for (i=0 ; i < tuplesize ; i++) { it = PyTuple_GET_ITEM(lz->ittuple, i); item = (*Py_TYPE(it)->tp_iternext)(it); - if (item == NULL) { - Py_DECREF(result); - return NULL; - } + if (item == NULL) + goto error; PyTuple_SET_ITEM(result, i, item); } } + Py_LeaveRecursiveCall(); return result; +error: + Py_XDECREF(result); + Py_LeaveRecursiveCall(); + return NULL; } PyDoc_STRVAR(izip_doc, @@ -3892,10 +3945,12 @@ izip_longest_dealloc(iziplongestobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->ittuple); Py_XDECREF(lz->result); Py_XDECREF(lz->fillvalue); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int diff -r 52eecab8ab03 -r d17d10c84d27 Objects/abstract.c --- a/Objects/abstract.c Sat Apr 06 18:54:19 2013 +0300 +++ b/Objects/abstract.c Sat Apr 06 22:04:10 2013 +0300 @@ -3104,7 +3104,10 @@ PyIter_Next(PyObject *iter) { PyObject *result; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; result = (*iter->ob_type->tp_iternext)(iter); + Py_LeaveRecursiveCall(); if (result == NULL && PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration))