To: vim_dev@googlegroups.com Subject: Patch 7.3.1172 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 7.3.1172 Problem: Python 2: loading modules doesn't work well. Solution: Fix the code. Add more tests. (ZyX) Files: runtime/doc/if_pyth.txt, src/if_py_both.h, src/if_python.c, src/testdir/python2/module.py, src/testdir/python3/module.py, src/testdir/python_after/after.py, src/testdir/python_before/before.py, src/testdir/test86.in, src/testdir/test86.ok, src/testdir/test87.in, src/testdir/test87.ok, Filelist *** ../vim-7.3.1171/runtime/doc/if_pyth.txt 2013-06-10 21:27:18.000000000 +0200 --- runtime/doc/if_pyth.txt 2013-06-12 14:12:13.000000000 +0200 *************** *** 315,366 **** {rtp}/python2 (or python3) and {rtp}/pythonx (for both python versions) for each {rtp} found in 'runtimepath'. ! Implementation for python 2 is the following: usual importing code with empty ! lists in place of sys.path_hooks and sys.meta_path. Code is similar to the ! below, but written in C: > ! # Assuming vim variable is already accessible and is set to the current ! # module import sys ! def find_module(fullname): ! return vim ! def load_module(fullname): ! # see vim._get_paths below ! new_path = _get_paths() ! ! try: old_path = sys.path ! except: pass ! try: old_meta_path = sys.meta_path ! except: pass ! try: old_path_hooks = sys.path_hooks ! except: pass ! ! sys.meta_path = [] ! sys.path_hooks = sys.meta_path ! sys.path = new_path ! ! try: ! exec ('import ' + fullname + ' as m') # No actual exec in C code ! return m ! finally: ! e = None ! try: sys.path = old_path ! except Exception as e: pass ! try: sys.meta_path = old_meta_path ! except Exception as e: pass ! try: sys.path_hooks = old_path_hooks ! except Exception as e: pass ! if e: ! raise e ! ! def path_hook(d): ! if d == VIM_SPECIAL_PATH: ! return vim ! raise ImportError ! ! sys.path_hooks.append(path_hook) Implementation for python 3 is cleaner: code is similar to the following, but, again, written in C: > --- 315,367 ---- {rtp}/python2 (or python3) and {rtp}/pythonx (for both python versions) for each {rtp} found in 'runtimepath'. ! Implementation for python 2 is similar to the following, but written in C: > ! from imp import find_module, load_module ! import vim import sys ! class VimModuleLoader(object): ! def __init__(self, module): ! self.module = module ! ! def load_module(self, fullname, path=None): ! return self.module ! ! def _find_module(fullname, oldtail, path): ! idx = oldtail.find('.') ! if idx > 0: ! name = oldtail[:idx] ! tail = oldtail[idx+1:] ! fmr = find_module(name, path) ! module = load_module(fullname[:-len(oldtail)] + name, *fmr) ! return _find_module(fullname, tail, module.__path__) ! else: ! fmr = find_module(fullname, path) ! return load_module(fullname, *fmr) ! ! # It uses vim module itself in place of VimPathFinder class: it does not ! # matter for python which object has find_module function attached to as ! # an attribute. ! class VimPathFinder(object): ! def find_module(cls, fullname, path=None): ! try: ! return VimModuleLoader(_find_module(fullname, fullname, path or vim._get_paths())) ! except ImportError: ! return None ! find_module = classmethod(find_module) ! ! def load_module(cls, fullname, path=None): ! return _find_module(fullname, fullname, path or vim._get_paths()) ! load_module = classmethod(load_module) ! ! def hook(path): ! if path == vim.VIM_SPECIAL_PATH: ! return VimPathFinder ! else: ! raise ImportError ! sys.path_hooks.append(hook) Implementation for python 3 is cleaner: code is similar to the following, but, again, written in C: > *************** *** 395,408 **** Note: you must not use value of this constant directly, always use vim.VIM_SPECIAL_PATH object. - vim.load_module(name) *python-load_module* vim.find_module(...) *python-find_module* vim.path_hook(path) *python-path_hook* Methods or objects used to implement path loading as described above. You should not be using any of these directly except for vim.path_hook in case you need to do something with sys.meta_path. It is not guaranteed that any of the objects will exist in the future vim ! versions. In fact, load_module and find_module methods do not exists in python3. vim._get_paths *python-_get_paths* --- 396,408 ---- Note: you must not use value of this constant directly, always use vim.VIM_SPECIAL_PATH object. vim.find_module(...) *python-find_module* vim.path_hook(path) *python-path_hook* Methods or objects used to implement path loading as described above. You should not be using any of these directly except for vim.path_hook in case you need to do something with sys.meta_path. It is not guaranteed that any of the objects will exist in the future vim ! versions. In fact, find_module methods do not exists in python3. vim._get_paths *python-_get_paths* *** ../vim-7.3.1171/src/if_py_both.h 2013-06-10 21:27:18.000000000 +0200 --- src/if_py_both.h 2013-06-12 14:12:13.000000000 +0200 *************** *** 940,946 **** {"foreach_rtp", VimForeachRTP, METH_VARARGS, "Call given callable for each path in &rtp"}, #if PY_MAJOR_VERSION < 3 {"find_module", FinderFindModule, METH_VARARGS, "Internal use only, returns loader object for any input it receives"}, - {"load_module", LoaderLoadModule, METH_VARARGS, "Internal use only, tries importing the given module from &rtp by temporary mocking sys.path (to an rtp-based one) and unsetting sys.meta_path and sys.path_hooks"}, #endif {"path_hook", VimPathHook, METH_VARARGS, "Hook function to install in sys.path_hooks"}, {"_get_paths", (PyCFunction)Vim_GetPaths, METH_NOARGS, "Get &rtp-based additions to sys.path"}, --- 940,945 ---- *************** *** 5195,5200 **** --- 5194,5206 ---- PyObject_HEAD } FinderObject; static PyTypeObject FinderType; + #else + typedef struct + { + PyObject_HEAD + PyObject *module; + } LoaderObject; + static PyTypeObject LoaderType; #endif static void *************** *** 5444,5449 **** --- 5450,5457 ---- PYTYPE_READY(OutputType); #if PY_MAJOR_VERSION >= 3 PYTYPE_READY(FinderType); + #else + PYTYPE_READY(LoaderType); #endif return 0; } *************** *** 5570,5575 **** --- 5578,5585 ---- {"Options", (PyObject *)&OptionsType}, #if PY_MAJOR_VERSION >= 3 {"Finder", (PyObject *)&FinderType}, + #else + {"Loader", (PyObject *)&LoaderType}, #endif }; *************** *** 5666,5671 **** --- 5676,5684 ---- ADD_CHECKED_OBJECT(m, "_find_module", (py_find_module = PyObject_GetAttrString(path_finder, "find_module"))); + #else + ADD_OBJECT(m, "_find_module", py_find_module); + ADD_OBJECT(m, "_load_module", py_load_module); #endif return 0; *** ../vim-7.3.1171/src/if_python.c 2013-06-10 21:27:18.000000000 +0200 --- src/if_python.c 2013-06-12 14:14:27.000000000 +0200 *************** *** 150,155 **** --- 150,156 ---- # undef Py_InitModule4 # undef Py_InitModule4_64 # undef PyObject_CallMethod + # undef PyObject_CallFunction /* * Wrapper defines *************** *** 219,224 **** --- 220,226 ---- # define PyObject_HasAttrString dll_PyObject_HasAttrString # define PyObject_SetAttrString dll_PyObject_SetAttrString # define PyObject_CallFunctionObjArgs dll_PyObject_CallFunctionObjArgs + # define PyObject_CallFunction dll_PyObject_CallFunction # define PyObject_Call dll_PyObject_Call # define PyString_AsString dll_PyString_AsString # define PyString_AsStringAndSize dll_PyString_AsStringAndSize *************** *** 357,362 **** --- 359,365 ---- static int (*dll_PyObject_HasAttrString)(PyObject *, const char *); static PyObject* (*dll_PyObject_SetAttrString)(PyObject *, const char *, PyObject *); static PyObject* (*dll_PyObject_CallFunctionObjArgs)(PyObject *, ...); + static PyObject* (*dll_PyObject_CallFunction)(PyObject *, char *, ...); static PyObject* (*dll_PyObject_Call)(PyObject *, PyObject *, PyObject *); static char*(*dll_PyString_AsString)(PyObject *); static int(*dll_PyString_AsStringAndSize)(PyObject *, char **, int *); *************** *** 528,533 **** --- 531,537 ---- {"PyObject_HasAttrString", (PYTHON_PROC*)&dll_PyObject_HasAttrString}, {"PyObject_SetAttrString", (PYTHON_PROC*)&dll_PyObject_SetAttrString}, {"PyObject_CallFunctionObjArgs", (PYTHON_PROC*)&dll_PyObject_CallFunctionObjArgs}, + {"PyObject_CallFunction", (PYTHON_PROC*)&dll_PyObject_CallFunction}, {"PyObject_Call", (PYTHON_PROC*)&dll_PyObject_Call}, {"PyString_AsString", (PYTHON_PROC*)&dll_PyString_AsString}, {"PyString_AsStringAndSize", (PYTHON_PROC*)&dll_PyString_AsStringAndSize}, *************** *** 748,757 **** static PyObject *ListGetattr(PyObject *, char *); static PyObject *FunctionGetattr(PyObject *, char *); - static PyObject *LoaderLoadModule(PyObject *, PyObject *); static PyObject *FinderFindModule(PyObject *, PyObject *); static PyObject *VimPathHook(PyObject *, PyObject *); #ifndef Py_VISIT # define Py_VISIT(obj) visit(obj, arg) #endif --- 752,763 ---- static PyObject *ListGetattr(PyObject *, char *); static PyObject *FunctionGetattr(PyObject *, char *); static PyObject *FinderFindModule(PyObject *, PyObject *); static PyObject *VimPathHook(PyObject *, PyObject *); + static PyObject *py_find_module; + static PyObject *py_load_module; + #ifndef Py_VISIT # define Py_VISIT(obj) visit(obj, arg) #endif *************** *** 1376,1465 **** } #endif static PyObject * ! LoaderLoadModule(PyObject *self, PyObject *args) { ! char *fullname; ! PyObject *path; ! PyObject *meta_path; ! PyObject *path_hooks; ! PyObject *new_path; ! PyObject *r; ! PyObject *new_list; ! if (!PyArg_ParseTuple(args, "s", &fullname)) ! return NULL; ! if (!(new_path = Vim_GetPaths(self))) return NULL; ! if (!(new_list = PyList_New(0))) return NULL; ! #define GET_SYS_OBJECT(objstr, obj) \ ! obj = PySys_GetObject(objstr); \ ! PyErr_Clear(); \ ! Py_XINCREF(obj); ! GET_SYS_OBJECT("meta_path", meta_path); ! if (PySys_SetObject("meta_path", new_list)) { ! Py_XDECREF(meta_path); ! Py_DECREF(new_list); ! return NULL; } ! Py_DECREF(new_list); /* Now it becomes a reference borrowed from ! sys.meta_path */ ! #define RESTORE_SYS_OBJECT(objstr, obj) \ ! if (obj) \ ! { \ ! PySys_SetObject(objstr, obj); \ ! Py_DECREF(obj); \ } ! GET_SYS_OBJECT("path_hooks", path_hooks); ! if (PySys_SetObject("path_hooks", new_list)) ! { ! RESTORE_SYS_OBJECT("meta_path", meta_path); ! Py_XDECREF(path_hooks); return NULL; - } ! GET_SYS_OBJECT("path", path); ! if (PySys_SetObject("path", new_path)) ! { ! RESTORE_SYS_OBJECT("meta_path", meta_path); ! RESTORE_SYS_OBJECT("path_hooks", path_hooks); ! Py_XDECREF(path); return NULL; - } - Py_DECREF(new_path); ! r = PyImport_ImportModule(fullname); ! RESTORE_SYS_OBJECT("meta_path", meta_path); ! RESTORE_SYS_OBJECT("path_hooks", path_hooks); ! RESTORE_SYS_OBJECT("path", path); ! if (PyErr_Occurred()) { ! Py_XDECREF(r); return NULL; } ! return r; ! } ! static PyObject * ! FinderFindModule(PyObject *self UNUSED, PyObject *args UNUSED) ! { ! /* ! * Don't bother actually finding the module, it is delegated to the "loader" ! * object (which is basically the same object: vim module). ! */ ! Py_INCREF(vim_module); ! return vim_module; } static PyObject * --- 1382,1534 ---- } #endif + static void + LoaderDestructor(LoaderObject *self) + { + Py_DECREF(self->module); + DESTRUCTOR_FINISH(self); + } + static PyObject * ! LoaderLoadModule(LoaderObject *self, PyObject *args UNUSED) { ! PyObject *r = self->module; ! Py_INCREF(r); ! return r; ! } ! static struct PyMethodDef LoaderMethods[] = { ! /* name, function, calling, doc */ ! {"load_module", (PyCFunction)LoaderLoadModule, METH_VARARGS, ""}, ! { NULL, NULL, 0, NULL} ! }; ! ! static PyObject * ! call_load_module(char *name, int len, PyObject *find_module_result) ! { ! PyObject *fd, *pathname, *description; ! ! if (!PyTuple_Check(find_module_result) ! || PyTuple_GET_SIZE(find_module_result) != 3) ! { ! PyErr_SetString(PyExc_TypeError, ! _("expected 3-tuple as imp.find_module() result")); return NULL; + } ! if (!(fd = PyTuple_GET_ITEM(find_module_result, 0)) ! || !(pathname = PyTuple_GET_ITEM(find_module_result, 1)) ! || !(description = PyTuple_GET_ITEM(find_module_result, 2))) ! { ! PyErr_SetString(PyExc_RuntimeError, ! _("internal error: imp.find_module returned tuple with NULL")); return NULL; + } + + return PyObject_CallFunction(py_load_module, + "s#OOO", name, len, fd, pathname, description); + } ! static PyObject * ! find_module(char *fullname, char *tail, PyObject *new_path) ! { ! PyObject *find_module_result; ! PyObject *module; ! char *dot; ! if ((dot = (char *) vim_strchr((char_u *) tail, '.'))) { ! /* ! * There is a dot in the name: call find_module recursively without the ! * first component ! */ ! PyObject *newest_path; ! int partlen = (int) (dot - 1 - tail); ! ! if (!(find_module_result = PyObject_CallFunction(py_find_module, ! "s#O", tail, partlen, new_path))) ! return NULL; ! ! if (!(module = call_load_module( ! fullname, ! ((int) (tail - fullname)) + partlen, ! find_module_result))) ! { ! Py_DECREF(find_module_result); ! return NULL; ! } ! ! Py_DECREF(find_module_result); ! ! if (!(newest_path = PyObject_GetAttrString(module, "__path__"))) ! { ! Py_DECREF(module); ! return NULL; ! } ! ! Py_DECREF(module); ! ! module = find_module(fullname, dot + 1, newest_path); ! ! Py_DECREF(newest_path); ! ! return module; } ! else ! { ! if (!(find_module_result = PyObject_CallFunction(py_find_module, ! "sO", tail, new_path))) ! return NULL; ! ! if (!(module = call_load_module( ! fullname, ! STRLEN(fullname), ! find_module_result))) ! { ! Py_DECREF(find_module_result); ! return NULL; ! } ! Py_DECREF(find_module_result); ! ! return module; } + } ! static PyObject * ! FinderFindModule(PyObject *self, PyObject *args) ! { ! char *fullname; ! PyObject *module; ! PyObject *new_path; ! LoaderObject *loader; ! ! if (!PyArg_ParseTuple(args, "s", &fullname)) return NULL; ! if (!(new_path = Vim_GetPaths(self))) return NULL; ! module = find_module(fullname, fullname, new_path); ! Py_DECREF(new_path); ! if (!module) { ! Py_INCREF(Py_None); ! return Py_None; ! } ! ! if (!(loader = PyObject_NEW(LoaderObject, &LoaderType))) ! { ! Py_DECREF(module); return NULL; } ! loader->module = module; ! return (PyObject *) loader; } static PyObject * *************** *** 1483,1489 **** PythonMod_Init(void) { /* The special value is removed from sys.path in Python_Init(). */ ! static char *(argv[2]) = {"/must>not&exist/foo", NULL}; if (init_types()) return -1; --- 1552,1585 ---- PythonMod_Init(void) { /* The special value is removed from sys.path in Python_Init(). */ ! static char *(argv[2]) = {"/must>not&exist/foo", NULL}; ! PyObject *imp; ! ! if (!(imp = PyImport_ImportModule("imp"))) ! return -1; ! ! if (!(py_find_module = PyObject_GetAttrString(imp, "find_module"))) ! { ! Py_DECREF(imp); ! return -1; ! } ! ! if (!(py_load_module = PyObject_GetAttrString(imp, "load_module"))) ! { ! Py_DECREF(py_find_module); ! Py_DECREF(imp); ! return -1; ! } ! ! Py_DECREF(imp); ! ! vim_memset(&LoaderType, 0, sizeof(LoaderType)); ! LoaderType.tp_name = "vim.Loader"; ! LoaderType.tp_basicsize = sizeof(LoaderObject); ! LoaderType.tp_flags = Py_TPFLAGS_DEFAULT; ! LoaderType.tp_doc = "vim message object"; ! LoaderType.tp_methods = LoaderMethods; ! LoaderType.tp_dealloc = (destructor)LoaderDestructor; if (init_types()) return -1; *** ../vim-7.3.1171/src/testdir/python2/module.py 2013-06-11 18:47:37.000000000 +0200 --- src/testdir/python2/module.py 2013-06-12 14:12:13.000000000 +0200 *************** *** 1 **** --- 1,2 ---- + import before_1 dir = '2' *** ../vim-7.3.1171/src/testdir/python3/module.py 2013-06-11 18:47:37.000000000 +0200 --- src/testdir/python3/module.py 2013-06-12 14:12:13.000000000 +0200 *************** *** 1 **** --- 1,2 ---- + import before_1 dir = '3' *** ../vim-7.3.1171/src/testdir/python_after/after.py 1970-01-01 01:00:00.000000000 +0100 --- src/testdir/python_after/after.py 2013-06-12 14:12:13.000000000 +0200 *************** *** 0 **** --- 1,2 ---- + import before_2 + dir = "after" *** ../vim-7.3.1171/src/testdir/python_before/before.py 1970-01-01 01:00:00.000000000 +0100 --- src/testdir/python_before/before.py 2013-06-12 14:12:13.000000000 +0200 *************** *** 0 **** --- 1 ---- + dir = "before" *** ../vim-7.3.1171/src/testdir/test86.in 2013-06-11 18:47:37.000000000 +0200 --- src/testdir/test86.in 2013-06-12 14:12:13.000000000 +0200 *************** *** 8,13 **** --- 8,14 ---- STARTTEST :so small.vim :set encoding=latin1 + :set noswapfile :if !has('python') | e! test.ok | wq! test.out | endif :lang C :py import vim *************** *** 1071,1080 **** --- 1072,1087 ---- :" :" Test import py << EOF + sys.path.insert(0, os.path.join(os.getcwd(), 'python_before')) + sys.path.append(os.path.join(os.getcwd(), 'python_after')) vim.options['rtp'] = os.getcwd().replace(',', '\\,').replace('\\', '\\\\') from module import dir as d from modulex import ddir cb.append(d + ',' + ddir) + import before + cb.append(before.dir) + import after + cb.append(after.dir) EOF :" :" Test exceptions *** ../vim-7.3.1171/src/testdir/test86.ok 2013-06-11 18:47:37.000000000 +0200 --- src/testdir/test86.ok 2013-06-12 14:12:13.000000000 +0200 *************** *** 1084,1089 **** --- 1084,1091 ---- vim.current.tabpage = True:(, TypeError('expected vim.TabPage object',)) vim.current.xxx = True:(, AttributeError('xxx',)) 2,xx + before + after vim.command("throw 'abc'"):(, error('abc',)) Exe("throw 'def'"):(, error('def',)) vim.eval("Exe('throw ''ghi''')"):(, error('ghi',)) *** ../vim-7.3.1171/src/testdir/test87.in 2013-06-11 18:47:37.000000000 +0200 --- src/testdir/test87.in 2013-06-12 14:12:13.000000000 +0200 *************** *** 2,7 **** --- 2,8 ---- STARTTEST :so small.vim + :set noswapfile :if !has('python3') | e! test.ok | wq! test.out | endif :lang C :py3 import vim *************** *** 1038,1047 **** --- 1039,1054 ---- :" :" Test import py3 << EOF + sys.path.insert(0, os.path.join(os.getcwd(), 'python_before')) + sys.path.append(os.path.join(os.getcwd(), 'python_after')) vim.options['rtp'] = os.getcwd().replace(',', '\\,').replace('\\', '\\\\') from module import dir as d from modulex import ddir cb.append(d + ',' + ddir) + import before + cb.append(before.dir) + import after + cb.append(after.dir) EOF :" :" Test exceptions *** ../vim-7.3.1171/src/testdir/test87.ok 2013-06-11 18:47:37.000000000 +0200 --- src/testdir/test87.ok 2013-06-12 14:12:13.000000000 +0200 *************** *** 1093,1098 **** --- 1093,1100 ---- vim.current.tabpage = True:(, TypeError('expected vim.TabPage object',)) vim.current.xxx = True:(, AttributeError('xxx',)) 3,xx + before + after vim.command("throw 'abc'"):(, error('abc',)) Exe("throw 'def'"):(, error('def',)) vim.eval("Exe('throw ''ghi''')"):(, error('ghi',)) *** ../vim-7.3.1171/Filelist 2013-06-11 18:47:37.000000000 +0200 --- Filelist 2013-06-12 14:13:57.000000000 +0200 *************** *** 87,92 **** --- 87,94 ---- src/testdir/python2/*.py \ src/testdir/python3/*.py \ src/testdir/pythonx/*.py \ + src/testdir/python_after/*.py \ + src/testdir/python_before/*.py \ src/proto.h \ src/proto/blowfish.pro \ src/proto/buffer.pro \ *** ../vim-7.3.1171/src/version.c 2013-06-12 14:10:23.000000000 +0200 --- src/version.c 2013-06-12 14:14:12.000000000 +0200 *************** *** 730,731 **** --- 730,733 ---- { /* Add new patch number below this line */ + /**/ + 1172, /**/ -- Microsoft says that MS-Windows is much better for you than Linux. That's like the Pope saying that catholicism is much better for you than protestantism. /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///