Prepare for Python "Limited API" in PL/Python

Using the Python Limited API would allow building PL/Python against
any Python 3.x version and using another Python 3.x version at run
time.  This commit does not activate that, but it prepares the code to
only use APIs supported by the Limited API.

Implementation details:

- Convert static types to heap types
  (https://docs.python.org/3/howto/isolating-extensions.html#heap-types).

- Replace PyRun_String() with component functions.

- Replace PyList_SET_ITEM() with PyList_SetItem().

This was previously committed as c47e8df815c and then reverted because
it wasn't working under Python older than 3.8.  That has been fixed in
this version.  There was a Python API change/bugfix between 3.7 and
3.8 that directly affects this patch.  The relevant commit is
<https://github.com/python/cpython/commit/364f0b0f19c>.  The
workarounds described there have been applied in this patch, and it
has been confirmed to work with Python 3.6 and 3.7.

Reviewed-by: Jakob Egger <[email protected]>
Discussion: https://www.postgresql.org/message-id/flat/ee410de1-1e0b-4770-b125-eeefd4726a24@eisentraut.org

diff --git a/src/pl/plpython/plpy_cursorobject.c b/src/pl/plpython/plpy_cursorobject.c
index bb3fa8a390990b002ffe256b97b90c8b8d72e2fe..1c6be7561200746ea9e5acbfb62efa414d18c875 100644 (file)
--- a/src/pl/plpython/plpy_cursorobject.c
+++ b/src/pl/plpython/plpy_cursorobject.c
@@ -20,7 +20,7 @@
 #include "utils/memutils.h"
 
 static PyObject *PLy_cursor_query(const char *query);
-static void PLy_cursor_dealloc(PyObject *arg);
+static void PLy_cursor_dealloc(PLyCursorObject *self);
 static PyObject *PLy_cursor_iternext(PyObject *self);
 static PyObject *PLy_cursor_fetch(PyObject *self, PyObject *args);
 static PyObject *PLy_cursor_close(PyObject *self, PyObject *unused);
@@ -33,22 +33,43 @@ static PyMethodDef PLy_cursor_methods[] = {
    {NULL, NULL, 0, NULL}
 };
 
-static PyTypeObject PLy_CursorType = {
-   PyVarObject_HEAD_INIT(NULL, 0)
-   .tp_name = "PLyCursor",
-   .tp_basicsize = sizeof(PLyCursorObject),
-   .tp_dealloc = PLy_cursor_dealloc,
-   .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
-   .tp_doc = PLy_cursor_doc,
-   .tp_iter = PyObject_SelfIter,
-   .tp_iternext = PLy_cursor_iternext,
-   .tp_methods = PLy_cursor_methods,
+static PyType_Slot PLyCursor_slots[] =
+{
+   {
+       Py_tp_dealloc, PLy_cursor_dealloc
+   },
+   {
+       Py_tp_doc, (char *) PLy_cursor_doc
+   },
+   {
+       Py_tp_iter, PyObject_SelfIter
+   },
+   {
+       Py_tp_iternext, PLy_cursor_iternext
+   },
+   {
+       Py_tp_methods, PLy_cursor_methods
+   },
+   {
+       0, NULL
+   }
 };
 
+static PyType_Spec PLyCursor_spec =
+{
+   .name = "PLyCursor",
+       .basicsize = sizeof(PLyCursorObject),
+       .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+       .slots = PLyCursor_slots,
+};
+
+static PyTypeObject *PLy_CursorType;
+
 void
 PLy_cursor_init_type(void)
 {
-   if (PyType_Ready(&PLy_CursorType) < 0)
+   PLy_CursorType = (PyTypeObject *) PyType_FromSpec(&PLyCursor_spec);
+   if (!PLy_CursorType)
        elog(ERROR, "could not initialize PLy_CursorType");
 }
 
@@ -80,8 +101,12 @@ PLy_cursor_query(const char *query)
    volatile MemoryContext oldcontext;
    volatile ResourceOwner oldowner;
 
-   if ((cursor = PyObject_New(PLyCursorObject, &PLy_CursorType)) == NULL)
+   if ((cursor = PyObject_New(PLyCursorObject, PLy_CursorType)) == NULL)
        return NULL;
+#if PY_VERSION_HEX < 0x03080000
+   /* Workaround for Python issue 35810; no longer necessary in Python 3.8 */
+   Py_INCREF(PLy_CursorType);
+#endif
    cursor->portalname = NULL;
    cursor->closed = false;
    cursor->mcxt = AllocSetContextCreate(TopMemoryContext,
@@ -177,8 +202,12 @@ PLy_cursor_plan(PyObject *ob, PyObject *args)
        return NULL;
    }
 
-   if ((cursor = PyObject_New(PLyCursorObject, &PLy_CursorType)) == NULL)
+   if ((cursor = PyObject_New(PLyCursorObject, PLy_CursorType)) == NULL)
        return NULL;
+#if PY_VERSION_HEX < 0x03080000
+   /* Workaround for Python issue 35810; no longer necessary in Python 3.8 */
+   Py_INCREF(PLy_CursorType);
+#endif
    cursor->portalname = NULL;
    cursor->closed = false;
    cursor->mcxt = AllocSetContextCreate(TopMemoryContext,
@@ -272,30 +301,35 @@ PLy_cursor_plan(PyObject *ob, PyObject *args)
 }
 
 static void
-PLy_cursor_dealloc(PyObject *arg)
+PLy_cursor_dealloc(PLyCursorObject *self)
 {
-   PLyCursorObject *cursor;
+#if PY_VERSION_HEX >= 0x03080000
+   PyTypeObject *tp = Py_TYPE(self);
+#endif
    Portal      portal;
 
-   cursor = (PLyCursorObject *) arg;
-
-   if (!cursor->closed)
+   if (!self->closed)
    {
-       portal = GetPortalByName(cursor->portalname);
+       portal = GetPortalByName(self->portalname);
 
        if (PortalIsValid(portal))
        {
            UnpinPortal(portal);
            SPI_cursor_close(portal);
        }
-       cursor->closed = true;
+       self->closed = true;
    }
-   if (cursor->mcxt)
+   if (self->mcxt)
    {
-       MemoryContextDelete(cursor->mcxt);
-       cursor->mcxt = NULL;
+       MemoryContextDelete(self->mcxt);
+       self->mcxt = NULL;
    }
-   arg->ob_type->tp_free(arg);
+
+   PyObject_Free(self);
+#if PY_VERSION_HEX >= 0x03080000
+   /* This was not needed before Python 3.8 (Python issue 35810) */
+   Py_DECREF(tp);
+#endif
 }
 
 static PyObject *

diff --git a/src/pl/plpython/plpy_planobject.c b/src/pl/plpython/plpy_planobject.c
index 9427674d2f4a6506ad7f6fb6f80d046e18eac903..3e385555e5e08d2bdc917e1b2dbdffeea6a229be 100644 (file)
--- a/src/pl/plpython/plpy_planobject.c
+++ b/src/pl/plpython/plpy_planobject.c
@@ -12,7 +12,7 @@
 #include "plpython.h"
 #include "utils/memutils.h"
 
-static void PLy_plan_dealloc(PyObject *arg);
+static void PLy_plan_dealloc(PLyPlanObject *self);
 static PyObject *PLy_plan_cursor(PyObject *self, PyObject *args);
 static PyObject *PLy_plan_execute(PyObject *self, PyObject *args);
 static PyObject *PLy_plan_status(PyObject *self, PyObject *args);
@@ -26,20 +26,37 @@ static PyMethodDef PLy_plan_methods[] = {
    {NULL, NULL, 0, NULL}
 };
 
-static PyTypeObject PLy_PlanType = {
-   PyVarObject_HEAD_INIT(NULL, 0)
-   .tp_name = "PLyPlan",
-   .tp_basicsize = sizeof(PLyPlanObject),
-   .tp_dealloc = PLy_plan_dealloc,
-   .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
-   .tp_doc = PLy_plan_doc,
-   .tp_methods = PLy_plan_methods,
+static PyType_Slot PLyPlan_slots[] =
+{
+   {
+       Py_tp_dealloc, PLy_plan_dealloc
+   },
+   {
+       Py_tp_doc, (char *) PLy_plan_doc
+   },
+   {
+       Py_tp_methods, PLy_plan_methods
+   },
+   {
+       0, NULL
+   }
 };
 
+static PyType_Spec PLyPlan_spec =
+{
+   .name = "PLyPlan",
+       .basicsize = sizeof(PLyPlanObject),
+       .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+       .slots = PLyPlan_slots,
+};
+
+static PyTypeObject *PLy_PlanType;
+
 void
 PLy_plan_init_type(void)
 {
-   if (PyType_Ready(&PLy_PlanType) < 0)
+   PLy_PlanType = (PyTypeObject *) PyType_FromSpec(&PLyPlan_spec);
+   if (!PLy_PlanType)
        elog(ERROR, "could not initialize PLy_PlanType");
 }
 
@@ -48,8 +65,12 @@ PLy_plan_new(void)
 {
    PLyPlanObject *ob;
 
-   if ((ob = PyObject_New(PLyPlanObject, &PLy_PlanType)) == NULL)
+   if ((ob = PyObject_New(PLyPlanObject, PLy_PlanType)) == NULL)
        return NULL;
+#if PY_VERSION_HEX < 0x03080000
+   /* Workaround for Python issue 35810; no longer necessary in Python 3.8 */
+   Py_INCREF(PLy_PlanType);
+#endif
 
    ob->plan = NULL;
    ob->nargs = 0;
@@ -63,25 +84,32 @@ PLy_plan_new(void)
 bool
 is_PLyPlanObject(PyObject *ob)
 {
-   return ob->ob_type == &PLy_PlanType;
+   return ob->ob_type == PLy_PlanType;
 }
 
 static void
-PLy_plan_dealloc(PyObject *arg)
+PLy_plan_dealloc(PLyPlanObject *self)
 {
-   PLyPlanObject *ob = (PLyPlanObject *) arg;
+#if PY_VERSION_HEX >= 0x03080000
+   PyTypeObject *tp = Py_TYPE(self);
+#endif
 
-   if (ob->plan)
+   if (self->plan)
    {
-       SPI_freeplan(ob->plan);
-       ob->plan = NULL;
+       SPI_freeplan(self->plan);
+       self->plan = NULL;
    }
-   if (ob->mcxt)
+   if (self->mcxt)
    {
-       MemoryContextDelete(ob->mcxt);
-       ob->mcxt = NULL;
+       MemoryContextDelete(self->mcxt);
+       self->mcxt = NULL;
    }
-   arg->ob_type->tp_free(arg);
+
+   PyObject_Free(self);
+#if PY_VERSION_HEX >= 0x03080000
+   /* This was not needed before Python 3.8 (Python issue 35810) */
+   Py_DECREF(tp);
+#endif
 }
 
 

diff --git a/src/pl/plpython/plpy_procedure.c b/src/pl/plpython/plpy_procedure.c
index c35a3b801abbfd7672fd735f2406ff6531cd57f5..b494eeb474f36e0f2d51747b0564fe299eb88e48 100644 (file)
--- a/src/pl/plpython/plpy_procedure.c
+++ b/src/pl/plpython/plpy_procedure.c
@@ -350,6 +350,7 @@ PLy_procedure_compile(PLyProcedure *proc, const char *src)
 {
    PyObject   *crv = NULL;
    char       *msrc;
+   PyObject   *code0;
 
    proc->globals = PyDict_Copy(PLy_interp_globals);
 
@@ -368,7 +369,9 @@ PLy_procedure_compile(PLyProcedure *proc, const char *src)
    msrc = PLy_procedure_munge_source(proc->pyname, src);
    /* Save the mangled source for later inclusion in tracebacks */
    proc->src = MemoryContextStrdup(proc->mcxt, msrc);
-   crv = PyRun_String(msrc, Py_file_input, proc->globals, NULL);
+   code0 = Py_CompileString(msrc, "<string>", Py_file_input);
+   if (code0)
+       crv = PyEval_EvalCode(code0, proc->globals, NULL);
    pfree(msrc);
 
    if (crv != NULL)

diff --git a/src/pl/plpython/plpy_resultobject.c b/src/pl/plpython/plpy_resultobject.c
index 95acce654935c8743b544535402bb0c92c3f0165..80fa1d7accc74729b66bf58300d35b39b5387337 100644 (file)
--- a/src/pl/plpython/plpy_resultobject.c
+++ b/src/pl/plpython/plpy_resultobject.c
@@ -10,7 +10,7 @@
 #include "plpy_resultobject.h"
 #include "plpython.h"
 
-static void PLy_result_dealloc(PyObject *arg);
+static void PLy_result_dealloc(PLyResultObject *self);
 static PyObject *PLy_result_colnames(PyObject *self, PyObject *unused);
 static PyObject *PLy_result_coltypes(PyObject *self, PyObject *unused);
 static PyObject *PLy_result_coltypmods(PyObject *self, PyObject *unused);
@@ -24,17 +24,6 @@ static int   PLy_result_ass_subscript(PyObject *arg, PyObject *item, PyObject *val
 
 static char PLy_result_doc[] = "Results of a PostgreSQL query";
 
-static PySequenceMethods PLy_result_as_sequence = {
-   .sq_length = PLy_result_length,
-   .sq_item = PLy_result_item,
-};
-
-static PyMappingMethods PLy_result_as_mapping = {
-   .mp_length = PLy_result_length,
-   .mp_subscript = PLy_result_subscript,
-   .mp_ass_subscript = PLy_result_ass_subscript,
-};
-
 static PyMethodDef PLy_result_methods[] = {
    {"colnames", PLy_result_colnames, METH_NOARGS, NULL},
    {"coltypes", PLy_result_coltypes, METH_NOARGS, NULL},
@@ -44,23 +33,55 @@ static PyMethodDef PLy_result_methods[] = {
    {NULL, NULL, 0, NULL}
 };
 
-static PyTypeObject PLy_ResultType = {
-   PyVarObject_HEAD_INIT(NULL, 0)
-   .tp_name = "PLyResult",
-   .tp_basicsize = sizeof(PLyResultObject),
-   .tp_dealloc = PLy_result_dealloc,
-   .tp_as_sequence = &PLy_result_as_sequence,
-   .tp_as_mapping = &PLy_result_as_mapping,
-   .tp_str = &PLy_result_str,
-   .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
-   .tp_doc = PLy_result_doc,
-   .tp_methods = PLy_result_methods,
+static PyType_Slot PLyResult_slots[] =
+{
+   {
+       Py_tp_dealloc, PLy_result_dealloc
+   },
+   {
+       Py_sq_length, PLy_result_length
+   },
+   {
+       Py_sq_item, PLy_result_item
+   },
+   {
+       Py_mp_length, PLy_result_length
+   },
+   {
+       Py_mp_subscript, PLy_result_subscript
+   },
+   {
+       Py_mp_ass_subscript, PLy_result_ass_subscript
+   },
+   {
+       Py_tp_str, PLy_result_str
+   },
+   {
+       Py_tp_doc, (char *) PLy_result_doc
+   },
+   {
+       Py_tp_methods, PLy_result_methods
+   },
+   {
+       0, NULL
+   }
+};
+
+static PyType_Spec PLyResult_spec =
+{
+   .name = "PLyResult",
+       .basicsize = sizeof(PLyResultObject),
+       .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+       .slots = PLyResult_slots,
 };
 
+static PyTypeObject *PLy_ResultType;
+
 void
 PLy_result_init_type(void)
 {
-   if (PyType_Ready(&PLy_ResultType) < 0)
+   PLy_ResultType = (PyTypeObject *) PyType_FromSpec(&PLyResult_spec);
+   if (!PLy_ResultType)
        elog(ERROR, "could not initialize PLy_ResultType");
 }
 
@@ -69,8 +90,12 @@ PLy_result_new(void)
 {
    PLyResultObject *ob;
 
-   if ((ob = PyObject_New(PLyResultObject, &PLy_ResultType)) == NULL)
+   if ((ob = PyObject_New(PLyResultObject, PLy_ResultType)) == NULL)
        return NULL;
+#if PY_VERSION_HEX < 0x03080000
+   /* Workaround for Python issue 35810; no longer necessary in Python 3.8 */
+   Py_INCREF(PLy_ResultType);
+#endif
 
    /* ob->tuples = NULL; */
 
@@ -89,20 +114,26 @@ PLy_result_new(void)
 }
 
 static void
-PLy_result_dealloc(PyObject *arg)
+PLy_result_dealloc(PLyResultObject *self)
 {
-   PLyResultObject *ob = (PLyResultObject *) arg;
-
-   Py_XDECREF(ob->nrows);
-   Py_XDECREF(ob->rows);
-   Py_XDECREF(ob->status);
-   if (ob->tupdesc)
+#if PY_VERSION_HEX >= 0x03080000
+   PyTypeObject *tp = Py_TYPE(self);
+#endif
+
+   Py_XDECREF(self->nrows);
+   Py_XDECREF(self->rows);
+   Py_XDECREF(self->status);
+   if (self->tupdesc)
    {
-       FreeTupleDesc(ob->tupdesc);
-       ob->tupdesc = NULL;
+       FreeTupleDesc(self->tupdesc);
+       self->tupdesc = NULL;
    }
 
-   arg->ob_type->tp_free(arg);
+   PyObject_Free(self);
+#if PY_VERSION_HEX >= 0x03080000
+   /* This was not needed before Python 3.8 (Python issue 35810) */
+   Py_DECREF(tp);
+#endif
 }
 
 static PyObject *
@@ -125,7 +156,7 @@ PLy_result_colnames(PyObject *self, PyObject *unused)
    {
        Form_pg_attribute attr = TupleDescAttr(ob->tupdesc, i);
 
-       PyList_SET_ITEM(list, i, PLyUnicode_FromString(NameStr(attr->attname)));
+       PyList_SetItem(list, i, PLyUnicode_FromString(NameStr(attr->attname)));
    }
 
    return list;
@@ -151,7 +182,7 @@ PLy_result_coltypes(PyObject *self, PyObject *unused)
    {
        Form_pg_attribute attr = TupleDescAttr(ob->tupdesc, i);
 
-       PyList_SET_ITEM(list, i, PyLong_FromLong(attr->atttypid));
+       PyList_SetItem(list, i, PyLong_FromLong(attr->atttypid));
    }
 
    return list;
@@ -177,7 +208,7 @@ PLy_result_coltypmods(PyObject *self, PyObject *unused)
    {
        Form_pg_attribute attr = TupleDescAttr(ob->tupdesc, i);
 
-       PyList_SET_ITEM(list, i, PyLong_FromLong(attr->atttypmod));
+       PyList_SetItem(list, i, PyLong_FromLong(attr->atttypmod));
    }
 
    return list;
@@ -227,7 +258,7 @@ PLy_result_str(PyObject *arg)
    PLyResultObject *ob = (PLyResultObject *) arg;
 
    return PyUnicode_FromFormat("<%s status=%S nrows=%S rows=%S>",
-                               Py_TYPE(ob)->tp_name,
+                               "PLyResult",
                                ob->status,
                                ob->nrows,
                                ob->rows);

diff --git a/src/pl/plpython/plpy_subxactobject.c b/src/pl/plpython/plpy_subxactobject.c
index 5c92a0e089a1b214e61178328047ceaa534286d0..ad24a9fc4b8c680de06ac4d279d87ec2ebf08312 100644 (file)
--- a/src/pl/plpython/plpy_subxactobject.c
+++ b/src/pl/plpython/plpy_subxactobject.c
@@ -15,7 +15,6 @@
 List      *explicit_subtransactions = NIL;
 
 
-static void PLy_subtransaction_dealloc(PyObject *subxact);
 static PyObject *PLy_subtransaction_enter(PyObject *self, PyObject *unused);
 static PyObject *PLy_subtransaction_exit(PyObject *self, PyObject *args);
 
@@ -31,21 +30,35 @@ static PyMethodDef PLy_subtransaction_methods[] = {
    {NULL, NULL, 0, NULL}
 };
 
-static PyTypeObject PLy_SubtransactionType = {
-   PyVarObject_HEAD_INIT(NULL, 0)
-   .tp_name = "PLySubtransaction",
-   .tp_basicsize = sizeof(PLySubtransactionObject),
-   .tp_dealloc = PLy_subtransaction_dealloc,
-   .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
-   .tp_doc = PLy_subtransaction_doc,
-   .tp_methods = PLy_subtransaction_methods,
+static PyType_Slot PLySubtransaction_slots[] =
+{
+   {
+       Py_tp_doc, (char *) PLy_subtransaction_doc
+   },
+   {
+       Py_tp_methods, PLy_subtransaction_methods
+   },
+   {
+       0, NULL
+   }
 };
 
+static PyType_Spec PLySubtransaction_spec =
+{
+   .name = "PLySubtransaction",
+       .basicsize = sizeof(PLySubtransactionObject),
+       .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+       .slots = PLySubtransaction_slots,
+};
+
+static PyTypeObject *PLy_SubtransactionType;
+
 
 void
 PLy_subtransaction_init_type(void)
 {
-   if (PyType_Ready(&PLy_SubtransactionType) < 0)
+   PLy_SubtransactionType = (PyTypeObject *) PyType_FromSpec(&PLySubtransaction_spec);
+   if (!PLy_SubtransactionType)
        elog(ERROR, "could not initialize PLy_SubtransactionType");
 }
 
@@ -55,10 +68,13 @@ PLy_subtransaction_new(PyObject *self, PyObject *unused)
 {
    PLySubtransactionObject *ob;
 
-   ob = PyObject_New(PLySubtransactionObject, &PLy_SubtransactionType);
-
+   ob = PyObject_New(PLySubtransactionObject, PLy_SubtransactionType);
    if (ob == NULL)
        return NULL;
+#if PY_VERSION_HEX < 0x03080000
+   /* Workaround for Python issue 35810; no longer necessary in Python 3.8 */
+   Py_INCREF(PLy_SubtransactionType);
+#endif
 
    ob->started = false;
    ob->exited = false;
@@ -66,12 +82,6 @@ PLy_subtransaction_new(PyObject *self, PyObject *unused)
    return (PyObject *) ob;
 }
 
-/* Python requires a dealloc function to be defined */
-static void
-PLy_subtransaction_dealloc(PyObject *subxact)
-{
-}
-
 /*
  * subxact.__enter__() or subxact.enter()
  *

diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c
index 51e1d610259d4f2083540321120474ab69dddd2b..1d127ae3ffe37c7c697f4344d6854459523ba22e 100644 (file)
--- a/src/pl/plpython/plpy_typeio.c
+++ b/src/pl/plpython/plpy_typeio.c
@@ -723,7 +723,7 @@ PLyList_FromArray_recurse(PLyDatumToOb *elm, int *dims, int ndim, int dim,
 
            sublist = PLyList_FromArray_recurse(elm, dims, ndim, dim + 1,
                                                dataptr_p, bitmap_p, bitmask_p);
-           PyList_SET_ITEM(list, i, sublist);
+           PyList_SetItem(list, i, sublist);
        }
    }
    else
@@ -742,14 +742,14 @@ PLyList_FromArray_recurse(PLyDatumToOb *elm, int *dims, int ndim, int dim,
            if (bitmap && (*bitmap & bitmask) == 0)
            {
                Py_INCREF(Py_None);
-               PyList_SET_ITEM(list, i, Py_None);
+               PyList_SetItem(list, i, Py_None);
            }
            else
            {
                Datum       itemvalue;
 
                itemvalue = fetch_att(dataptr, elm->typbyval, elm->typlen);
-               PyList_SET_ITEM(list, i, elm->func(elm, itemvalue));
+               PyList_SetItem(list, i, elm->func(elm, itemvalue));
                dataptr = att_addlength_pointer(dataptr, elm->typlen, dataptr);
                dataptr = (char *) att_align_nominal(dataptr, elm->typalign);
            }