使用线程同时在c中调用多个Python-extensions

Question

我有一个使用 2 个线程的 c 程序。现在我想在两个线程中调用一个 Python 函数。当我只在一个线程中调用一个 python 函数时它确实有效，但是当同时调用它们时会出现此错误：
致命 Python 错误：已跟踪 GC 对象。

环顾四周，我发现这是由 python 函数引起的，垃圾收集器不会区分两者。现在我似乎找不到关于这个主题的任何例子。我确实找到了一些可以提供帮助的线索，但同样没有关于如何实施它们的示例。我想我需要让 GC 以类似于 malloc() 的方式了解两者之间的区别。

我以为我可以使用下面的函数 PyObject_GC_New()，但我不知道如何实现它。

在我的代码下方，当您注释掉 speakerPlay() 中的 audioSend() 并将其替换为 print 语句时，它会起作用。

#include <stdio.h>
#include <Python.h>
#include <numpy/ndarrayobject.h>
#include <pthread.h>

typedef struct{
    int length;
    int duration;
}args;

void* micRecord(void* val);
void* calc(void* val);
void* speakerPlay(void* val);

npy_float** audioReceive(int length);
int audioSend(int length);

int main(){
  // init python imports
  Py_Initialize();
  import_array();

  int length = 2;
  int duration = 10;

  args* tArgs = &(args){.length = length, .duration = duration};

  pthread_t recordHandle;
  if( pthread_create(&recordHandle , NULL, micRecord, tArgs))exit( EXIT_FAILURE );

  pthread_t playHandle;
  if( pthread_create(&playHandle , NULL, speakerPlay, tArgs))exit( EXIT_FAILURE );

  void* result;
  if(pthread_join(recordHandle, &result)==-1)
  {
      exit(EXIT_FAILURE);
      printf("could not join\n");
  }

  if(pthread_join(playHandle, &result)==-1)
  {
      exit(EXIT_FAILURE);
      printf("could not join\n");
  }
  printf("Threads joined\n");

  Py_Finalize();
  return 0;
}

void* micRecord(void* rArgs){
  printf("in micRecord\n\n");

  args tArgs = *(args*)rArgs;
  int length = tArgs.length;
  int duration = tArgs.duration;

  npy_float** tempArray;

  for(int i = 0; i<duration; i++){
    tempArray = audioReceive(length);
  }

  printf("micRecord thread done\n");
  pthread_exit(NULL);
}

void* speakerPlay(void* pArgs){
  printf("in speakerPlay\n\n");

  args tArgs = *(args*)pArgs;
  int length = tArgs.length;
  int duration = tArgs.duration;

  for(int i = 0; i < duration; i++){
    audioSend(length);
    //printf("sending\n");
  }

  printf("Speaker thread done\n");
  pthread_exit(NULL);
}

npy_float** audioReceive(int length){
  //variables init
  PyObject *pName, *pModule, *pFunc, *pArgs;
  PyObject* pValue;
  npy_float** array;

  // import the .py file in which the to-call python function is located
  pName = PyUnicode_FromString("receiveSound");
  pModule = PyImport_Import(pName);
  Py_DECREF(pName);
  if(pModule == NULL) printf("its null\n");

  if (pModule != NULL) {
    // Get the reference to the to-call python function and checks if its callable
    pFunc = PyObject_GetAttrString(pModule, "recordSound");
    if (pFunc && PyCallable_Check(pFunc)) {
      // set arguments you want to pass along to the python function
      pArgs = PyTuple_New(1);
      PyTuple_SetItem(pArgs, 0, PyLong_FromLong(length));

      // call the python function and return the numpy array in pValue
      pValue = PyObject_CallObject(pFunc, pArgs);
      if (pValue == NULL) {
        Py_DECREF(pFunc);
        Py_DECREF(pModule);
        PyErr_Print();
        fprintf(stderr,"Call failed\n");
        return NULL;
      }

      // get the type description from the array content
      PyArray_Descr *descr;
      descr = PyArray_DescrFromType(PyArray_TYPE(pValue));

      // convert the numpy array to, a by c-compiler useable format, npy_float array
      if (PyArray_AsCArray(&pValue, (void*)&array, PyArray_DIMS(pValue), PyArray_NDIM(pValue), descr) < 0)  {
        PyErr_SetString(PyExc_TypeError, "error converting to c array");
        return NULL;
      }
      //printf("input\n%"NPY_FLOAT_FMT"\n%"NPY_FLOAT_FMT"\n%"NPY_FLOAT_FMT"\n%"NPY_FLOAT_FMT"\n", array[0], array[5], array[10], array[12]);
      Py_DECREF(pValue);
    }
    // if there was no such function in the .py file
    else {
        if (PyErr_Occurred())
            PyErr_Print();
        fprintf(stderr, "Cannot find function \n");
    }
    Py_XDECREF(pFunc);
    Py_DECREF(pModule);
  }

  // if there was no such .py file
  else {
    PyErr_Print();
    fprintf(stderr, "Failed to load \n");
    return NULL;
  }

  return array;
}

int audioSend(int length){
  // init variables
  PyObject *pName, *pModule, *pFunc;
  PyObject *pValue;

  // import the .py file in which the to-call python function is located
  pName = PyUnicode_FromString("ss");
  pModule = PyImport_Import(pName);
  Py_DECREF(pName);
  if(pModule == NULL) printf("its null\n");

  if (pModule != NULL) {
      // Get the reference to the to-call python function and checks if its callable
      pFunc = PyObject_GetAttrString(pModule, "playSound");
      if (pFunc && PyCallable_Check(pFunc)) {

        // call the python function to play the sound by reading the array
        pValue = PyObject_CallObject(pFunc, NULL);

        // check if array is played
        if (pValue != NULL) {
            Py_DECREF(pValue);
        }

        // if the array was not correctly read
        else {
            Py_DECREF(pFunc);
            Py_DECREF(pModule);
            PyErr_Print();
            fprintf(stderr,"Call failed\n");
            return 1;
        }
      }

      // if no such function exists in the .py file
      else {
          if (PyErr_Occurred())
              PyErr_Print();
          fprintf(stderr, "Cannot find function \n");
      }
      Py_XDECREF(pFunc);
      Py_DECREF(pModule);
    }
    // if no such .py file exists
    else {
        PyErr_Print();
        fprintf(stderr, "Failed to load \n");
        return 1;
    }
  return 0;
}

这是两个 python 代码文件，它们非常基础。它们包含在我设备上的文件 receiveSound.py 和 ss.py

中

receiveSound.py:

import numpy as np
from array import array

def recordSound(length=50):

    print ("-----------------RECORDING SOUND------------------")

    # Make the array `rec_array`
    recArray = np.array([[0,1,2,3,4], [5,6,7,8,9]], dtype=np.float)

    return recArray

ss.py

def playSound():

    print ("----------------SOUND IS PLAYED------------------")#recArray[49]
    return 1

这是我的 makefile，basicANC.c 是我的 C 代码

basicANCmake: basicANC.c
    gcc -o basicANC -I/usr/include/python3.6m basicANC.c -lpython3.6m -lpthread
run: basicANCmake
    PYTHONPATH=. ./basicANC

Answer 1

Python 通过全局锁保护其对象 space 免受并发访问，恰当地称为 "Global Interpreter Lock"。您需要尊重并适应这一点，以便从主机 C 程序的多个线程使用相同的嵌入式 Python 解释器。该文档包含 a section describing these requirements。有一些方法可以对所有这些进行细粒度控制，但是，根据文档，

[t]he typical idiom for calling into Python from a C thread is:

PyGILState_STATE gstate;
gstate = PyGILState_Ensure();

/* Perform Python actions here. */
result = CallSomeFunction();
/* evaluate result or handle exception */

/* Release the thread. No Python API allowed beyond this point. */
PyGILState_Release(gstate);

我建议通过更新 micRecord() 和 speakerPlay() 函数将每个对 audioReceive() 和 audioSend() 的调用包含在 [=15= 之间] 和 PyGILState_Release().

更新：

此外，

根据您使用的 Python 版本，您可能需要在使用任何线程和 GIL 操作函数之前调用 PyEval_InitThreads()。您应该在同一线程中调用 Py_Initialize() 后执行此操作。即使在实际上不需要它的 Python 版本上这样做也是安全的。
您可能需要主线程在其他线程获取 GIL 之前释放它。在主线程中调用 PyGILState_Check() 函数将确认（或反驳）这一点。有几种方法可以释放 GIL，但也许最简单的方法是将 main() 中创建并加入 Py_BEGIN_ALLOW_THREADS 和 Py_END_ALLOW_THREADS 之间的子线程的部分括起来。请阅读这些宏的文档以了解有关这些宏以及如何使用它们的详细信息。

使用线程同时在c中调用多个Python-extensions

Call multiple Python-extensions in c at the same time using threads

python

garbage-collection

cpython

pthreads