1
#include <Python.h>
#include <fstream>
#include <iostream>
#include <string>
#include <filesystem>
#include <sys/types.h>

#include <dirent.h>


static const char * sPythonCode =
"class Test :\n"
"   def __init__(self) : \n"
"     self.Disc_ = 0. \n"
"   def getset(self) : \n"
"     self.Disc_ = 7. \n"
"     return self.Disc_ \n";


std::string writeFile()
{
   static int iFile = 0;
   std::string sFileName(std::string("test") + std::to_string(iFile));
   std::ofstream out("py/" + sFileName + ".py");
   out << sPythonCode;
   out.flush();
   out.close();
   iFile++;
   return sFileName;
}

static bool bPythonOpen = false;
#define PYTHONPATHLEN 501

static void _PyInit()
{
   if (!Py_IsInitialized())
   {
      Py_InitializeEx(0);
   }
}

void openPython(void)
{
   if (!bPythonOpen)
   {
      const size_t szBufferN = 1000;
      char acLoadPath[szBufferN];
      const char *pypath = "./py";

      _PyInit();

      PyRun_SimpleString("import sys");
      PyRun_SimpleString("print('python (%d.%d.%d) initialized' % (sys.version_info.major, sys.version_info.minor, sys.version_info.micro))");
      PyRun_SimpleString("print('--------------------------')");


      snprintf(acLoadPath, szBufferN, "sys.path.append('%s')", pypath);
      PyRun_SimpleString(acLoadPath);

      bPythonOpen = true;
   }
}

PyObject *loadPythonModule(const char *acModule)
{
   PyObject *pyModule = NULL;

   if (bPythonOpen && acModule && strcmp(acModule, ""))
   {
      printf("%s\n", acModule);
      pyModule = PyImport_ImportModule(acModule);
      if (!pyModule)
      {
         PyErr_Print();
      }
   }
   return pyModule;
}

void loadPython()
{

   std::string sFileName = writeFile();

   openPython();
   //sleep(1);
   PyObject *pPythonModule = loadPythonModule(sFileName.c_str());
   
   if (pPythonModule)
      PyDict_DelItemString(PyImport_GetModuleDict(), PyModule_GetName((PyObject *)pPythonModule));
}

int main(int argc, char **argv)
{
   for (int i = 0; i < 10; i++)
   {     
      loadPython();
   }
}

My working env:

  • gcc version 8.3.1 20190311 (Red Hat 8.3.1-3) (GCC)
  • Red Hat Enterprise Linux Server release 7.6 (Maipo)
  • problem with python 3.6.10 / 3.8.3

Command to compile:

g++ pythontest.cpp -I/opt/python/python3.6.10/include/python3.6m -L/opt/python/python3.6.10/lib -lpython3.6m

create py directory:

mkdir py

When I run this code I have random error on different test file that I load.

Example of output:

python (3.6.10) initialized
--------------------------
test0
test1
test2
test3
ModuleNotFoundError: No module named 'test3'
test4
test5
ModuleNotFoundError: No module named 'test5'
test6
test7
ModuleNotFoundError: No module named 'test7'
test8
test9
ModuleNotFoundError: No module named 'test9'

Good to know:

  • If I uncomment the line with the sleep it works well
  • If I remove the iFile++, it works also as it used after an already created file
  • If I relaunch a second without rm -rf py directory it works also
  • If I erase file after each run in the loadPython function and remove iFile++ it works also
  • If I use strace to launch the executable I don't see the problem

For the moment it seems that the Python loader does not see the file on disk, however in case of failure if I print what I have in the directory thanks to dirent I see the testx.py

Please note that we reproduce the error on different Linux servers (not a hardware problem and even on Windows), with Python 2.7.x it works perfectly well.

1 Answer 1

3

You should call __import__('importlib').invalidate_caches() each time you modify modules folders to let C Python knows it must read directories again.

Sign up to request clarification or add additional context in comments.

1 Comment

@parisjohn Please see stackoverflow.com/help/someone-answers. If the answer has solved your problem, accept it.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.