libc/dlfcn: Count opens so a library can be shared

Documentation/implementation/kernel_modules_vs_shared_libraries.rst

@@ -67,12 +67,15 @@ of a shared library in that environment.
 In this case kernel modules really only differ from shared
 libraries in their usage semantics:
 
-For the FLAT build, I have added the standard ``include/dllfcn.h``
-and have implemented the FLAT shared library support as a thin wrapper
-around the kernel module support:
-
-* ``dlopen()`` maps to ``insmod()``.
-* ``dlclose()`` maps to ``rmmod()``.
+For the FLAT build, the standard ``include/dlfcn.h`` interfaces are
+implemented as a thin wrapper around the same module library that the kernel
+module support uses:
+
+* ``dlopen()`` loads the library, or takes an additional reference on it if
+  it is already loaded, and returns a handle to it. See
+  `Opening a Library More Than Once`_.
+* ``dlclose()`` releases one reference. The library is unloaded, as
+  ``rmmod()`` would, only when the last handle is closed.
 * ``dlsym()`` maps to ``modsym()``.
 * ``dlerror()`` is only a stub at the present time.
 
@@ -101,6 +104,43 @@ The shared library functions no longer call the kernel module logic but rather
 implement their one top-level management logic using the lower-level routines
 in the module library.
 
+The user space copy of the module library keeps the name of each loaded
+library whenever ``CONFIG_LIBC_DLFCN`` is enabled, since the name is the only
+way to tell that a library is already loaded. This costs ``NAME_MAX`` bytes
+per loaded library, but it makes ``dlopen()`` behave exactly as it does in the
+FLAT build.
+
+
+Opening a Library More Than Once
+================================
+
+In the FLAT and PROTECTED builds, ``dlopen()`` of a library that is already
+loaded does not load a second copy and does not fail. It returns a handle to
+the library that is already loaded and takes an additional reference on it.
+Each successful ``dlopen()`` must be matched by a ``dlclose()``; the library
+is unloaded only when the last handle is closed. Up to 255 handles may be
+outstanding on one library; beyond that ``dlopen()`` fails with ``EMFILE``.
+
+Some consequences worth keeping in mind:
+
+* A library is identified by the *basename* of the path passed to
+  ``dlopen()``. Two files with the same basename in different directories
+  are treated as the same library, and the second ``dlopen()`` will return
+  the first one.
+* There is only one instance of the library's ``.data`` and ``.bss``. Global
+  and static data are shared by every user of the library, and by every task
+  group in the system.
+* Constructors in ``.init_array`` run once, when the library is first loaded,
+  and destructors in ``.fini_array`` run once, when the last handle is
+  closed. They do not run per ``dlopen()``/``dlclose()`` pair.
+* Symbols obtained with ``dlsym()`` remain valid until the last handle is
+  closed, not until the caller's own handle is closed.
+
+Kernel modules deliberately behave differently: ``insmod()`` fails with
+``EEXIST`` if a module of that name is already installed, and ``rmmod()``
+removes it immediately. A kernel module is a singleton and is not reference
+counted.
+
 
 Better FLAT and PROTECTED Mode Shared Libraries
 ===============================================
@@ -112,7 +152,7 @@ for each NuttX task group.
 A task group is the moral equivalent of a Unix process.
 That is how a shared library would have to work in uClinux, for example.
 But that would be a substantial effort! For example, since each
-``.bss``/``.data`` would lie at a different physical addres,
+``.bss``/``.data`` would lie at a different physical address,
 the ``.text`` section logic would need support
 Position-Independent-Data (PID).
 Embedded PID support, however, is pretty much broken on all current GCC
@@ -174,3 +214,6 @@ are loaded into memory before the programs ``main()`` logic is called.
 .. note:: There is not yet any shared library support in the KERNEL build mode.
           This would be quite a large effort and not on the plan of record
           at the present time.
+          ``dlopen()`` always fails and returns ``NULL`` in the KERNEL build,
+          so none of the reference counting behaviour described above applies
+          there.

include/nuttx/lib/elf.h

@@ -79,7 +79,12 @@
  *   portion of the build
  */
 
-#if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__)
+/* dlopen() needs a name too: it is the only way to tell that a library is
+ * already loaded.
+ */
+
+#if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__) || \
+    defined(CONFIG_LIBC_DLFCN)
 #  define HAVE_LIBC_ELF_NAMES
 #  define LIBC_ELF_NAMEMAX NAME_MAX
 #endif
@@ -174,6 +179,12 @@ struct module_s
   size_t datasize;                     /* Size of the kernel .bss/.data memory allocation */
 #endif
 
+  uint8_t nopen;                       /* Outstanding references: insmod()
+                                        * and dlopen() each take one, rmmod()
+                                        * and dlclose() give one back, and the
+                                        * module goes when the last does
+                                        */
+
 #if CONFIG_LIBC_ELF_MAXDEPEND > 0
   uint8_t dependents;                  /* Number of modules that depend on this module */
 

libs/libc/dlfcn/lib_dlclose.c

@@ -82,6 +82,7 @@
 int dlclose(FAR void *handle)
 {
 #if defined(CONFIG_BUILD_FLAT) || defined(CONFIG_BUILD_PROTECTED)
+
   /* In the FLAT build, a shared library is essentially the same as a kernel
    * module.
    *
@@ -93,6 +94,10 @@ int dlclose(FAR void *handle)
    * dlremove() is essentially a clone of rmmod().
    */
 
+  /* libelf_remove() gives back a reference and unloads only when the last
+   * one goes, so closing one of two handles leaves the other usable.
+   */
+
   return libelf_remove(handle);
 
 #else /* if defined(CONFIG_BUILD_KERNEL) */

libs/libc/dlfcn/lib_dlopen.c

@@ -26,8 +26,10 @@
 
 #include <nuttx/config.h>
 
-#include <libgen.h>
 #include <dlfcn.h>
+#include <errno.h>
+#include <stdint.h>
+#include <string.h>
 
 #include <nuttx/envpath.h>
 #include <nuttx/lib/elf.h>
@@ -82,24 +84,21 @@
 
 static inline FAR void *dlinsert(FAR const char *filename)
 {
-  FAR void *handle;
-  FAR char *name;
+  FAR const char *modname;
 
   DEBUGASSERT(filename != NULL);
 
-  name = strdup(filename);
-  if (name == NULL)
-    {
-      return NULL;
-    }
+  /* The module name is the basename of the file */
 
-  /* Then install the file using the basename of the file as the module
-   * name.
+  modname = strrchr(filename, '/');
+  modname = modname != NULL ? modname + 1 : filename;
+
+  /* libelf_insert() returns the module already loaded under this name,
+   * with another reference taken, so a library opened twice is one
+   * instance shared by both callers.
    */
 
-  handle = libelf_insert(filename, basename(name));
-  lib_free(name);
-  return handle;
+  return libelf_insert(filename, modname);
 }
 #else /* if defined(CONFIG_BUILD_KERNEL) */
 /* The KERNEL build is considerably more complex:  In order to be shared,

libs/libc/elf/elf_insert.c

@@ -307,14 +307,26 @@ FAR void *libelf_insert(FAR const char *filename, FAR const char *modname)
 
   libelf_registry_lock();
 
-  /* Check if this module is already installed */
+  /* Already installed?  Take another reference rather than load a second
+   * copy: there is one instance of a module per name, and every caller
+   * shares it.  The count is kept here so that insmod()/rmmod() and
+   * dlopen()/dlclose() get the same behaviour from the same code.
+   */
 
 #ifdef HAVE_LIBC_ELF_NAMES
-  if (libelf_registry_find(modname) != NULL)
+  modp = libelf_registry_find(modname);
+  if (modp != NULL)
     {
+      if (modp->nopen == UINT8_MAX)
+        {
+          libelf_registry_unlock();
+          set_errno(EMFILE);
+          return NULL;
+        }
+
+      modp->nopen++;
       libelf_registry_unlock();
-      set_errno(EEXIST);
-      return NULL;
+      return modp;
     }
 #endif
 
@@ -342,6 +354,7 @@ FAR void *libelf_insert(FAR const char *filename, FAR const char *modname)
   /* Save the module name in the registry entry */
 
   strlcpy(modp->modname, modname, sizeof(modp->modname));
+  modp->nopen = 1;
 #endif
 
   /* Load the program binary */

libs/libc/elf/elf_remove.c

@@ -205,6 +205,20 @@ int libelf_remove(FAR void *handle)
       goto errout_with_lock;
     }
 
+  /* Give back a reference.  The module goes only when the last one does,
+   * so an rmmod() cannot pull a module out from under a dlopen() that is
+   * still holding it.
+   */
+
+  if (modp->nopen > 1)
+    {
+      modp->nopen--;
+      libelf_registry_unlock();
+      return OK;
+    }
+
+  modp->nopen = 0;
+
   ret = libelf_uninit(modp);
   if (ret < 0)
     {

sched/module/mod_insmod.c

@@ -26,6 +26,8 @@
 
 #include <nuttx/config.h>
 
+#include <errno.h>
+
 #include <nuttx/module.h>
 #include <nuttx/lib/elf.h>
 
@@ -63,6 +65,17 @@
 
 FAR void *insmod(FAR const char *filename, FAR const char *modname)
 {
+  /* A duplicate name is an error here, where it has always been.
+   * libelf_insert() would take a reference instead, which is what
+   * dlopen() wants and insmod() does not.
+   */
+
+  if (libelf_gethandle(modname) != NULL)
+    {
+      set_errno(EEXIST);
+      return NULL;
+    }
+
   return libelf_insert(filename, modname);
 }