;;; dynlib-run.el --- runtime support for dynamic libraries

;; Copyright (C) 1994 Hans Chalupsky

;; Author: Hans Chalupsky <hans@cs.buffalo.edu>
;; Created: 26 Feb 1994
;; Version: dynlib-run.el,v 1.5 1994/11/21 10:49:03 hans Exp
;; Keywords: lisp, internal

;; This program is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 2, or (at your option)
;; any later version.
;;
;; This program is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.
;;
;; A copy of the GNU General Public License can be obtained from this
;; program's author (send electronic mail to hans@cs.buffalo.edu) or from
;; the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA
;; 02139, USA.


;;; Commentary:

;; The Dynlib package compiles Emacs Lisp files into dynamic libraries and
;; provides run-time support for such libraries.  Dynamic libraries
;; implement a dynamic loading mechanism for Emacs Lisp files.  When a
;; dynamic library is loaded only small stub definitions are allocated for
;; the individual functions, and only when a function is run for the first
;; time it gets defined with its full definition which is retrieved
;; individually from an associated library file.  Only the individual
;; function that is run will be defined and nothing else.  This is
;; different from autoloading where running an autoload function will
;; trigger the loading of one or more complete files.  As a positive
;; side-effect of this scheme the documentation strings of dynamically
;; loaded functions are kept in the library file and are not allocated
;; permanently as part of the function definition.


;; Main advantages of Dynlib:

;; - Reduced compiled (.elc) file size (20-30% of the original) which
;;   results in reduced loading time (usually less than 50% of the time
;;   necessary to load the original).
;; - Only functions that have been used at least once are actually fully
;;   defined.  I usually make use of 20-30% of the loaded functions, the
;;   rest remain as stubs (you can use `M-x dynlib-stats' to check your
;;   usage).
;; - No permanent allocation of documentation strings in function and
;;   variable definitions.  The mechanism used is almost as efficient as
;;   the standard DOC file mechanism where the documentation string is
;;   replaced by an index.  We just need one extra cons cell per docstring
;;   and one file-name string per compiled file.  The little bit of extra
;;   cost is vastly compensated by the fact that we don't have to preload
;;   a file to achieve the added efficiency (btw, the docstring of 
;;   `calendar-mode' alone is 13k!!).
;; - Provides an automatic, fine-grained autoload mechanism, i.e.,
;;   implementors of big packages such as VM or AUC-TeX don't have to do
;;   the extra work to come up with a clever autoloading scheme that
;;   will only load what is actually needed.
;; - Because there is less stuff allocated in the Emacs image, the image
;;   remains smaller and garbage collection time is reduced.


;; Disclaimer: 

;; This is not yet for the feable-hearted! Only use this package if you
;; have some basic familiarity with byte-compilation and enjoy chasing
;; after weird bugs.  I've used this package successfully for quite a
;; while now, e.g., I run gnus, diary, vm, the Franz ACL interface,
;; AUC-TeX, ange-ftp, etc. with it and haven't stumbled upon any
;; Dynlib-related bugs in a while.  But I'm sure they exist!  In
;; particular, there are still some intricate interactions with Advice
;; that need to be ironed out.  Just wanted you to be warned.

;; Also, the savings in load and startup time may vary! I am sure
;; different machines and operating systems will greatly affect the
;; speedups (or slowdowns?).  I am working on a SPARC 10 and it works
;; well for me.


;; Requirements:

;; Dynlib needs Emacs 19.26 or later and Advice 2.14 (the proper version
;; of Advice already comes with the required Emacs version).  There is no
;; reason why this shouldn't also work in X/Lemacs, but I haven't tested
;; it.  Some compatibility stuff I thought of is already included, the main
;; stumbling blocks are probably an older version of Advice and maybe some
;; differences in the byte compiler.


;; Some Benchmark Data:

;; Will be included "sometime soon" (famous last words).  I've run all
;; kinds of benchmarks, but they still need to be compiled into a proper
;; format.  I have a list of test files which when loaded in sequence on a
;; SPARC 10 take about 25 secs in the standard setup and about 12 secs
;; with Dynlib.  What's easiest to compare are the sizes of the compiled
;; .elc files, and since their sizes directly influence the loading time
;; these sizes give a good indication of the expectable savings.  Here are
;; the sizes of the compiled .elc files of a few packages I use (all sizes
;; are in k-bytes):

;; Package       Dynlib size   standard size   ratio (%)
;; -----------------------------------------------------
;; advice.el          8.1            44.8       18.0
;; ange-ftp.el       18.0            86.2       20.8
;; calendar.el       20.2           101.0       19.9
;; crypt++.el         5.8            29.1       19.9
;; diary-lib.el       7.5            54.1       13.8
;; dired.el          15.2            56.0       27.1
;; dired-aux.el       4.7            36.9       12.7
;; dired-x.el         7.7            33.8       22.7
;; gnus.el           37.8           167.6       22.5
;; holidays.el        1.5            15.6        9.6
;; info.el            5.7            34.0       16.7
;; man.el             4.8            19.1       25.1
;; nntp.el            3.0            12.7       23.6
;; supercite.el      11.9            50.6       23.5
;; VM 5.72           57.4         184.7/355.0 31.0/16.1
;; Franz ACL int.    97.0           307.0       31.5

;; VM 5.72 already does medium-grained autoloading "by hand", hence, the
;; savings are not quite as dramatic.  If one compares the Dynlib version
;; of vm.elc with the combined sizes of the standard .elc files of VM 5.72
;; (355k) then the savings are a lot bigger.

;; In case you do some benchmarking yourself feel free to send me the results
;; (be they positive or negative).


;; How it works:

;; When Dynlib compiles a file it creates 2 output files:

;;    1) a stub file with the familiar extension .elc
;;    2) a library file with the extension .ell

;; The stub file looks just like a normal byte-compiled file with the
;; main difference that for every function definition it only contains
;; a small stub definition that points to a position in the library
;; file where the real definition is stored.  With a similar mechanism
;; documentation strings of functions/variables/autoloads are replaced
;; with such pointers to places in the library file where the actual
;; documentation lives.

;; The library file is a file which contains the documentation strings and
;; compiled function definitions individually separated by \C-_ characters
;; (similar to the DOC file format).  All data and compiled code is properly
;; quoted so that it does not contain `C-_' or `C-@' (NULL) characters.  This
;; makes it possible to use a file position and the `documentation' function
;; to efficiently read a single item (docstring or function) from the library
;; file (the actual internal function used is `get_doc_string' defined in
;; `src/doc.c' which in turn uses `lseek' for direct file access).

;; Because most of the information that is normally part of a byte-compiled
;; file is now stored in an associated library file the stub file is usually
;; much smaller (about 20-30% of the original .elc file) and loading time
;; is decreased (mainly due to fewer garbage collections).

;; When the stub file gets loaded it defines everything in the usual way
;; (just faster) again with the difference that functions only get small
;; stub definitions, and that instead of large documentation strings only
;; small library pointers are allocated.

;; When a stub definition is executed for the first time it fetches its
;; real definition from the library file and redefines itself with it.
;; If it had a documentation string it will still just use a library
;; pointer for that.  As mentioned above, the lookup of the real definition
;; is done with the `documentation' function, i.e., with a fast, direct
;; access disk operation.  This "dynamic" loading of individual functions
;; from a library when they are run gives Dynlib its name.

;; The reduced loading time is somewhat compensated by the runtime overhead
;; when a stub definition is executed for the first time.  How big that
;; overhead is depends on the particular file system/machine you are using.
;; On my system it is an average 10ms (approx. the avg. disk access time)
;; which is pretty fast.  Since most packages don't call too many functions
;; on startup (rarely more than 50) the startup overhead is less than 0.5s.
;; Of course, if you are in a slow NFS environment things could be quite
;; different - even though caching should take care of this.


;; Where Dynlib doesn't gain as much - yet:

;; - Small files 
;; - Packages that contain lots of data, e.g., the byte compiler
;;   which has tons of constant definitions.
;; - Packages that contain lots of function objects that are not bound to
;;   names, e.g., the various setf methods defined in the cl-macs package,
;;   or packages that do everything via advice.

;; Summing up, the main advantage (so far) is saved space.  The startup
;; time reduction that I had hoped to achieve is not as dramatic, partly,
;; because of the extra first-runtime overhead, partly because big
;; packages normally do lots of things on startup which masks the reduced
;; loading time.  There is still room for optimization, in particular,
;; supporting stub-definition handling at the C-level would help a lot.
;; Using `documentation' as the lookup function is fast, but it still
;; has some avoidable overhead, e.g., lots of unnecessary open/close
;; operations.  And of course, everything hinges on a fast, direct-access
;; file system.  If such access is slow, using dynamic libraries might
;; not be worthwhile.

;; One could imagine to have one big library file for the whole Emacs lisp
;; directory which is kept open for one complete Emacs session.  Actually,
;; for people who cannot use `unexec' (or `dump') this might really be
;; an option.

;; The real reason why I wrote this package is that I had trouble in
;; coming up with a small runtime system for the Advice package.  Using
;; Dynlib to compile Advice saves some loading time, it's not as much
;; as I had hoped for though.  Sigh...


;; Installation:

;; Put `dynlib-run.el' and `dyblib-cmp.el' somewhere into your Emacs
;; `load-path' and byte-compile them in that order.  Then add the
;; following autoloads to your .emacs or site setup file:

;;   (autoload 'dynlib-find-library "dynlib-run")
;;   (autoload 'defstubf "dynlib-run")
;;   (autoload 'defstubfi "dynlib-run")
;;   (autoload 'defstubm "dynlib-run")
;;   (autoload 'defdocv "dynlib-run")
;;   (autoload 'defdoca "dynlib-run")
;;   (autoload 'dynlib-compile-file "dynlib-cmp" nil t)
;;   (autoload 'dynlib-compile-to-target "dynlib-cmp" nil t)
;;   (autoload 'batch-dynlib-compile "dynlib-cmp")
;;   (autoload 'batch-dynlib-compile-to-target "dynlib-cmp")

;; Now you are ready to create dynamic libraries for your favorite big
;; Emacs Lisp packages.  Look at the documentation in `dynlib-cmp.el' to
;; find out how to do that.

;; If you use the Advice package you should make sure that it gets loaded
;; BEFORE `dynlib-run' gets loaded.  Things will work even if Advice gets
;; loaded after `dynlib-run', but not quite as efficiently, because the
;; cached advised definition of the `documentation' function will not be
;; usable anymore which will cause Advice to rebuild it at run-time. This
;; is not a big deal, but it will load the byte-compiler which is somewhat
;; inefficient.  If you decided to make Advice into a dynamic library then
;; the proper loading order will be established automatically.


;; Dynlib statistics:

;; You can use `M-x dynlib-stats' to find out about the current Dynlib
;; statistics in your Emacs session.  It will tell you how many stubs
;; are defined, and how many of them have been actually used and loaded.


;; Things to watch out for:

;; - Do not move library files (.ell files) to a different directory while
;;   you use them, because then stubs defined in such a file suddenly
;;   won't find their associated library file anymore when they are run.


;; To do:

;; - use a `dynlib-load-stack' instead of the library-file-vars to push
;;   and pop the full name of the currently loaded library file.  This
;;   saves an argument to the `defstub..'s and would even make it possible
;;   to make them macros.
;; - Need more robust installation (with all the advice and stuff).
;; - Think about interactions with Advice: cache verification, what counts
;;   as a proper definition, automatic advice activation, docstrings, etc.
;; - Think about proper access to arglists so we won't see DYNLIB-ARGS
;;   for as yet undefined stub functions.
;; - Do something useful when `library-lookup' is used on a library file
;;   which got deleted or moved (better error message)!!
;; - Test with Lemacs.  Handle `defalias' unavailability.  Check whether the
;;   defmumble used there is actually the same as used in here.
;; - investigate how garbage collection time is decreased by keeping the
;;   image smaller as is the case when Dynlib is used.



;;; Code:

(eval-when-compile
  (require 'advice))

(defvar dynlib-default-library-directory "/")

(defun dynlib-find-library (library-file)
  ;; Tries to find the actual location of LIBRARY-FILE.
  ;; If it exists and is readable, just return it.  Otherwise try to find
  ;; it somewhere in the `load-path', if it still can't be found signal
  ;; an error.  This function tries to deal with the problem that at load-time
  ;; of a stub file we have to find the actual location of the associated
  ;; library file, because its location might have changed since the
  ;; compilation, e.g., in an installation procedure.
  (let ((default-directory dynlib-default-library-directory)
	(load-path load-path)
	found-library-file)
    (if (file-readable-p library-file)
	(setq found-library-file library-file)
      (setq library-file (file-name-nondirectory library-file))
      (while load-path
	(setq default-directory (car load-path))
	(and (file-readable-p library-file)
	     (not (file-directory-p library-file))
	     (setq found-library-file (expand-file-name library-file))
	     (setq load-path nil))
	(setq load-path (cdr load-path))))
    (if (null found-library-file)
	(error "dynlib-find-library: Can't find `%s'." library-file)
      found-library-file)))

;; Library pointer accessors:
(defmacro dynlib-make-library-pointer (file position)
  (` (cons (, file) (, position))))
(defmacro dynlib-library-pointer-file (library-pointer)
  (` (car (, library-pointer))))
(defmacro dynlib-library-pointer-position (library-pointer)
  (` (cdr (, library-pointer))))
(defmacro dynlib-ncopy-library-pointer (from-pointer to-pointer)
  (` (progn (setcar (, to-pointer) (car (, from-pointer)))
	    (setcdr (, to-pointer) (cdr (, from-pointer))))))
(defmacro dynlib-library-pointer-p (thing)
  (` (consp (, thing))))

(if (not (fboundp 'dynlib-real-documentation-property))
    (fset 'dynlib-real-documentation-property
	  (symbol-function 'documentation-property)))

;; We need the built-in version, for now just bail out if we don't have it
;; (in the future, try to find it by mapping over all symbols which is slow
;; but avoids disaster in case we already use dynamic libraries):
(if (not (subrp (symbol-function 'dynlib-real-documentation-property)))
    (error "dynlib: Somebody advised or redefined `documentation-property'."))

(defun dynlib-lookup (library-pointer)
  (let ((data-directory "")
	(doc-directory "")
	(internal-doc-file-name
	 (dynlib-library-pointer-file library-pointer)))
    (put 'dynlib-lookup-symbol 'library-position
	 (dynlib-library-pointer-position library-pointer))
    (dynlib-real-documentation-property
     'dynlib-lookup-symbol 'library-position t)))


;; Stub functions:

;; Byte-code accessors:
(defmacro dynlib-code-arglist (code)
  (` (aref (, code) 0)))
(defmacro dynlib-code-string (code)
  (` (aref (, code) 1)))
(defmacro dynlib-code-constants (code)
  (` (aref (, code) 2)))
(defmacro dynlib-code-depth (code)
  (` (aref (, code) 3)))
(defmacro dynlib-code-doc (code)
  (` (aref (, code) 4)))
(defmacro dynlib-code-inter (code)
  (` (aref (, code) 5)))
(defmacro dynlib-code-safe-doc (code)
  (` (and (>= (length (, code)) 5)
	  (aref (, code) 4))))
(defmacro dynlib-code-safe-inter (code)
  (` (and (>= (length (, code)) 6)
	  (aref (, code) 5))))

;; Stub definitions are the place holders for actual function definitions
;; until the stub definition is run for the first time.  Ideally, we would use
;; some sort of `autoload' definition that's recognized by the interpreter,
;; but because we don't have that we have to use actual function definitions.
;; Stub definitions are subject to the following requirements and constraints:
;;
;; - They should be as small as possible.
;; - They have to be of the same kind as the original (function, macro, 
;;   interactive).
;; - They have to contain a pointer to the actual definition.
;; - They have to contain a pointer to the actual documentation.
;; - When they are run they have to load the actual definition and apply
;;   it to the arguments.
;; - They have to be able to "redefine" themselves to the actual definition.
;;   This part is tricky: If it were possible to destructively modify a
;;   byte-code object then we could do that and everybody pointing to the
;;   definition would automatically point to the real definition now.
;;   However, that doesn't work.  Hence, we have to figure out who called the
;;   definition and if it was called via a function name we redefine that
;;   function.  If it was not called via a function name (e.g., with something
;;   like `(apply defnvar args)' then we have to destructively modify the
;;   stub definition as good as we can so that it will do the right thing
;;   even if it doesn't do it quite as efficiently.  Either way, we have to
;;   look at the stack via `backtrace-frame' to figure out how we got called.
;;   Originally, I stored the function name in the stub but that method fails
;;   if that function gets redefined to something else before the stub is run.
;;   Because we might have to modify the stub it has to be able to get a hold
;;   of itself when it is run.  This could either be achieved by putting itself
;;   into its constant vector, thus making it circular and unprintable, or
;;   by retrieving it from the stack.  I'll do the latter.
;; - Various packages such as the byte-compiler, Advice, etc.  access bits of
;;   a function definition to do their thing, e.g., they might need an argument
;;   list, the byte-code string, etc.  Since stub definition look like real
;;   definitions these packages will probably do the wrong thing.  There is no
;;   way for Dynlib to fix this by itself, these packages just have to be made
;;   aware that there is a new kind of autoload around which has to be treated
;;   specially.
;;
;; An actual stub definition is a copy of one of the prototypes that shares
;; its argument list, its code string and its depth.  For the constant vector
;; we allocate a copy that contains a library pointer to the definition of
;; the function. 

;; Plain stub function:
(defvar dynlib-stubf-prototype
  (function
   ;; Have to make it very unlikely that the lambda variable would
   ;; shadow any variables referenced by the defined function:
   (lambda (&rest dynlib-ArgS)
     (dynlib-load-and-run-function 'definition-pointer))))

;; Interactive stub function:
(defvar dynlib-stubfi-prototype
  (function
   (lambda (&rest dynlib-ArgS)
     (interactive)
     (dynlib-load-and-run-function 'definition-pointer (interactive-p)))))

;; Stub macro:
(defvar dynlib-stubm-prototype
  (function
   (lambda (&rest dynlib-ArgS)
     (dynlib-load-and-run-macro 'definition-pointer))))

;; Define the arguments variable, because it is referenced in
;; `dynlib-load-and-run-function' and `dynlib-load-and-run-macro':
(defvar dynlib-ArgS nil)

(defmacro dynlib-macro-p (definition)
  ;; Non-nil if DEFINITION is a macro.
  (` (eq (car-safe (, definition)) 'macro)))
(defmacro dynlib-lambda-p (definition)
  ;; Non-nil if DEFINITION is a lambda expression.
  (` (eq (car-safe (, definition)) 'lambda)))
(defmacro dynlib-lambdafy (definition)
  ;; Takes a macro function DEFINITION and makes a lambda out of it.
  (` (cdr (, definition))))

;; Emacs/Lemacs cross-compatibility
;; (compiled-function-p is an obsolete function in Emacs):
(if (and (not (fboundp 'byte-code-function-p))
	 (fboundp 'compiled-function-p))
    (fset 'byte-code-function-p 'compiled-function-p))

(defun dynlib-stub-definition-p (definition)
  ;; Returns non-nil if DEFINITION is a stub.
  (if (dynlib-macro-p definition)
      (setq definition (dynlib-lambdafy definition)))
  (and (byte-code-function-p definition)
       (equal (dynlib-code-arglist dynlib-stubf-prototype)
	      (dynlib-code-arglist definition))
       (memq (aref (dynlib-code-constants definition) 0)
	     '(dynlib-load-and-run-function
	       dynlib-load-and-run-macro))))

;; Accessors to the definition pointers:
(defmacro dynlib-stub-code-pointer (stub-code)
  (` (aref (dynlib-code-constants (, stub-code)) 1)))
;; Safe version of the above which also work for macros:
(defmacro dynlib-stub-definition-pointer (stub-definition)
  (` (dynlib-stub-code-pointer
      (if (dynlib-macro-p (, stub-definition))
	  (dynlib-lambdafy (, stub-definition))
	(, stub-definition)))))

;; Compatibility:
(if (not (fboundp 'defalias))
    (fset 'defalias 'fset))

;; Stub function definers:

;; These functions are run when the stub file gets loaded.  Every function
;; whose definition got saved to the library file got repaced with
;; something like
;;
;;   (defstubf 'foo lib-file-var 69 42)
;;
;; where `lib-file-var' is the variable used in the current library file to
;; hold the actual, fully expanded location of the library file, the first
;; number is the index at which the definition of `foo' can be found in the
;; library file, and the second number (optional) is the index at which its
;; documentation can be found.  With that information the stub definers can
;; build an actual stub function from one of the prototypes above.

;; Some counters for benchmarking purposes:
(defvar dynlib-defined-stubs 0)
(defvar dynlib-defined-functions 0)

(defun defstubf (name file defindex &optional documentation)
  ;; Create a stub definition for the interactive function NAME.
  (let* ((stub-constants
	  (make-vector 2 'dynlib-load-and-run-function))
	 (stub-code
	  (if documentation
	      (make-byte-code
	       ;; Arglist and code-string are shared with the prototype:
	       (dynlib-code-arglist dynlib-stubf-prototype)
	       (dynlib-code-string dynlib-stubf-prototype)
	       stub-constants
	       (dynlib-code-depth dynlib-stubf-prototype)
	       (if (natnump documentation)
		   (dynlib-make-library-pointer file documentation)
		 documentation))
	    (make-byte-code
	     (dynlib-code-arglist dynlib-stubf-prototype)
	     (dynlib-code-string dynlib-stubf-prototype)
	     stub-constants
	     (dynlib-code-depth dynlib-stubf-prototype)))))
    ;; Store the definition pointer:
    (aset stub-constants 1 (dynlib-make-library-pointer file defindex))
    (setq dynlib-defined-stubs (1+ dynlib-defined-stubs))
    ;; Define the function, use `defalias' to store `load-history' info:
    (defalias name stub-code)))

(defun defstubfi (name file defindex &optional documentation)
  ;; Create a stub definition for the interactive function NAME.
  (let* ((stub-constants
	  (make-vector 2 'dynlib-load-and-run-function))
	 (stub-code
	  (make-byte-code
	   ;; Arglist and code-string are shared with the prototype:
	   (dynlib-code-arglist dynlib-stubfi-prototype)
	   (dynlib-code-string dynlib-stubfi-prototype)
	   stub-constants
	   (dynlib-code-depth dynlib-stubfi-prototype)
	   (if (natnump documentation)
	       (dynlib-make-library-pointer file documentation)
	     documentation)
	   (dynlib-code-inter dynlib-stubfi-prototype))))
    ;; Store the definition pointer:
    (aset stub-constants 1 (dynlib-make-library-pointer file defindex))
    (setq dynlib-defined-stubs (1+ dynlib-defined-stubs))
    ;; Define the function, use `defalias' to store `load-history' info:
    (defalias name stub-code)))

(defun defstubm (name file defindex &optional documentation)
  ;; Create a stub definition for the macro NAME.
  (let* ((stub-constants
	  (make-vector 2 'dynlib-load-and-run-macro))
	 (stub-code
	  (if documentation
	      (make-byte-code
	       ;; Arglist and code-string are shared with the prototype:
	       (dynlib-code-arglist dynlib-stubm-prototype)
	       (dynlib-code-string dynlib-stubm-prototype)
	       stub-constants
	       (dynlib-code-depth dynlib-stubm-prototype)
	       (if (natnump documentation)
		   (dynlib-make-library-pointer file documentation)
		 documentation))
	    (make-byte-code
	     (dynlib-code-arglist dynlib-stubm-prototype)
	     (dynlib-code-string dynlib-stubm-prototype)
	     stub-constants
	     (dynlib-code-depth dynlib-stubm-prototype)))))
    ;; Store the definition pointer:
    (aset stub-constants 1 (dynlib-make-library-pointer file defindex))
    (setq dynlib-defined-stubs (1+ dynlib-defined-stubs))
    ;; Define the macro, use `defalias' to store `load-history' info:
    (defalias name (cons 'macro stub-code))))

;; Documentation definers:

(defun defdocv (name file index)
  ;; Creates a documentation pointer for the variable NAME.
  (put name 'variable-documentation
       (dynlib-make-library-pointer file index)))

(defun defdoca (name file index)
  ;; Creates a documentation pointer for the autoloaded function NAME.
  (let ((definition
	  (if (fboundp name)
	      (symbol-function name))))
    (if (eq (car-safe definition) 'autoload)
	(setcar (nthcdr 2 definition)
		(dynlib-make-library-pointer file index)))))

(defun dynlib-documentation (definition)
  ;; Returns the raw documentation object of DEFINITION.
  ;; We can't use the standard `documentation' or `safe-documentation'
  ;; functions, because they won't work on library pointers.
  (if (dynlib-macro-p definition)
      (setq definition (dynlib-lambdafy definition)))
  (let ((documentation
	 (cond ((byte-code-function-p definition)
		(dynlib-code-safe-doc definition))
	       ((eq (car-safe definition) 'autoload)
		(nth 2 definition))
	       ((dynlib-lambda-p definition)
		(car (cdr (cdr definition)))))))
    (if (or (stringp documentation)
	    (natnump documentation)
	    (and (dynlib-library-pointer-p documentation)
		 (not (dynlib-lambda-p definition))))
	documentation)))

(defun dynlib-unkwote (thing)
  (cond ((eq (car-safe thing) 'quote)
	 (car (cdr thing)))
	(t thing)))

;; When a stub definition is run we have to figure out who called it so
;; that we can redefine the caller with the real definition (if we could
;; modify the stub byte-code object destructively, we would be all set, but
;; that alley is blocked).  The only way to reliably figure out who called
;; the stub is by looking at the stack with `backtrace-frame'.  If we were
;; called via the function cell of a symbol we are lucky, because then we
;; can redefine that symbol with the actual function definition.  If we were
;; invoked via some apply/funcall construction that involved a complex
;; s-expression in the definition part, then we are out of luck.  In that
;; case we destructively modify the stub definition so that it runs the
;; actual definition somehow not too inefficiently.
;;
;; Here's how to figure out from the stack who invoked a stub definition:
;;
;; Scenario:
;;
;; (defun foo (n) ...)     ;; foo is a stub
;; (defmacro foom (n) ...) ;; foom is a stub
;; (fset 'ifoo 'foo)       ;; indirection
;; (fset 'iifoo 'ifoo)
;; (fset 'ifoom 'foom)
;; (fset 'iifoom 'ifoom)
;; (setq foovar (symbol-function 'foo))
;; (fset 'mexp 'macroexpand)
;;
;; Calling pattern            Stack frame prior to stub invocation
;;
;; (foo 1)                    (t foo 1)
;; (funcall 'foo 1)           (t foo 1)
;; (apply 'foo '(1))          (t foo 1)
;; (iifoo 1)                  (t iifoo 1)
;; (apply foovar '(1))        (t #[(&rest dynlib-ArgS) ...] 1)
;; (funcall foovar 1)         (t #[(&rest dynlib-ArgS) ...] 1)
;;
;;                            For macros we have to go 1 level deeper
;;
;; (foom 1)                   (nil foom 1)
;; (iifoom 1)                 (nil iifoom 1)
;; (macroexpand '(foom 1))    (t #<subr macroexpand> (foom 1) nil)
;; (macroexpand '(iifoom 1))  (t #<subr macroexpand> (iifoom 1) nil)
;; (mexp '(foom 1))           (t #<subr macroexpand> (foom 1) nil)
;; (mexp '(iifoom 1))         (t #<subr macroexpand> (iifoom 1) nil)

(defun dynlib-load-function (caller-frame &optional macro-p)
  ;; Loads the actual definition of a stub function according to CALLER-FRAME.
  ;; It retrieves the stub definition that invoked this function from the
  ;; caller frame, and if possible also the function symbol that is bound
  ;; to that stub.  It then loads the actual definition and redefines the
  ;; calling function if we had one.  Otherwise, it destructively modifies
  ;; the stub function so that it runs the actual definition from now on.
  (let* ((caller
	  (if (and macro-p
		   (car caller-frame))
	      ;; A macro called via `macroexpand':
	      (car (nth 2 caller-frame))
	    (car (cdr caller-frame))))
	 (stub-definition
	  (if (and (symbolp caller)
		   (fboundp caller))
	      (progn
		(setq caller (dynlib-real-caller caller))
		(symbol-function caller))
	    caller))
	 definition stub-constants)
    (if (not (dynlib-stub-definition-p stub-definition))
	(error "dynlib: Internal error, unknown caller frame format!"))
    (setq definition (dynlib-lookup-definition-from-stub stub-definition))
    (if (symbolp caller)
	;; Define the caller with its real definition:
	(fset caller definition)
      ;; Otherwise make the stub runnable because something else points to it:
      (setq stub-constants
	    (dynlib-code-constants
	     (if macro-p
		 (dynlib-lambdafy stub-definition)
	       stub-definition)))
      (aset stub-constants 0
	    (if macro-p 'dynlib-run-macro 'dynlib-run-function))
      ;; Replace the definition pointer with the actual definition:
      (aset stub-constants 1 definition))
    (setq dynlib-defined-functions (1+ dynlib-defined-functions))
    definition))

(defun dynlib-real-caller (caller)
  ;; Chases function indirection starting from CALLER until it finds the
  ;; name that points to a non-symbol definition object.
  (if (and (fboundp caller)
	   (symbolp (symbol-function caller)))
      (dynlib-real-caller (symbol-function caller))
    caller))

(defun dynlib-lookup-definition-from-stub (stub-definition)
  ;; Returns the real definition of STUB-DEFINITION.
  (let* ((macro-p (dynlib-macro-p stub-definition))
	 (stub-constants
	  (dynlib-code-constants
	   (if macro-p
	       (dynlib-lambdafy stub-definition)
	     stub-definition)))
	 (stub-documentation (dynlib-documentation stub-definition))
	 (definition-pointer
	   (dynlib-stub-definition-pointer stub-definition))
	 (definition
	   (car (read-from-string (dynlib-lookup definition-pointer)))))
    ;; Copy the documentation pointer from the stub to the actual definition:
    (if (dynlib-library-pointer-p stub-documentation)
	(dynlib-ncopy-library-pointer
	 stub-documentation (dynlib-documentation definition)))
    definition))

(defun dynlib-define-function (function)
  ;; Defines FUNCTION with its real definition if it is currently a stub.
  (let ((definition (symbol-function function)))
    (and (dynlib-stub-definition-p definition)
	 (defalias function (dynlib-lookup-definition-from-stub definition)))))

;; The actual defining functions are slightly complicated by the fact that
;; we don't want to shield any variables referenced by the defined function
;; when it gets applied to the calling arguments.  We use semi-cryptic
;; argument names to accomplish this:

(defun dynlib-run-function (dynlib-ArG-1 &optional dynlib-ArG-2)
  ;;                       (function-definition &optional interactive-p)
  (if dynlib-ArG-2
      (call-interactively dynlib-ArG-1)
    (apply dynlib-ArG-1 dynlib-ArgS)))

(defun dynlib-run-macro (dynlib-ArG-1)
  ;;                    (macro-definition)
  (fset 'dynlib-dummy-function dynlib-ArG-1)
  (macroexpand
   (cons 'dynlib-dummy-function dynlib-ArgS)))

(defun dynlib-load-and-run-function (dynlib-ArG-1 &optional dynlib-ArG-2)
  ;;                                (definition-pointer &opt interactive-p)
  (setq dynlib-ArG-1 (dynlib-load-function (backtrace-frame 2)))
  (if dynlib-ArG-2
      (call-interactively dynlib-ArG-1)
    (apply dynlib-ArG-1 dynlib-ArgS)))

(defun dynlib-load-and-run-macro (dynlib-ArG-1)
  ;;                             (definition-pointer)
  (fset 'dynlib-dummy-function (dynlib-load-function (backtrace-frame 3) t))
  (macroexpand
   (cons 'dynlib-dummy-function dynlib-ArgS)))

;; Define these, so we can compile this file even if Advice is
;; compiled as a dynamic library and this file is not yet loaded:
(eval-when-compile
  (autoload 'dynlib-find-library "dynlib-run")
  (autoload 'defstubf "dynlib-run")
  (autoload 'defstubfi "dynlib-run")
  (autoload 'defstubm "dynlib-run")
  (autoload 'defdocv "dynlib-run")
  (autoload 'defdoca "dynlib-run"))

;; Statistics:

(defun dynlib-stats ()
  "Displays some statistics about currently loaded Dynlib functions."
  (interactive)
  (message "Loaded stubs: %d   Defined functions: %d   Usage ratio: %s%%"
	   dynlib-defined-stubs
	   dynlib-defined-functions
	   (if (= dynlib-defined-stubs 0)
	       "0.0"
	     (substring
	      (format "%s    " (/ (* dynlib-defined-functions 100.0)
				  dynlib-defined-stubs))
	      0 4))))

;; Documentation lookup:

(defadvice documentation (around dynlib-docstring first freeze)
  "Retrieves the docstring if FUNCTION is a stub or library function."
  (let* ((definition
	   (if (symbolp (ad-get-arg 0))
	       (symbol-function (ad-get-arg 0))
	     (ad-get-arg 0)))
	 (documentation (dynlib-documentation definition)))
    (cond ((dynlib-library-pointer-p documentation)
	   (setq ad-return-value (dynlib-lookup documentation))
	   ;; Handle optional `raw' argument:
	   (if (not (ad-get-arg 1))
	       (setq ad-return-value
		     (substitute-command-keys ad-return-value))))
	  (t ad-do-it))))

(defadvice documentation-property (around dynlib-docstring first freeze)
  "Retrieves the docstring if the property value is a library pointer."
  (let* ((documentation (get (ad-get-arg 0) (ad-get-arg 1))))
    (cond ((dynlib-library-pointer-p documentation)
	   (setq ad-return-value (dynlib-lookup documentation))
	   ;; Handle optional `raw' argument: 
	   (if (not (ad-get-arg 2))
	       (setq ad-return-value
		     (substitute-command-keys ad-return-value))))
	  (t ad-do-it))))

;; Install a patch for Advice:
(defun dynlib-ad-has-proper-definition (function)
  (and (symbolp function)
       (fboundp function)
       (not (eq (car-safe (symbol-function function)) 'autoload))
       ;; If we have a stub return t, but load the real function first:
       (if (dynlib-stub-definition-p (symbol-function function))
	   (dynlib-define-function function)
	 t)))

(if (fboundp 'ad-has-proper-definition)
    (fset 'ad-has-proper-definition 'dynlib-ad-has-proper-definition)
  (eval-after-load
   "advice"
   '(fset 'ad-has-proper-definition 'dynlib-ad-has-proper-definition)))

;; Once `dynlib-run' is loaded there will be stubs around which need to
;; get properly loaded in case they are `defsubst's, i.e., they get
;; compiled inline.  For that reason we have to make sure that `dynlib-cmp'
;; is loaded whenever `bytecomp' is or gets loaded:
(if (and (not (featurep 'dynlib-cmp))
	 ;; In batch mode this would get into the way if the
	 ;; default `load-path' doesn't contain `dynlib-cmp':
	 (not noninteractive))
    (if (featurep 'byte-compile)
        ;; If the compiler is already loaded load `dynlib-cmp' right away:
        (require 'dynlib-cmp)
      ;; Otherwise load it right after the compiler gets loaded:
      (eval-after-load "bytecomp" '(require 'dynlib-cmp))))

(provide 'dynlib-run)

;;; dynlib-run.el ends here