testing: allow to select tests

* mk/default.mk: Add default for new variable BTEST_RE to select all
  tests.  (BTEST_OB_LANGUAGES): Remove sh (always needed, as
  emacs-lisp) and add comment for ruby.  (BTEST): Use BTEST_RE to
  select tests from the test suite.  Reorganize pre-loading of Org,
  Babel, Ox and babel languages into a known clean environment.

* mk/targets.mk (CONF_TEST): Add BTEST_RE to the output of
  config-test.

* testing/org-batch-test-init.el: New file.  Remove all traces of any
  Org built into Emacs or otherwise present in the environment so it
  can not be picked up spuriously during testing or conceal errors in
  the Org version under test.

* testing/org-test.el (org-test-run-batch-tests): Showtest selection
  in messages.  (org-test-run-all-tests): Update ID locations.

mk/default.mk

@@ -39,8 +39,9 @@ BTEST_POST  =
               # -L <path-to>/ert      # needed for Emacs23, Emacs24 has ert built in
               # -L <path-to>/ess      # needed for running R tests
               # -L <path-to>/htmlize  # need at least version 1.34 for source code formatting
-BTEST_OB_LANGUAGES = awk C fortran maxima lilypond octave python sh perl
+BTEST_OB_LANGUAGES = awk C fortran maxima lilypond octave perl python
               # R                     # requires ESS to be installed and configured
+              # ruby                  # requires inf-ruby to be installed and configured
 # extra packages to require for testing
 BTEST_EXTRA =
               # ess-site  # load ESS for R tests
@@ -50,18 +51,26 @@ BTEST_EXTRA =
 
 # How to run tests
 req-ob-lang = --eval '(require '"'"'ob-$(ob-lang))'
+lst-ob-lang = ($(ob-lang) . t)
 req-extra   = --eval '(require '"'"'$(req))'
+BTEST_RE ?= \\(org\\|ob\\)
 BTEST	= $(BATCH) \
 	  $(BTEST_PRE) \
-	  --eval '(add-to-list '"'"'load-path "./lisp")' \
-	  --eval '(add-to-list '"'"'load-path "./testing")' \
+	  --eval '(add-to-list '"'"'load-path (concat default-directory "lisp"))' \
+	  --eval '(add-to-list '"'"'load-path (concat default-directory "testing"))' \
 	  $(BTEST_POST) \
+	  -l org-batch-test-init \
+	  --eval '(setq \
+		org-batch-test t \
+		org-babel-load-languages \
+	          (quote ($(foreach ob-lang,$(BTEST_OB_LANGUAGES) emacs-lisp sh org,$(lst-ob-lang)))) \
+	    org-test-select-re "$(BTEST_RE)" \
+	  )' \
 	  -l org-loaddefs.el \
-	  -l testing/org-test.el \
-	  $(foreach ob-lang,$(BTEST_OB_LANGUAGES),$(req-ob-lang)) \
+	  -l cl -l testing/org-test.el \
+	  -l ert -l org -l ox \
 	  $(foreach req,$(BTEST_EXTRA),$(req-extra)) \
-	  --eval '(setq org-confirm-babel-evaluate nil)' \
-	  -f org-test-run-batch-tests
+	  --eval '(org-test-run-batch-tests org-test-select-re)'
 
 # Using emacs in batch mode.
 # BATCH = $(EMACS) -batch -vanilla # XEmacs

mk/targets.mk

@@ -35,7 +35,7 @@ endif
 
 CONF_BASE = EMACS DESTDIR ORGCM ORG_MAKE_DOC
 CONF_DEST = lispdir infodir datadir testdir
-CONF_TEST = BTEST_PRE BTEST_POST BTEST_OB_LANGUAGES BTEST_EXTRA
+CONF_TEST = BTEST_PRE BTEST_POST BTEST_OB_LANGUAGES BTEST_EXTRA BTEST_RE
 CONF_EXEC = CP MKDIR RM RMR FIND SUDO PDFTEX TEXI2PDF TEXI2HTML MAKEINFO INSTALL_INFO
 CONF_CALL = BATCH BATCHL ELC ELCDIR BTEST MAKE_LOCAL_MK MAKE_ORG_INSTALL MAKE_ORG_VERSION
 config-eol:: EOL = \#

testing/org-batch-test-init.el

@@ -0,0 +1,20 @@
+;;
+;; Remove Org remnants built into Emacs
+;;
+
+;; clean load-path
+(setq load-path
+      (delq nil (mapcar
+		 (function (lambda (p)
+			     (unless (string-match "lisp\\(/packages\\)?/org$" p)
+			       p)))
+		 load-path)))
+;; remove property list to defeat cus-load and remove autoloads
+(mapatoms (function  (lambda (s)
+		       (let ((sn (symbol-name s)))
+			 (when (string-match "^\\(org\\|ob\\|ox\\)\\(-.*\\)?$" sn)
+			   (setplist s nil)
+			   (when (eq 'autoload (car-safe s))
+			     (unintern s)))))))
+
+;; we should now start from a clean slate

testing/org-test.el

@@ -424,6 +424,7 @@ Load all test files first."
     (org-test-touch-all-examples)
     (org-test-update-id-locations)
     (org-test-load)
+    (message "selected tests: %s" org-test-selector)
     (ert-run-tests-batch-and-exit org-test-selector)))
 
 (defun org-test-run-all-tests ()
@@ -431,6 +432,7 @@ Load all test files first."
 Load all test files first."
   (interactive)
   (org-test-touch-all-examples)
+  (org-test-update-id-locations)
   (org-test-load)
   (ert "\\(org\\|ob\\)")
   (org-test-kill-all-examples))