From 8d2fcfea996541e3b713250c53bbb76b436b46bb Mon Sep 17 00:00:00 2001 From: Ilya Chernyshov Date: Mon, 22 Jan 2024 01:33:56 +0600 Subject: [PATCH] Add testing/lisp/test-duplicates-detector.el * testing/lisp/test-duplicates-detector.el: Add test unit that checks for duplicate ert-deftests and forms inside of them. --- testing/lisp/test-duplicates-detector.el | 317 +++++++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 testing/lisp/test-duplicates-detector.el diff --git a/testing/lisp/test-duplicates-detector.el b/testing/lisp/test-duplicates-detector.el new file mode 100644 index 000000000..58da27c07 --- /dev/null +++ b/testing/lisp/test-duplicates-detector.el @@ -0,0 +1,317 @@ +;;; test-duplicates-detector.el --- Tests for finding duplicates in Org tests -*- lexical-binding: t; -*- + +;; Copyright (C) 2023 Ilya Chernyshov +;; Authors: Ilya Chernyshov + +;; This file is not part of GNU Emacs. + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . +;; +;;; Commentary: + +;; Unit tests that check for duplicate forms and tests in all Org test files. + +;; Forms are considered duplicate if they: + +;; 1. are `equal-including-properties', +;; 2. have the same nesting path, +;; 3. either are `should-' macros or have `should-' macros inside. + +;; To ignore a form or a group of forms, wrap them in +;; `org-test-ignore-duplicate'. + +;; `ert-deftest' are considered duplicate if their body are +;; `equal-including-properties.' When comparing, the docstrings are not taken +;; into account. + +;;; Code: + +(require 'org-test "../testing/org-test") + +;;;; Variables + +(defvar test-duplicates-progn-forms + '( progn prog1 let dolist dotimes + org-test-with-temp-text + org-test-with-temp-text-in-file + org-test-at-id + org-test-ignore-duplicate) + "List of forms equivalent to `progn'. +Immediate children inside these are not checked for duplicates.") + +(defvar test-duplicates-detector-file-path + (expand-file-name "test-duplicates-detector.el" + (expand-file-name "lisp" org-test-dir))) + +(defvar test-duplicates-detector-files + (remove + test-duplicates-detector-file-path + (directory-files + (expand-file-name "lisp" org-test-dir) t "\\.el$"))) + +(defvar test-duplicates-detector-duplicate-forms nil + "A list where each element is either: + + ((file test-name [(form-1 . numerical-order) + (form-2 . numerical-order) ...]) + (dup-form-1 . (numerical-order [numerical-order ...])) + [ (dup-form-2 . (numerical-order [numerical-order ...])) + (dup-form-3 . (numerical-order [numerical-order ...])) + ...]) + +or + + (test-1-symbol . duplicate-of-test-1-symbol) + + +Where + + (file test-name [(form-1 . numerical-order) + (form-2 . numerical-order) ...]) + +is a path to duplicates. For example, the path for the +duplicates in the following test: + + test-file.el + + (ertdeftest test-name () + \"Docstring.\" + (let ((var-1 \"value\")) + (when var-1 + (should-not + (equal 2 (some-func \"string\" \"x\" nil))) + (some-func \"string\" \"x=2\") + (should-not + (equal 2 (some-func \"string\" \"x\" nil))) + (some-func \"string\" \"x=2\")))) + +would look like this: + + (\"/absolute/path/to/test-file.el\" + test-name + (let . 4) (when . 2)) + +And the records about the duplicates would look like this: + + ((should-not + (equal 2 (some-func \"string\" \"x\" nil))) 4 2)") + +(defvar test-duplicates-detector-forms nil + "Nested alist of found forms and paths to them (not filtered).") + +;;;; Macros + +(defmacro org-test-ignore-duplicate (&rest body) + "Eval BODY forms sequentially and return value of last one. + +The macro's body will be ignored by `test-duplicates-detector.el' +tests to skip duplicate forms inside the body." + (declare (indent 0)) + `(progn ,@body)) + +;;;; ERT tests + +(ert-deftest test-org-tests/find-duplicates () + "Try to find duplicate forms and ert-deftests in FILES." + (should-not + (test-duplicates-detector--find-duplicates + test-duplicates-detector-files))) + +;;;; Auxiliary functions + +(defun test-duplicates-detector--find-duplicates (files) + "Try to find duplicate forms and ert-deftests in FILES. + +Duplicate forms will be written to +`test-duplicates-detector-duplicate-forms'. + +`message' paths to them in a human-readable format." + (setq test-duplicates-detector-forms nil) + (let (found-deftests duplicate-tests) + (dolist (file files) + (with-current-buffer (find-file-noselect file) + (save-excursion + (goto-char (point-min)) + (while (search-forward "(ert-deftest" nil t) + (goto-char (match-beginning 0)) + (ignore-errors + (while-let ((deftest (or (read (current-buffer)) t)) + ((eq (car deftest) 'ert-deftest)) + (test-name (cadr deftest))) + (if-let ((f (seq-find + (lambda (x) + (equal-including-properties + ;; if cadddr is a docstring + (if (stringp (cadddr deftest)) + (cddddr deftest) + (cdddr deftest)) + (if (stringp (cadddr x)) + (cddddr x) + (cdddr x)))) + found-deftests))) + (push (cons test-name (cadr f)) duplicate-tests) + (push deftest found-deftests) + (test-duplicates-detector--search-forms-recursively + deftest (list file test-name))))))))) + (setq test-duplicates-detector-duplicate-forms + (seq-filter + #'cdr + (mapcar + (lambda (file) + (cons + (car file) + (seq-filter + (lambda (x) + (and (caddr x) + (seq-intersection + '(should-not should should-error) + (flatten-list (car x))))) + (cdr file)))) + test-duplicates-detector-forms))) + (when test-duplicates-detector-duplicate-forms + (message + "Found duplicates (To ignore the duplicate forms, +wrap them in `org-test-ignore-duplicate'): +%s" + (mapconcat + (lambda (path) + (let* ((file (file-relative-name (caar path))) + (test-name (symbol-name (cadar path))) + (string-path (append (list file test-name) + (mapcar (lambda (x) + (symbol-name (car x))) + (cddar path)))) + (indent -1) + (print-level 3)) + (concat + (mapconcat + (lambda (x) + (concat (make-string (* (setq indent (1+ indent)) 2) ? ) + x "\n")) + string-path) + (mapconcat + (lambda (x) + (format "%s%S: %d times\n" + (make-string (* indent 2) ? ) + (car x) + (length (cdr x)))) + (cdr path))))) + test-duplicates-detector-duplicate-forms))) + (when duplicate-tests + (message "Duplicate ERT tests found:\n%s\n" + (mapconcat (lambda (x) (format "%S" x)) + duplicate-tests "\n"))) + (append test-duplicates-detector-duplicate-forms + duplicate-tests))) + +(defun test-duplicates-detector--search-forms-recursively (form form-path) + "Search for forms recursively in FORM. + +FORM-PATH is list of the form: + (\"file-path\" ert-test-symbol + (symbol-1 . sexp-order-1) (symbol-2 . sexp-order-2)) + +Write each form to `test-duplicates-detector-forms'" + (let ((idx 0)) + (dolist (sub-form form) + (when (consp sub-form) + (unless (memq (car-safe form) test-duplicates-progn-forms) + (push idx (alist-get + sub-form + (alist-get form-path test-duplicates-detector-forms + nil nil #'equal) + nil nil #'equal-including-properties))) + (unless (memq (car sub-form) + '(should-not should should-error)) + (test-duplicates-detector--search-forms-recursively + sub-form + (append form-path (list (cons (car sub-form) idx)))))) + (cl-incf idx)))) + +;;;; Testing the detector itself + +(ert-deftest test-org-tests/test-duplicates-detector-testing-find-duplicates () + "Test `test-duplicates-detector--find-duplicates'." + (should + (equal + (test-duplicates-detector--find-duplicates + (list test-duplicates-detector-file-path)) + `(((,test-duplicates-detector-file-path + test-org-tests/test-with-nested-duplicates) + ((let ((var "string")) (should (message "123 %s" var))) 6 4)) + ((,test-duplicates-detector-file-path + test-org-tests/test-with-duplicates-at-root) + ((should (message "123")) 6 4)) + (test-org-tests/duplicate-test-2 . test-org-tests/duplicate-test-1))))) + +;;;;; Tests with duplicate forms + +(ert-deftest test-org-tests/test-with-duplicates-at-root () + "Test with duplicates at the root." + (should (message "123")) + (format "%s" "string") + (should + (message "123"))) + +(ert-deftest test-org-tests/test-with-nested-duplicates () + "Test with nested duplicates." + (let ((var "string")) + (should + (message "123 %s" var))) + (format "%s" "string") + (let ((var "string")) + (should (message "123 %s" var))) + (format "%s" "string")) + +;;;;; Tests without duplicates + +(ert-deftest test-org-tests/test-without-duplicates-1 () + "Test without duplicates." + (let ((var-1 "asd")) + (concat "string" var-1)) + (should + (let ((var-1 "asd")) + (concat "string" var-1)))) + +(ert-deftest test-org-tests/test-without-duplicates-2 () + "Test without duplicates. +Equal `should' macros, but different nesting paths." + (let ((var "string")) + (should (format "123 %s" "asd"))) + (+ 5 6 9) + (should (format "123 %s" "asd"))) + +;;;;; Duplicate deftests (maybe different names, but same body) + +(ert-deftest test-org-tests/duplicate-test-1 () + "Docstring of duplicate-test-1." + (let ((var 99)) + (+ 5 6 9 var) + (should (format "123 %s" "asd"))) + (should (format "123 %s" "asd"))) + +(ert-deftest test-org-tests/duplicate-test-2 () + "Docstring of duplicate-test-2." + (let ((var 99)) + (+ 5 6 9 var) + (should (format "123 %s" "asd"))) + (should (format "123 %s" "asd"))) + +(provide 'test-duplicates-detector) + +;; Local Variables: +;; outline-regexp: "\\(;\\{3,\\} \\)" +;; End: + +;;; test-duplicates-detector.el ends here