[test] Separate test runner configuration from main compiler config

.circleci/config.yml

@@ -96,10 +96,11 @@ commands:
               wget https://nodejs.org/dist/v${version}/node-v${version}-linux-x64.tar.xz
             fi
             tar xf node-v${version}-linux-x64.tar.xz
-            echo "NODE_JS_TEST = '~/node-v${version}-linux-x64/bin/node'" >> ~/emsdk/.emscripten
-            echo "JS_ENGINES = [NODE_JS_TEST]" >> ~/emsdk/.emscripten
-            echo "if os.path.exists(os.path.expanduser(V8_ENGINE)): JS_ENGINES.append(V8_ENGINE)" >> ~/emsdk/.emscripten
-            cat ~/emsdk/.emscripten
+            cd -
+            echo "NODE_JS_TEST = '~/node-v${version}-linux-x64/bin/node'" >> test/config.py
+            echo "JS_ENGINES = [NODE_JS_TEST]" >> test/config.py
+            echo "if os.path.exists(V8_ENGINE): JS_ENGINES.append(V8_ENGINE)" >> test/config.py
+            cat test/config.py
             echo "export PATH=\"$HOME/node-v${version}-linux-x64/bin:\$PATH\"" >> $BASH_ENV
   install-node-oldest:
     description: "install node 18.3.0 (oldest)"
@@ -155,21 +156,25 @@ commands:
             # persistent workspace and to avoid any confusion with the version
             # we are trying to test.
             rm -Rf emscripten
-            echo "import os" >> .emscripten
             # We use an out-of-tree cache directory so it can be part of the
             # persisted workspace (see below).
-            echo "CACHE = os.path.expanduser('~/cache')" >> .emscripten
-            # Refer to commit 0bc3640 if we need to pin V8 version.
-            echo "V8_ENGINE = '~/.jsvu/bin/v8'" >> .emscripten
-            echo "JS_ENGINES = [NODE_JS]" >> .emscripten
-            echo "if os.path.exists(os.path.expanduser(V8_ENGINE)): JS_ENGINES.append(V8_ENGINE)" >> .emscripten
-            echo "WASM_ENGINES = []" >> .emscripten
-            test -f ~/vms/wasmtime && echo "WASMTIME = '~/vms/wasmtime'; WASM_ENGINES.append(WASMTIME)" >> .emscripten || true
-            test -f ~/vms/wasmer && echo "WASMER = '~/vms/wasmer'; WASM_ENGINES.append(WASMER)" >> .emscripten || true
-            test -d ~/wasi-sdk && cp -a ~/wasi-sdk/lib/ $(~/emsdk/upstream/bin/clang -print-resource-dir)
-            cd -
+            echo "CACHE = '~/cache'" >> .emscripten
+
             echo "final .emscripten:"
             cat ~/emsdk/.emscripten
+
+            # Refer to commit 0bc3640 if we need to pin V8 version.
+            cd -
+            echo "import os" >> test/config.py
+            echo "V8_ENGINE = '~/.jsvu/bin/v8'" >> test/config.py
+            echo "JS_ENGINES = [NODE_JS]" >> test/config.py
+            echo "if os.path.exists(V8_ENGINE): JS_ENGINES.append(V8_ENGINE)" >> test/config.py
+            echo "WASM_ENGINES = []" >> test/config.py
+            test -f ~/vms/wasmtime && echo "WASMTIME = '~/vms/wasmtime'; WASM_ENGINES.append(WASMTIME)" >> test/config.py || true
+            test -f ~/vms/wasmer && echo "WASMER = '~/vms/wasmer'; WASM_ENGINES.append(WASMER)" >> test/config.py || true
+            test -d ~/wasi-sdk && cp -a ~/wasi-sdk/lib/ $(~/emsdk/upstream/bin/clang -print-resource-dir)
+            echo "final test/config.py:"
+            cat test/config.py
       - emsdk-env
       - bootstrap
   build-libs:
@@ -752,7 +757,7 @@ jobs:
       # tests).
       - run:
           name: configure node (newest)
-          command: echo "NODE_JS = NODE_JS_TEST" >> ~/emsdk/.emscripten
+          command: echo "NODE_JS = '$(which node)'" >> ~/emsdk/.emscripten
       - run-tests:
           title: "wasm64_4gb"
           test_targets: "wasm64_4gb"
@@ -769,7 +774,7 @@ jobs:
       # tests).
       - run:
           name: configure node (newest)
-          command: echo "NODE_JS = NODE_JS_TEST" >> ~/emsdk/.emscripten
+          command: echo "NODE_JS = '$(which node)'" >> ~/emsdk/.emscripten
       - run-tests:
           title: "wasm64"
           test_targets: "
@@ -799,8 +804,8 @@ jobs:
           name: install bun
           command: |
             curl -fsSL https://bun.com/install | bash
-            echo "BUN = '~/.bun/bin/bun'" >> ~/emsdk/.emscripten
-            echo "JS_ENGINES = [BUN]" >> ~/emsdk/.emscripten
+            echo "BUN = '~/.bun/bin/bun'" >> test/config.py
+            echo "JS_ENGINES = [BUN]" >> test/config.py
       - run-tests:
           test_targets: "--crossplatform-only"
   test-deno:
@@ -815,8 +820,8 @@ jobs:
           name: install deno
           command: |
             curl -fsSL https://deno.land/install.sh | sh
-            echo "DENO = '~/.deno/bin/deno'" >> ~/emsdk/.emscripten
-            echo "JS_ENGINES = [DENO]" >> ~/emsdk/.emscripten
+            echo "DENO = '~/.deno/bin/deno'" >> test/config.py
+            echo "JS_ENGINES = [DENO]" >> test/config.py
       - run-tests:
           test_targets: "--crossplatform-only"
   test-jsc:
@@ -834,8 +839,8 @@ jobs:
             export PATH="`pwd`/node-v18.12.0-linux-x64/bin:${PATH}"
             npm install jsvu -g
             jsvu --os=default --engines=javascriptcore
-            echo "JSC_ENGINE = '~/.jsvu/bin/javascriptcore'" >> ~/emsdk/.emscripten
-            echo "JS_ENGINES = [JSC_ENGINE]" >> ~/emsdk/.emscripten
+            echo "JSC_ENGINE = '~/.jsvu/bin/javascriptcore'" >> test/config.py
+            echo "JS_ENGINES = [JSC_ENGINE]" >> test/config.py
       - run-tests:
           test_targets: "
             core0.test_hello_argc
@@ -856,8 +861,8 @@ jobs:
             export PATH="`pwd`/node-v18.12.0-linux-x64/bin:${PATH}"
             npm install jsvu -g
             jsvu --os=default --engines=spidermonkey
-            echo "SPIDERMONKEY_ENGINE = '~/.jsvu/bin/spidermonkey'" >> ~/emsdk/.emscripten
-            echo "JS_ENGINES = [SPIDERMONKEY_ENGINE]" >> ~/emsdk/.emscripten
+            echo "SPIDERMONKEY_ENGINE = '~/.jsvu/bin/spidermonkey'" >> test/config.py
+            echo "JS_ENGINES = [SPIDERMONKEY_ENGINE]" >> test/config.py
       - run-tests:
           test_targets: "
             other.test_prepost_jspi

.gitignore

@@ -2,6 +2,9 @@
 .emscripten
 .emscripten_backup
 
+# Test config file
+test/config.py
+
 # Emsdk generated version info
 .emsdk_version
 

site/source/docs/getting_started/test-suite.rst

@@ -20,6 +20,11 @@ To run the tests, you need an emscripten setup, as it will run ``emcc`` and othe
 commands. See the :ref:`developer's guide <developers-guide-setting-up>` for
 how best to do that.
 
+By default, the test suite uses Node.js to run JavaScript tests. If you want to
+run tests with additional JS engines (such as V8/d8 or SpiderMonkey) or Wasm
+engines (such as Wasmer or Wasmtime), you can customize ``test/config.py`` (which
+is automatically created from ``test/config_template.py`` on the first test run).
+
 Running tests
 =============
 

test/benchmark/benchmark_sse.py

@@ -18,9 +18,8 @@
 sys.path.insert(0, __testdir__)
 
 import clang_native
-from common import EMRUN, test_file
+from common import EMRUN, test_config, test_file
 
-from tools import config
 from tools.shared import CLANG_CXX, EMCC
 from tools.utils import WINDOWS, run_process, write_file
 
@@ -56,7 +55,7 @@ def run_benchmark(benchmark_file, results_file, build_args):
     print(' '.join(cmd))
     run_process(cmd)
 
-    cmd = config.V8_ENGINE + [os.path.basename(out_file)]
+    cmd = test_config.V8_ENGINE + [os.path.basename(out_file)]
     print(' '.join(cmd))
     wasm_results = subprocess.check_output(cmd, cwd=out_dir, text=True)
 

test/common.py

@@ -41,6 +41,12 @@
   write_binary,
 )
 
+test_config_file = path_from_root('test/config.py')
+if not os.path.isfile(test_config_file):
+  shutil.copyfile(path_from_root('test/config_template.py'), test_config_file)
+
+import config as test_config
+
 logger = logging.getLogger('common')
 
 # If we are drawing a parallel swimlane graph of test output, we need to use a temp
@@ -84,19 +90,18 @@
 def setup_test_config():
   assert config.NODE_JS # assert for mypy's benefit
   # By default we run the tests in the same version of node as emscripten itself used.
-  if not config.NODE_JS_TEST:
-    config.NODE_JS_TEST = config.NODE_JS
-  # The default set of JS_ENGINES contains just node.
-  if not config.JS_ENGINES:
-    config.JS_ENGINES = [config.NODE_JS_TEST]
-  if not config.WASM_ENGINES:
-    config.WASM_ENGINES = []
+  if not getattr(test_config, 'NODE_JS_TEST', None):
+    test_config.NODE_JS_TEST = config.NODE_JS
+  if not getattr(test_config, 'JS_ENGINES', None):
+    test_config.JS_ENGINES = [test_config.NODE_JS_TEST]
+  if not getattr(test_config, 'WASM_ENGINES', None):
+    test_config.WASM_ENGINES = []
 
-  config.SPIDERMONKEY_ENGINE = config.listify(config.SPIDERMONKEY_ENGINE)
-  config.NODE_JS_TEST = config.listify(config.NODE_JS_TEST)
-  config.V8_ENGINE = config.listify(config.V8_ENGINE)
-  config.JS_ENGINES = [config.listify(e) for e in config.JS_ENGINES]
-  config.WASM_ENGINES = [config.listify(e) for e in config.WASM_ENGINES]
+  test_config.SPIDERMONKEY_ENGINE = config.listify(getattr(test_config, 'SPIDERMONKEY_ENGINE', None))
+  test_config.NODE_JS_TEST = config.listify(getattr(test_config, 'NODE_JS_TEST', None))
+  test_config.V8_ENGINE = config.listify(getattr(test_config, 'V8_ENGINE', None))
+  test_config.JS_ENGINES = [config.listify(e) for e in test_config.JS_ENGINES]
+  test_config.WASM_ENGINES = [config.listify(e) for e in test_config.WASM_ENGINES]
 
 
 setup_test_config()
@@ -320,7 +325,7 @@ def engine_is_bun(engine):
 
 def get_engine(predicate):
   """Return engine that satifies predicate, if one is configured, otherwise None."""
-  for engine in config.JS_ENGINES:
+  for engine in test_config.JS_ENGINES:
     if predicate(engine):
       return engine
   return None
@@ -735,7 +740,7 @@ def setUpClass(cls):
 
   def setUp(self):
     super().setUp()
-    self.js_engines = config.JS_ENGINES.copy()
+    self.js_engines = test_config.JS_ENGINES.copy()
     self.settings_mods = {}
     self.skip_exec = None
     self.flaky = False
@@ -770,7 +775,7 @@ def setUp(self):
     self.env = {}
     self.temp_files_before_run = []
     self.required_engine = None
-    self.wasm_engines = config.WASM_ENGINES.copy()
+    self.wasm_engines = test_config.WASM_ENGINES.copy()
     self.use_all_engines = EMTEST_ALL_ENGINES
     engine = self.get_current_js_engine()
     if not engine_is_node(engine) and not engine_is_bun(engine) and not engine_is_deno(engine):

test/config_template.py

@@ -0,0 +1,19 @@
+# Configuration file for emscripten test suite.
+#
+# Alternative JS engines to use during testing:
+#
+# NODE_JS_TEST = 'node' # executable
+# SPIDERMONKEY_ENGINE = ['js'] # executable
+# V8_ENGINE = 'd8' # executable
+#
+# All JS engines to use when running the automatic tests. Not all the engines in
+# this list must exist (if they don't, they will be skipped in the test runner).
+#
+# JS_ENGINES = [NODE_JS_TEST] # add V8_ENGINE or SPIDERMONKEY_ENGINE if you have them installed too.
+#
+# WASMER = 'wasmer'
+# WASMTIME = 'wasmtime'
+#
+# Wasm engines to use in STANDALONE_WASM tests.
+#
+# WASM_ENGINES = [] # add WASMER or WASMTIME if you have them installed

test/runner.py

@@ -40,15 +40,12 @@
 __rootpath__ = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, __rootpath__)
 
-# Let config.py know to parse test config settings
-os.environ['_EM_TEST_RUNNER'] = '1'
-
 import browser_common
 import common
 import jsrun
 import parallel_testsuite
 from color_runner import ColorTextRunner
-from common import errlog
+from common import errlog, test_config
 from single_line_runner import SingleLineTestRunner
 
 from tools import building, colored_logger, config, shared, utils
@@ -137,7 +134,7 @@
 
 
 def check_js_engines():
-  if not all(jsrun.check_engine(e) for e in config.JS_ENGINES):
+  if not all(jsrun.check_engine(e) for e in test_config.JS_ENGINES):
     errlog('Not all the JS engines in JS_ENGINES appear to work.')
     sys.exit(1)
 
@@ -619,7 +616,7 @@ def log_test_environment():
   node_js_version = utils.run_process(config.NODE_JS + ['--version'], stdout=subprocess.PIPE).stdout.strip()
   print(f'NODE_JS: {config.NODE_JS}. Version: {node_js_version}')
 
-  print(f'JS_ENGINES: {config.JS_ENGINES}')
+  print(f'JS_ENGINES: {test_config.JS_ENGINES}')
   print(f'BINARYEN_ROOT: {config.BINARYEN_ROOT}')
   wasm_opt_version = building.get_binaryen_version(building.get_binaryen_bin()).strip()
   print(f'wasm-opt version: {wasm_opt_version}')

test/test_benchmark.py

@@ -24,7 +24,7 @@
 import clang_native
 import common
 import jsrun
-from common import copy_asset, read_binary, read_file, test_file
+from common import copy_asset, read_binary, read_file, test_config, test_file
 from decorators import needs_make, parameterized
 
 from tools import utils
@@ -385,7 +385,7 @@ def get_output_files(self):
 # avoid the baseline compiler running, because it adds a lot of noise
 # (the nondeterministic time it takes to get to the full compiler ends up
 # mattering as much as the actual benchmark)
-aot_v8 = (config.V8_ENGINE if config.V8_ENGINE else []) + ['--no-liftoff']
+aot_v8 = (test_config.V8_ENGINE if test_config.V8_ENGINE else []) + ['--no-liftoff']
 
 named_benchmarkers = {
   'clang': NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]),
@@ -397,12 +397,12 @@ def get_output_files(self):
   'v8-lto': EmscriptenBenchmarker('v8-lto', aot_v8, ['-flto']),
   'v8-ctors': EmscriptenBenchmarker('v8-ctors', aot_v8, ['-sEVAL_CTORS']),
   'v8-64': EmscriptenBenchmarker('v8-64', aot_v8, ['-m64']),
-  'node': EmscriptenBenchmarker('node', config.NODE_JS_TEST),
-  'node-64': EmscriptenBenchmarker('node-64', config.NODE_JS_TEST, ['-m64']),
+  'node': EmscriptenBenchmarker('node', test_config.NODE_JS_TEST),
+  'node-64': EmscriptenBenchmarker('node-64', test_config.NODE_JS_TEST, ['-m64']),
   'cherp-v8': CheerpBenchmarker('cheerp-v8-wasm', aot_v8),
   # TODO: ensure no baseline compiler is used, see v8
-  'sm': EmscriptenBenchmarker('sm', config.SPIDERMONKEY_ENGINE),
-  'cherp-sm': CheerpBenchmarker('cheerp-sm-wasm', config.SPIDERMONKEY_ENGINE),
+  'sm': EmscriptenBenchmarker('sm', test_config.SPIDERMONKEY_ENGINE),
+  'cherp-sm': CheerpBenchmarker('cheerp-sm-wasm', test_config.SPIDERMONKEY_ENGINE),
   'clang-build': ToolchainBenchmarker('clang', [CLANG_CC]),
   'emcc-build': ToolchainBenchmarker('emcc', [EMCC]),
 }

test/test_core.py

@@ -36,6 +36,7 @@
   path_from_root,
   read_binary,
   read_file,
+  test_config,
   test_file,
 )
 from decorators import (
@@ -79,7 +80,7 @@
   with_env_modify,
 )
 
-from tools import building, config, shared, utils, webassembly
+from tools import building, shared, utils, webassembly
 from tools.shared import EMAR, EMCC, EMXX, FILE_PACKAGER, LLVM_COV, LLVM_PROFDATA, PIPE
 from tools.utils import LINUX, MACOS, WINDOWS, delete_file, write_file
 
@@ -2943,7 +2944,7 @@ def test_dlfcn_missing(self):
       }
       '''
 
-    if self.get_current_js_engine() == config.V8_ENGINE:
+    if self.get_current_js_engine() == test_config.V8_ENGINE:
       expected = "error: could not load dynamic lib: libfoo.so\nError: Error reading file"
     else:
       expected = "error: could not load dynamic lib: libfoo.so\nError: ENOENT: no such file or directory"
@@ -7432,8 +7433,8 @@ def test_demangle_stacks_symbol_map(self):
     self.assertIsNotNone(full_aborter)
     self.assertIsNotNone(short_aborter)
     print('full:', full_aborter, 'short:', short_aborter)
-    if config.SPIDERMONKEY_ENGINE:
-      output = self.run_js('test_demangle_stacks.js', engine=config.SPIDERMONKEY_ENGINE, assert_returncode=NON_ZERO)
+    if test_config.SPIDERMONKEY_ENGINE:
+      output = self.run_js('test_demangle_stacks.js', engine=test_config.SPIDERMONKEY_ENGINE, assert_returncode=NON_ZERO)
       # we may see the full one, if -g, or the short one if not
       if ' ' + short_aborter + ' ' not in output and ' ' + full_aborter + ' ' not in output:
         # stack traces may also be ' name ' or 'name@' etc