From 94889357bd4f9fc1275e4b33441c02224546cadf Mon Sep 17 00:00:00 2001
From: Magnus Ihse Bursie <ihse@openjdk.org>
Date: Thu, 24 Jan 2019 11:15:31 +0100
Subject: [PATCH] 8217634: RunTest documentation and usability update

Reviewed-by: erikj
---
 doc/testing.html  | 39 ++++++++++++++++++++---
 doc/testing.md    | 80 +++++++++++++++++++++++++++++++++++++++++++++--
 make/RunTests.gmk | 40 ++++++++++++++----------
 3 files changed, 134 insertions(+), 25 deletions(-)

diff --git a/doc/testing.html b/doc/testing.html
index 52999754726..ef258db25df 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -34,6 +34,7 @@
 </ul></li>
 <li><a href="#test-results-and-summary">Test results and summary</a></li>
 <li><a href="#test-suite-control">Test suite control</a><ul>
+<li><a href="#general-keywords-test_opts">General keywords (TEST_OPTS)</a></li>
 <li><a href="#jtreg-keywords">JTReg keywords</a></li>
 <li><a href="#gtest-keywords">Gtest keywords</a></li>
 <li><a href="#microbenchmark-keywords">Microbenchmark keywords</a></li>
@@ -102,11 +103,29 @@ TEST FAILURE</code></pre>
 <p>To separate multiple keyword=value pairs, use <code>;</code> (semicolon). Since the shell normally eats <code>;</code>, the recommended usage is to write the assignment inside qoutes, e.g. <code>JTREG=&quot;...;...&quot;</code>. This will also make sure spaces are preserved, as in <code>JTREG=&quot;VM_OPTIONS=-XshowSettings -Xlog:gc+ref=debug&quot;</code>.</p>
 <p>(Other ways are possible, e.g. using backslash: <code>JTREG=JOBS=1\;TIMEOUT=8</code>. Also, as a special technique, the string <code>%20</code> will be replaced with space for certain options, e.g. <code>JTREG=VM_OPTIONS=-XshowSettings%20-Xlog:gc+ref=debug</code>. This can be useful if you have layers of scripts and have trouble getting proper quoting of command line arguments through.)</p>
 <p>As far as possible, the names of the keywords have been standardized between test suites.</p>
-<h3 id="jtreg-keywords">JTReg keywords</h3>
+<h3 id="general-keywords-test_opts">General keywords (TEST_OPTS)</h3>
+<p>Some keywords are valid across different test suites. If you want to run tests from multiple test suites, or just don't want to care which test suite specific control variable to use, then you can use the general TEST_OPTS control variable.</p>
+<p>There are also some keywords that applies globally to the test runner system, not to any specific test suites. These are also available as TEST_OPTS keywords.</p>
 <h4 id="jobs">JOBS</h4>
+<p>Currently only applies to JTReg.</p>
+<h4 id="timeout_factor">TIMEOUT_FACTOR</h4>
+<p>Currently only applies to JTReg.</p>
+<h4 id="vm_options">VM_OPTIONS</h4>
+<p>Applies to JTReg, GTest and Micro.</p>
+<h4 id="java_options">JAVA_OPTIONS</h4>
+<p>Applies to JTReg, GTest and Micro.</p>
+<h4 id="aot_modules">AOT_MODULES</h4>
+<p>Applies to JTReg and GTest.</p>
+<h4 id="jcov">JCOV</h4>
+<p>This keywords applies globally to the test runner system. If set to <code>true</code>, it enables JCov coverage reporting for all tests run. To be useful, the JDK under test must be run with a JDK built with JCov instrumentation (<code>configure --with-jcov=&lt;path to directory containing lib/jcov.jar&gt;</code>, <code>make jcov-image</code>).</p>
+<p>The simplest way to run tests with JCov coverage report is to use the special target <code>jcov-test</code> instead of <code>test</code>, e.g. <code>make jcov-test TEST=jdk_lang</code>. This will make sure the JCov image is built, and that JCov reporting is enabled.</p>
+<p>The JCov report is stored in <code>build/$BUILD/test-results/jcov-output</code>.</p>
+<p>Please note that running with JCov reporting can be very memory intensive.</p>
+<h3 id="jtreg-keywords">JTReg keywords</h3>
+<h4 id="jobs-1">JOBS</h4>
 <p>The test concurrency (<code>-concurrency</code>).</p>
 <p>Defaults to TEST_JOBS (if set by <code>--with-test-jobs=</code>), otherwise it defaults to JOBS, except for Hotspot, where the default is <em>number of CPU cores/2</em>, but never more than 12.</p>
-<h4 id="timeout">TIMEOUT</h4>
+<h4 id="timeout_factor-1">TIMEOUT_FACTOR</h4>
 <p>The timeout factor (<code>-timeoutFactor</code>).</p>
 <p>Defaults to 4.</p>
 <h4 id="test_mode">TEST_MODE</h4>
@@ -125,13 +144,21 @@ TEST FAILURE</code></pre>
 <p>Limit memory consumption (<code>-Xmx</code> and <code>-vmoption:-Xmx</code>, or none).</p>
 <p>Limit memory consumption for JTReg test framework and VM under test. Set to 0 to disable the limits.</p>
 <p>Defaults to 512m, except for hotspot, where it defaults to 0 (no limit).</p>
+<h4 id="keywords">KEYWORDS</h4>
+<p>JTReg kewords sent to JTReg using <code>-k</code>. Please be careful in making sure that spaces and special characters (like <code>!</code>) are properly quoted. To avoid some issues, the special value <code>%20</code> can be used instead of space.</p>
+<h4 id="extra_problem_lists">EXTRA_PROBLEM_LISTS</h4>
+<p>Use additional problem lists file or files, in addition to the default ProblemList.txt located at the JTReg test roots.</p>
+<p>If multiple file names are specified, they should be separated by space (or, to help avoid quoting issues, the special value <code>%20</code>).</p>
+<p>The file names should be either absolute, or relative to the JTReg test root of the tests to be run.</p>
 <h4 id="options">OPTIONS</h4>
 <p>Additional options to the JTReg test framework.</p>
 <p>Use <code>JTREG=&quot;OPTIONS=--help all&quot;</code> to see all available JTReg options.</p>
-<h4 id="java_options">JAVA_OPTIONS</h4>
+<h4 id="java_options-1">JAVA_OPTIONS</h4>
 <p>Additional Java options to JTReg (<code>-javaoption</code>).</p>
-<h4 id="vm_options">VM_OPTIONS</h4>
+<h4 id="vm_options-1">VM_OPTIONS</h4>
 <p>Additional VM options to JTReg (<code>-vmoption</code>).</p>
+<h4 id="aot_modules-1">AOT_MODULES</h4>
+<p>Generate AOT modules before testing for the specified module, or set of modules. If multiple modules are specified, they should be separated by space (or, to help avoid quoting issues, the special value <code>%20</code>).</p>
 <h3 id="gtest-keywords">Gtest keywords</h3>
 <h4 id="repeat">REPEAT</h4>
 <p>The number of times to repeat the tests (<code>--gtest_repeat</code>).</p>
@@ -139,6 +166,8 @@ TEST FAILURE</code></pre>
 <h4 id="options-1">OPTIONS</h4>
 <p>Additional options to the Gtest test framework.</p>
 <p>Use <code>GTEST=&quot;OPTIONS=--help&quot;</code> to see all available Gtest options.</p>
+<h4 id="aot_modules-2">AOT_MODULES</h4>
+<p>Generate AOT modules before testing for the specified module, or set of modules. If multiple modules are specified, they should be separated by space (or, to help avoid quoting issues, the special value <code>%20</code>).</p>
 <h3 id="microbenchmark-keywords">Microbenchmark keywords</h3>
 <h4 id="fork">FORK</h4>
 <p>Override the number of benchmark forks to spawn. Same as specifying <code>-f &lt;num&gt;</code>.</p>
@@ -152,7 +181,7 @@ TEST FAILURE</code></pre>
 <p>Amount of time to spend in each warmup iteration. Same as specifying <code>-w &lt;num&gt;</code>.</p>
 <h4 id="results_format">RESULTS_FORMAT</h4>
 <p>Specify to have the test run save a log of the values. Accepts the same values as <code>-rff</code>, i.e., <code>text</code>, <code>csv</code>, <code>scsv</code>, <code>json</code>, or <code>latex</code>.</p>
-<h4 id="vm_options-1">VM_OPTIONS</h4>
+<h4 id="vm_options-2">VM_OPTIONS</h4>
 <p>Additional VM arguments to provide to forked off VMs. Same as <code>-jvmArgs &lt;args&gt;</code></p>
 <h4 id="options-2">OPTIONS</h4>
 <p>Additional arguments to send to JMH.</p>
diff --git a/doc/testing.md b/doc/testing.md
index 70f648cc6c7..b447cfdfe8b 100644
--- a/doc/testing.md
+++ b/doc/testing.md
@@ -40,8 +40,8 @@ environment variable to point to the JTReg home before running `configure`.)
 
 To be able to run microbenchmarks, `configure` needs to know where to find
 the JMH dependency. Use `--with-jmh=<path to JMH jars>` to point to a directory
-containing the core JMH and transitive dependencies. The recommended dependencies 
-can be retrieved by running `sh make/devkit/createJMHBundle.sh`, after which 
+containing the core JMH and transitive dependencies. The recommended dependencies
+can be retrieved by running `sh make/devkit/createJMHBundle.sh`, after which
 `--with-jmh=build/jmh/jars` should work.
 
 ## Test selection
@@ -202,6 +202,50 @@ proper quoting of command line arguments through.)
 As far as possible, the names of the keywords have been standardized between
 test suites.
 
+### General keywords (TEST_OPTS)
+
+Some keywords are valid across different test suites. If you want to run
+tests from multiple test suites, or just don't want to care which test suite specific
+control variable to use, then you can use the general TEST_OPTS control variable.
+
+There are also some keywords that applies globally to the test runner system,
+not to any specific test suites. These are also available as TEST_OPTS keywords.
+
+#### JOBS
+
+Currently only applies to JTReg.
+
+#### TIMEOUT_FACTOR
+
+Currently only applies to JTReg.
+
+#### VM_OPTIONS
+
+Applies to JTReg, GTest and Micro.
+
+#### JAVA_OPTIONS
+
+Applies to JTReg, GTest and Micro.
+
+#### AOT_MODULES
+
+Applies to JTReg and GTest.
+
+#### JCOV
+
+This keywords applies globally to the test runner system. If set to `true`, it
+enables JCov coverage reporting for all tests run. To be useful, the JDK under
+test must be run with a JDK built with JCov instrumentation (`configure
+--with-jcov=<path to directory containing lib/jcov.jar>`, `make jcov-image`).
+
+The simplest way to run tests with JCov coverage report is to use the special
+target `jcov-test` instead of `test`, e.g. `make jcov-test TEST=jdk_lang`. This
+will make sure the JCov image is built, and that JCov reporting is enabled.
+
+The JCov report is stored in `build/$BUILD/test-results/jcov-output`.
+
+Please note that running with JCov reporting can be very memory intensive.
+
 ### JTReg keywords
 
 #### JOBS
@@ -211,7 +255,7 @@ Defaults to TEST_JOBS (if set by `--with-test-jobs=`), otherwise it defaults to
 JOBS, except for Hotspot, where the default is *number of CPU cores/2*, but
 never more than 12.
 
-#### TIMEOUT
+#### TIMEOUT_FACTOR
 The timeout factor (`-timeoutFactor`).
 
 Defaults to 4.
@@ -245,6 +289,24 @@ to disable the limits.
 
 Defaults to 512m, except for hotspot, where it defaults to 0 (no limit).
 
+#### KEYWORDS
+
+JTReg kewords sent to JTReg using `-k`. Please be careful in making sure that
+spaces and special characters (like `!`) are properly quoted. To avoid some
+issues, the special value `%20` can be used instead of space.
+
+#### EXTRA_PROBLEM_LISTS
+
+Use additional problem lists file or files, in addition to the default
+ProblemList.txt located at the JTReg test roots.
+
+If multiple file names are specified, they should be separated by space (or, to
+help avoid quoting issues, the special value `%20`).
+
+The file names should be either absolute, or relative to the JTReg test root of
+the tests to be run.
+
+
 #### OPTIONS
 Additional options to the JTReg test framework.
 
@@ -256,6 +318,12 @@ Additional Java options to JTReg (`-javaoption`).
 #### VM_OPTIONS
 Additional VM options to JTReg (`-vmoption`).
 
+#### AOT_MODULES
+
+Generate AOT modules before testing for the specified module, or set of
+modules. If multiple modules are specified, they should be separated by space
+(or, to help avoid quoting issues, the special value `%20`).
+
 ### Gtest keywords
 
 #### REPEAT
@@ -270,6 +338,12 @@ Additional options to the Gtest test framework.
 
 Use `GTEST="OPTIONS=--help"` to see all available Gtest options.
 
+#### AOT_MODULES
+
+Generate AOT modules before testing for the specified module, or set of
+modules. If multiple modules are specified, they should be separated by space
+(or, to help avoid quoting issues, the special value `%20`).
+
 ### Microbenchmark keywords
 
 #### FORK
diff --git a/make/RunTests.gmk b/make/RunTests.gmk
index 13b0b829595..d44833a3b12 100644
--- a/make/RunTests.gmk
+++ b/make/RunTests.gmk
@@ -45,8 +45,8 @@ ifneq ($(TEST_VM_OPTS), )
 endif
 
 $(eval $(call ParseKeywordVariable, TEST_OPTS, \
-    SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR AOT_MODULES JCOV, \
-    STRING_KEYWORDS := VM_OPTIONS JAVA_OPTIONS, \
+    SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR JCOV, \
+    STRING_KEYWORDS := VM_OPTIONS JAVA_OPTIONS AOT_MODULES, \
 ))
 
 # Helper function to propagate TEST_OPTS values.
@@ -104,8 +104,10 @@ ifneq ($(wildcard $(JTREG_FAILURE_HANDLER)), )
       -timeoutHandlerTimeout:0
 endif
 
-GTEST_LAUNCHER_DIRS := $(patsubst %/gtestLauncher, %, $(wildcard $(TEST_IMAGE_DIR)/hotspot/gtest/*/gtestLauncher))
-GTEST_VARIANTS := $(strip $(patsubst $(TEST_IMAGE_DIR)/hotspot/gtest/%, %, $(GTEST_LAUNCHER_DIRS)))
+GTEST_LAUNCHER_DIRS := $(patsubst %/gtestLauncher, %, \
+    $(wildcard $(TEST_IMAGE_DIR)/hotspot/gtest/*/gtestLauncher))
+GTEST_VARIANTS := $(strip $(patsubst $(TEST_IMAGE_DIR)/hotspot/gtest/%, %, \
+    $(GTEST_LAUNCHER_DIRS)))
 
 ################################################################################
 # Optionally create AOT libraries for specified modules before running tests.
@@ -130,7 +132,8 @@ define SetupAotModuleBody
   $1_AOT_CCLIST := $$(wildcard $$(TOPDIR)/test/hotspot/jtreg/compiler/aot/scripts/$$($1_MODULE)-list.txt)
 
   # Create jaotc flags.
-  # VM flags which don't affect AOT code generation are filtered out: -Xcomp, -XX:+-TieredCompilation
+  # VM flags which don't affect AOT code generation are filtered out:
+  # -Xcomp, -XX:+-TieredCompilation
   $1_JAOTC_OPTS := \
       -J-Xmx4g --info \
       $$(addprefix -J, $$(filter-out -Xcomp %TieredCompilation, $$($1_VM_OPTIONS))) \
@@ -245,9 +248,10 @@ $(eval $(call SetTestOpt,JOBS,JTREG))
 $(eval $(call SetTestOpt,TIMEOUT_FACTOR,JTREG))
 
 $(eval $(call ParseKeywordVariable, JTREG, \
-    SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM \
-        EXTRA_PROBLEM_LISTS KEYWORDS AOT_MODULES, \
-    STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \
+    SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR TEST_MODE ASSERT VERBOSE RETAIN \
+        MAX_MEM, \
+    STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS KEYWORDS \
+        EXTRA_PROBLEM_LISTS AOT_MODULES, \
 ))
 
 ifneq ($(JTREG), )
@@ -262,8 +266,8 @@ $(eval $(call SetTestOpt,JAVA_OPTIONS,GTEST))
 $(eval $(call SetTestOpt,AOT_MODULES,GTEST))
 
 $(eval $(call ParseKeywordVariable, GTEST, \
-    SINGLE_KEYWORDS := REPEAT AOT_MODULES, \
-    STRING_KEYWORDS := OPTIONS VM_OPTIONS JAVA_OPTIONS, \
+    SINGLE_KEYWORDS := REPEAT, \
+    STRING_KEYWORDS := OPTIONS VM_OPTIONS JAVA_OPTIONS AOT_MODULES, \
 ))
 
 ifneq ($(GTEST), )
@@ -278,7 +282,8 @@ $(eval $(call SetTestOpt,JAVA_OPTIONS,MICRO))
 
 $(eval $(call ParseKeywordVariable, MICRO, \
     SINGLE_KEYWORDS := ITER FORK TIME WARMUP_ITER WARMUP_TIME, \
-    STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS RESULTS_FORMAT TEST_JDK BENCHMARKS_JAR, \
+    STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS RESULTS_FORMAT TEST_JDK \
+        BENCHMARKS_JAR, \
 ))
 
 ifneq ($(MICRO), )
@@ -344,8 +349,8 @@ endef
 
 # Helper function to determine if a test specification is a microbenchmark test
 #
-# It is a microbenchmark test if it is either "micro", or "micro:" followed by an optional
-# test filter string.
+# It is a microbenchmark test if it is either "micro", or "micro:" followed by
+# an optional test filter string.
 define ParseMicroTestSelection
   $(if $(filter micro%, $1), \
     $(if $(filter micro, $1), \
@@ -435,8 +440,8 @@ endef
 # Helper function to determine if a test specification is a special test
 #
 # It is a special test if it is "special:" followed by a test name,
-# if it is "make:" or "make-" followed by a make test, or any of the special test names
-# as a single word.
+# if it is "make:" or "make-" followed by a make test, or any of the special
+# test names as a single word.
 define ParseSpecialTestSelection
   $(if $(filter special:%, $1), \
     $1 \
@@ -633,7 +638,8 @@ define SetupRunMicroTestBody
 
   # Save output as JSON or CSV file
   ifneq ($$(MICRO_RESULTS_FORMAT), )
-    $1_MICRO_BASIC_OPTIONS += -rf $$(MICRO_RESULTS_FORMAT) -rff $$($1_TEST_RESULTS_DIR)/jmh-result.$(MICRO_RESULTS_FORMAT)
+    $1_MICRO_BASIC_OPTIONS += -rf $$(MICRO_RESULTS_FORMAT)
+    $1_MICRO_BASIC_OPTIONS += -rff $$($1_TEST_RESULTS_DIR)/jmh-result.$(MICRO_RESULTS_FORMAT)
   endif
 
   ifneq ($$(MICRO_VM_OPTIONS)$$(MICRO_JAVA_OPTIONS), )
@@ -664,7 +670,7 @@ define SetupRunMicroTestBody
 	    $$($1_MICRO_TEST_JDK)/bin/java $$($1_MICRO_JAVA_OPTIONS) -jar $$($1_MICRO_BENCHMARKS_JAR) \
 	        $$($1_MICRO_ITER) $$($1_MICRO_FORK) $$($1_MICRO_TIME) \
 	        $$($1_MICRO_WARMUP_ITER) $$($1_MICRO_WARMUP_TIME) \
-	        $$($1_MICRO_VM_OPTIONS) $$($1_MICRO_BASIC_OPTIONS) $$(MICRO_OPTIONS)  \
+	        $$($1_MICRO_VM_OPTIONS) $$($1_MICRO_BASIC_OPTIONS) $$(MICRO_OPTIONS) \
 	        $$($1_TEST_NAME) \
 	        > >($(TEE) $$($1_TEST_RESULTS_DIR)/micro.txt) \
 	    && $$(ECHO) $$$$? > $$($1_EXITCODE) \