annotate make/RunTests.gmk @ 2608:69879afbd98f

8175825: Stop including pubs repo Reviewed-by: erikj
author ihse
date Fri, 12 May 2017 19:09:50 +0200
parents 459d61aa3b54
children f6e365a5e489
rev   line source
ihse@2533 1 #
ihse@2533 2 # Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
ihse@2533 3 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
ihse@2533 4 #
ihse@2533 5 # This code is free software; you can redistribute it and/or modify it
ihse@2533 6 # under the terms of the GNU General Public License version 2 only, as
ihse@2533 7 # published by the Free Software Foundation. Oracle designates this
ihse@2533 8 # particular file as subject to the "Classpath" exception as provided
ihse@2533 9 # by Oracle in the LICENSE file that accompanied this code.
ihse@2533 10 #
ihse@2533 11 # This code is distributed in the hope that it will be useful, but WITHOUT
ihse@2533 12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
ihse@2533 13 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
ihse@2533 14 # version 2 for more details (a copy is included in the LICENSE file that
ihse@2533 15 # accompanied this code).
ihse@2533 16 #
ihse@2533 17 # You should have received a copy of the GNU General Public License version
ihse@2533 18 # 2 along with this work; if not, write to the Free Software Foundation,
ihse@2533 19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
ihse@2533 20 #
ihse@2533 21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
ihse@2533 22 # or visit www.oracle.com if you need additional information or have any
ihse@2533 23 # questions.
ihse@2533 24 #
ihse@2533 25
ihse@2533 26 default: all
ihse@2533 27
ihse@2533 28 include $(SPEC)
ihse@2533 29 include MakeBase.gmk
ihse@2533 30 include FindTests.gmk
ihse@2533 31
ihse@2533 32 # We will always run multiple tests serially
ihse@2533 33 .NOTPARALLEL:
ihse@2533 34
ihse@2533 35 # Hook to include the corresponding custom file, if present.
ihse@2533 36 $(eval $(call IncludeCustomExtension, , RunTests.gmk))
ihse@2533 37
ihse@2533 38 TEST_RESULTS_DIR := $(BUILD_OUTPUT)/test-results
ihse@2533 39 TEST_SUPPORT_DIR := $(BUILD_OUTPUT)/test-support
ihse@2533 40
ihse@2533 41
ihse@2533 42 ################################################################################
ihse@2533 43 # Parse control variables
ihse@2533 44 ################################################################################
ihse@2533 45
ihse@2533 46 $(eval $(call ParseKeywordVariable, JTREG, \
ihse@2533 47 KEYWORDS := JOBS TIMEOUT TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM, \
ihse@2533 48 STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \
ihse@2533 49 ))
ihse@2533 50
ihse@2533 51 ifneq ($(JTREG), )
ihse@2533 52 # Inform the user
ihse@2533 53 $(info Running tests using JTREG control variable '$(JTREG)')
ihse@2533 54 endif
ihse@2533 55
ihse@2533 56 $(eval $(call ParseKeywordVariable, GTEST, \
ihse@2533 57 KEYWORDS := REPEAT, \
ihse@2533 58 STRING_KEYWORDS := OPTIONS, \
ihse@2533 59 ))
ihse@2533 60
ihse@2533 61 ifneq ($(GTEST), )
ihse@2533 62 # Inform the user
ihse@2533 63 $(info Running tests using GTEST control variable '$(GTEST)')
ihse@2533 64 endif
ihse@2533 65
ihse@2533 66
ihse@2533 67 ################################################################################
ihse@2533 68 # Component-specific Jtreg settings
ihse@2533 69 ################################################################################
ihse@2533 70
ihse@2533 71 ifeq ($(TEST_JOBS), 0)
ihse@2533 72 # If TEST_JOBS is not specified, hotspot fallback default is
ihse@2533 73 # min(num_cores / 2, 12).
ihse@2533 74 hotspot_JTREG_JOBS := $(shell $(EXPR) $(NUM_CORES) / 2)
ihse@2533 75 ifeq ($(hotspot_JTREG_JOBS), 0)
ihse@2533 76 hotspot_JTREG_JOBS := 1
ihse@2533 77 else ifeq ($(shell $(EXPR) $(hotspot_JTREG_JOBS) \> 12), 1)
ihse@2533 78 hotspot_JTREG_JOBS := 12
ihse@2533 79 endif
ihse@2533 80 endif
ihse@2533 81
ihse@2533 82 hotspot_JTREG_MAX_MEM := 0
ihse@2533 83 hotspot_JTREG_ASSERT := false
ihse@2533 84 hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native
ihse@2533 85 jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native
ihse@2533 86
ihse@2533 87
ihse@2533 88 ################################################################################
ihse@2533 89 # Parse test selection
ihse@2533 90 #
ihse@2533 91 # The user has given a test selection in the TEST variable. We must parse it
ihse@2533 92 # and determine what that means in terms of actual calls to the test framework.
ihse@2533 93 #
ihse@2533 94 # The parse functions take as argument a test specification as given by the
ihse@2533 95 # user, and returns a fully qualified test descriptor if it was a match, or
ihse@2533 96 # nothing if not. A single test specification can result in multiple test
ihse@2533 97 # descriptors being returned. A valid test descriptor must always be accepted
ihse@2533 98 # and returned identically.
ihse@2533 99 ################################################################################
ihse@2533 100
ihse@2533 101 # Helper function to determine if a test specification is a Gtest test
ihse@2533 102 #
ihse@2533 103 # It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional
ihse@2533 104 # test filter string.
ihse@2533 105 define ParseGtestTestSelection
ihse@2533 106 $(if $(filter gtest%, $1), \
ihse@2533 107 $(if $(filter gtest, $1), \
ihse@2533 108 gtest:all \
ihse@2533 109 , \
ihse@2533 110 $(if $(filter gtest:, $1), \
ihse@2533 111 gtest:all \
ihse@2533 112 , \
ihse@2533 113 $1 \
ihse@2533 114 ) \
ihse@2533 115 ) \
ihse@2533 116 )
ihse@2533 117 endef
ihse@2533 118
ihse@2533 119 # Helper function to determine if a test specification is a Jtreg test
ihse@2533 120 #
ihse@2533 121 # It is a Jtreg test if it optionally begins with jtreg:, and then is either
ihse@2533 122 # an unspecified group name (possibly prefixed by :), or a group in a
ihse@2533 123 # specified <component>/test directory, or a path to a test or test directory,
ihse@2533 124 # either absolute or relative to TOPDIR.
ihse@2533 125 define ParseJtregTestSelection
ihse@2533 126 $(eval TEST_NAME := $(strip $(patsubst jtreg:%, %, $1))) \
ihse@2533 127 $(if $(or $(findstring :, $(TEST_NAME)), $(findstring /, $(TEST_NAME))), , \
ihse@2533 128 $(eval TEST_NAME := :$(TEST_NAME)) \
ihse@2533 129 ) \
ihse@2533 130 $(if $(findstring :, $(TEST_NAME)), \
ihse@2533 131 $(if $(filter :%, $(TEST_NAME)), \
ihse@2533 132 $(foreach component, $(JTREG_COMPONENTS), \
ihse@2533 133 $(if $(filter $(patsubst :%, %, $(TEST_NAME)), \
ihse@2533 134 $($(component)_JTREG_TEST_GROUPS)), \
ihse@2533 135 jtreg:$(component)/test:$(patsubst :%,%,$(TEST_NAME)) \
ihse@2533 136 ) \
ihse@2533 137 ) \
ihse@2533 138 , \
ihse@2533 139 $(eval COMPONENT := $(word 1, $(subst /, $(SPACE), $(TEST_NAME)))) \
ihse@2533 140 $(eval GROUP := $(word 2, $(subst :, $(SPACE), $(TEST_NAME)))) \
ihse@2533 141 $(if $(filter $(COMPONENT), $(JTREG_COMPONENTS)), \
ihse@2533 142 $(if $(filter $(GROUP), $($(COMPONENT)_JTREG_TEST_GROUPS)), \
ihse@2533 143 jtreg:$(TEST_NAME) \
ihse@2533 144 ) \
ihse@2533 145 ) \
ihse@2533 146 ) \
ihse@2533 147 , \
ihse@2533 148 $(if $(filter /%, $(TEST_NAME)), \
ihse@2533 149 $(if $(wildcard $(TEST_NAME)), \
ihse@2533 150 jtreg:$(TEST_NAME) \
ihse@2533 151 ) \
ihse@2533 152 , \
ihse@2533 153 $(if $(wildcard $(TOPDIR)/$(TEST_NAME)), \
ihse@2533 154 jtreg:$(TEST_NAME) \
ihse@2533 155 ) \
ihse@2533 156 ) \
ihse@2533 157 )
ihse@2533 158 endef
ihse@2533 159
ihse@2533 160 ifeq ($(TEST), )
ihse@2533 161 $(info No test selection given in TEST!)
ihse@2533 162 $(info Please use e.g. 'run-test TEST=tier1' or 'run-test-tier1')
ihse@2560 163 $(info See common/doc/testing.[md|html] for help)
ihse@2533 164 $(error Cannot continue)
ihse@2533 165 endif
ihse@2533 166
ihse@2533 167 # Now intelligently convert the test selection given by the user in TEST
ihse@2533 168 # into a list of fully qualified test descriptors of the tests to run.
ihse@2533 169 TESTS_TO_RUN :=
ihse@2533 170 $(foreach test, $(TEST), \
ihse@2533 171 $(eval PARSED_TESTS := $(call ParseCustomTestSelection, $(test))) \
ihse@2533 172 $(if $(strip $(PARSED_TESTS)), , \
ihse@2533 173 $(eval PARSED_TESTS += $(call ParseGtestTestSelection, $(test))) \
ihse@2533 174 ) \
ihse@2533 175 $(if $(strip $(PARSED_TESTS)), , \
ihse@2533 176 $(eval PARSED_TESTS += $(call ParseJtregTestSelection, $(test))) \
ihse@2533 177 ) \
ihse@2533 178 $(if $(strip $(PARSED_TESTS)), , \
ihse@2533 179 $(eval UNKNOWN_TEST := $(test)) \
ihse@2533 180 ) \
ihse@2533 181 $(eval TESTS_TO_RUN += $(PARSED_TESTS)) \
ihse@2533 182 )
ihse@2533 183
ihse@2533 184 ifneq ($(UNKNOWN_TEST), )
ihse@2533 185 $(info Unknown test selection: '$(UNKNOWN_TEST)')
ihse@2560 186 $(info See common/doc/testing.[md|html] for help)
ihse@2533 187 $(error Cannot continue)
ihse@2533 188 endif
ihse@2533 189
ihse@2533 190 TESTS_TO_RUN := $(strip $(TESTS_TO_RUN))
ihse@2533 191
ihse@2533 192
ihse@2533 193 # Present the result of our parsing to the user
ihse@2533 194 $(info Test selection '$(TEST)', will run:)
ihse@2533 195 $(foreach test, $(TESTS_TO_RUN), $(info * $(test)))
ihse@2533 196
ihse@2533 197
ihse@2533 198 ################################################################################
ihse@2533 199 # Functions for setting up rules for running the selected tests
ihse@2533 200 #
ihse@2533 201 # The SetupRun*Test functions all have the same interface:
ihse@2533 202 #
ihse@2533 203 # Parameter 1 is the name of the rule. This is the test id, based on the test
ihse@2533 204 # descriptor, and this is also used as variable prefix, and the targets
ihse@2533 205 # generated are listed in a variable by that name.
ihse@2533 206 #
ihse@2533 207 # Remaining parameters are named arguments. Currently this is only:
ihse@2533 208 # TEST -- The properly formatted fully qualified test descriptor
ihse@2533 209 #
ihse@2533 210 # After the rule named by the test id has been executed, the following
ihse@2533 211 # variables will be available:
ihse@2533 212 # testid_TOTAL - the total number of tests run
ihse@2533 213 # testid_PASSED - the number of successful tests
ihse@2533 214 # testid_FAILED - the number of failed tests
ihse@2533 215 # testid_ERROR - the number of tests was neither successful or failed
ihse@2533 216 #
ihse@2533 217 ################################################################################
ihse@2533 218
ihse@2533 219 ### Rules for Gtest
ihse@2533 220
ihse@2533 221 SetupRunGtestTest = $(NamedParamsMacroTemplate)
ihse@2533 222 define SetupRunGtestTestBody
ihse@2533 223 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
ihse@2533 224 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
ihse@2533 225
ihse@2533 226 $1_TEST_NAME := $$(strip $$(patsubst gtest:%, %, $$($1_TEST)))
ihse@2533 227 ifneq ($$($1_TEST_NAME), all)
ihse@2533 228 $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)*
ihse@2533 229 endif
ihse@2533 230
ihse@2533 231 ifneq ($$(GTEST_REPEAT), )
ihse@2533 232 $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT)
ihse@2533 233 endif
ihse@2533 234
ihse@2533 235 run-test-$1:
ihse@2533 236 $$(call LogWarn)
ihse@2533 237 $$(call LogWarn, Running test '$$($1_TEST)')
ihse@2533 238 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
ihse@2533 239 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, \
ihse@2533 240 $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/server/gtestLauncher \
ihse@2533 241 -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \
ihse@2533 242 --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
ihse@2533 243 $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) \
ihse@2533 244 > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) || true )
ihse@2533 245
ihse@2533 246 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
ihse@2533 247
ihse@2533 248 parse-test-$1: run-test-$1
ihse@2533 249 $$(call LogWarn, Finished running test '$$($1_TEST)')
ihse@2533 250 $$(call LogWarn, Test report is stored in $$(strip \
ihse@2533 251 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
ihse@2533 252 $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \
ihse@2533 253 test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE)))
ihse@2533 254 $$(eval $1_PASSED := $$(shell $$(AWK) '/\[ PASSED \] .* tests?./ \
ihse@2533 255 { print $$$$4 }' $$($1_RESULT_FILE)))
ihse@2533 256 $$(eval $1_FAILED := $$(shell $$(AWK) '/\[ FAILED \] .* tests?, \
ihse@2533 257 listed below/ { print $$$$4 }' $$($1_RESULT_FILE)))
ihse@2533 258 $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0))
ihse@2533 259 $$(eval $1_ERROR := $$(shell \
ihse@2533 260 $$(EXPR) $$($1_TOTAL) - $$($1_PASSED) - $$($1_FAILED)))
ihse@2533 261
ihse@2533 262 $1: run-test-$1 parse-test-$1
ihse@2533 263
ihse@2533 264 TARGETS += $1
ihse@2533 265 endef
ihse@2533 266
ihse@2533 267 ################################################################################
ihse@2533 268
ihse@2533 269 ### Rules for Jtreg
ihse@2533 270
ihse@2533 271 # Helper function for SetupRunJtregTest. Set a JTREG_* variable from, in order:
ihse@2533 272 # 1) Specified by user on command line
ihse@2533 273 # 2) Component-specific default
ihse@2533 274 # 3) Generic default
ihse@2533 275 #
ihse@2533 276 # Note: No spaces are allowed around the arguments.
ihse@2533 277 # Arg $1 The test ID (i.e. $1 in SetupRunJtregTest)
ihse@2533 278 # Arg $2 Base variable, e.g. JTREG_JOBS
ihse@2533 279 # Arg $3 The default value (optional)
ihse@2533 280 define SetJtregValue
ihse@2533 281 ifneq ($$($2), )
ihse@2533 282 $1_$2 := $$($2)
ihse@2533 283 else
ihse@2533 284 ifneq ($$($$($1_COMPONENT)_$2), )
ihse@2533 285 $1_$2 := $$($$($1_COMPONENT)_$2)
ihse@2533 286 else
ihse@2533 287 ifneq ($3, )
ihse@2533 288 $1_$2 := $3
ihse@2533 289 endif
ihse@2533 290 endif
ihse@2533 291 endif
ihse@2533 292 endef
ihse@2533 293
ihse@2533 294 SetupRunJtregTest = $(NamedParamsMacroTemplate)
ihse@2533 295 define SetupRunJtregTestBody
ihse@2533 296 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
ihse@2533 297 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
ihse@2533 298
ihse@2533 299 $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST)))
ihse@2533 300 $1_COMPONENT := $$(firstword $$(subst /, $$(SPACE), $$($1_TEST_NAME)))
ihse@2533 301
ihse@2533 302 # Unfortunately, we need different defaults for some JTREG values,
ihse@2533 303 # depending on what component we're running.
ihse@2533 304
ihse@2533 305 # Convert JTREG_foo into $1_JTREG_foo with a suitable value.
ihse@2533 306 $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm))
ihse@2533 307 $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true))
ihse@2533 308 $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,512m))
ihse@2533 309 $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH))
ihse@2533 310 $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS))
ihse@2533 311
ihse@2533 312 ifneq ($(TEST_JOBS), 0)
ihse@2533 313 # User has specified TEST_JOBS, use that as fallback default
ihse@2533 314 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
ihse@2533 315 else
ihse@2533 316 # Use JOBS as default (except for hotspot)
ihse@2533 317 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS)))
ihse@2533 318 endif
ihse@2533 319
ihse@2533 320 ifeq ($$(shell $$(EXPR) $$($1_JTREG_JOBS) \> 50), 1)
ihse@2533 321 # Until CODETOOLS-7901892 is fixed, JTreg cannot handle more than 50 jobs
ihse@2533 322 $1_JTREG_JOBS := 50
ihse@2533 323 endif
ihse@2533 324
ihse@2533 325 # Make sure MaxRAMFraction is high enough to not cause OOM or swapping since
ihse@2533 326 # we may end up with a lot of JVM's
ihse@2533 327 $1_JTREG_MAX_RAM_FRACTION := $$(shell $$(EXPR) $$($1_JTREG_JOBS) \* 4)
ihse@2533 328
ihse@2533 329 JTREG_TIMEOUT ?= 4
ihse@2533 330 JTREG_VERBOSE ?= fail,error,summary
ihse@2533 331 JTREG_RETAIN ?= fail,error
ihse@2533 332
ihse@2533 333 ifneq ($$($1_JTREG_MAX_MEM), 0)
ihse@2533 334 $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM)
ihse@2533 335 $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM)
ihse@2533 336 endif
ihse@2533 337
ihse@2533 338 $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \
ihse@2533 339 -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \
ihse@2533 340 -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT) \
ihse@2533 341 -vmoption:-XX:MaxRAMFraction=$$($1_JTREG_MAX_RAM_FRACTION)
ihse@2533 342
ihse@2533 343 $1_JTREG_BASIC_OPTIONS += -automatic -keywords:\!ignore -ignore:quiet
ihse@2533 344
ihse@2533 345 # Some tests needs to find a boot JDK using the JDK8_HOME variable.
ihse@2533 346 $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK)
ihse@2533 347
ihse@2533 348 $1_JTREG_BASIC_OPTIONS += \
ihse@2533 349 $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \
ihse@2533 350 $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \
ihse@2533 351 #
ihse@2533 352
ihse@2533 353 ifeq ($$($1_JTREG_ASSERT), true)
ihse@2533 354 $1_JTREG_BASIC_OPTIONS += -ea -esa
ihse@2533 355 endif
ihse@2533 356
ihse@2533 357 ifneq ($$($1_JTREG_NATIVEPATH), )
ihse@2533 358 $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH)
ihse@2533 359 endif
ihse@2533 360
ihse@2533 361 run-test-$1:
ihse@2533 362 $$(call LogWarn)
ihse@2533 363 $$(call LogWarn, Running test '$$($1_TEST)')
ihse@2533 364 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
ihse@2533 365 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, \
ihse@2533 366 $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
ihse@2533 367 -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
ihse@2533 368 $$($1_JTREG_BASIC_OPTIONS) \
ihse@2533 369 -testjdk:$$(JDK_IMAGE_DIR) \
ihse@2533 370 -dir:$$(TOPDIR) \
ihse@2533 371 -reportDir:$$($1_TEST_RESULTS_DIR) \
ihse@2533 372 -workDir:$$($1_TEST_SUPPORT_DIR) \
ihse@2533 373 $$(JTREG_OPTIONS) \
ihse@2533 374 $$($1_TEST_NAME) || true )
ihse@2533 375
ihse@2533 376 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt
ihse@2533 377
ihse@2533 378 parse-test-$1: run-test-$1
ihse@2533 379 $$(call LogWarn, Finished running test '$$($1_TEST)')
ihse@2533 380 $$(call LogWarn, Test report is stored in $$(strip \
ihse@2533 381 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
ihse@2533 382 $$(eval $1_PASSED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \
ihse@2533 383 for (i=1; i<=NF; i++) { if ($$$$i == "passed:") \
ihse@2533 384 print $$$$(i+1) } }' $$($1_RESULT_FILE)))
ihse@2533 385 $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0))
ihse@2533 386 $$(eval $1_FAILED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
ihse@2533 387 for (i=1; i<=NF; i++) { if ($$$$i == "failed:") \
ihse@2533 388 print $$$$(i+1) } }' $$($1_RESULT_FILE)))
ihse@2533 389 $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0))
ihse@2533 390 $$(eval $1_ERROR := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
ihse@2533 391 for (i=1; i<=NF; i++) { if ($$$$i == "error:") \
ihse@2533 392 print $$$$(i+1) } }' $$($1_RESULT_FILE)))
ihse@2533 393 $$(if $$($1_ERROR), , $$(eval $1_ERROR := 0))
ihse@2533 394 $$(eval $1_TOTAL := $$(shell \
ihse@2533 395 $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR)))
ihse@2533 396
ihse@2533 397 $1: run-test-$1 parse-test-$1
ihse@2533 398
ihse@2533 399 TARGETS += $1
ihse@2533 400 endef
ihse@2533 401
ihse@2533 402
ihse@2533 403 ################################################################################
ihse@2533 404 # Setup and execute make rules for all selected tests
ihse@2533 405 ################################################################################
ihse@2533 406
ihse@2533 407 # Helper function to determine which handler to use for the given test
ihse@2533 408 UseGtestTestHandler = \
ihse@2533 409 $(if $(filter gtest:%, $1), true)
ihse@2533 410
ihse@2533 411 UseJtregTestHandler = \
ihse@2533 412 $(if $(filter jtreg:%, $1), true)
ihse@2533 413
ihse@2533 414 # Now process each test to run and setup a proper make rule
ihse@2533 415 $(foreach test, $(TESTS_TO_RUN), \
ihse@2533 416 $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
ihse@2533 417 $(TR) -cs '[a-z][A-Z][0-9]\n' '_')) \
ihse@2533 418 $(eval ALL_TEST_IDS += $(TEST_ID)) \
ihse@2533 419 $(if $(call UseCustomTestHandler, $(test)), \
ihse@2533 420 $(eval $(call SetupRunCustomTest, $(TEST_ID), \
ihse@2533 421 TEST := $(test), \
ihse@2533 422 )) \
ihse@2533 423 ) \
ihse@2533 424 $(if $(call UseGtestTestHandler, $(test)), \
ihse@2533 425 $(eval $(call SetupRunGtestTest, $(TEST_ID), \
ihse@2533 426 TEST := $(test), \
ihse@2533 427 )) \
ihse@2533 428 ) \
ihse@2533 429 $(if $(call UseJtregTestHandler, $(test)), \
ihse@2533 430 $(eval $(call SetupRunJtregTest, $(TEST_ID), \
ihse@2533 431 TEST := $(test), \
ihse@2533 432 )) \
ihse@2533 433 ) \
ihse@2533 434 )
ihse@2533 435
ihse@2533 436 # Sort also removes duplicates, so if there is any we'll get fewer words.
ihse@2533 437 ifneq ($(words $(ALL_TEST_IDS)), $(words $(sort $(ALL_TEST_IDS))))
ihse@2533 438 $(error Duplicate test specification)
ihse@2533 439 endif
ihse@2533 440
ihse@2533 441
ihse@2533 442 ################################################################################
ihse@2533 443 # The main target for RunTests.gmk
ihse@2533 444 ################################################################################
ihse@2533 445
ihse@2533 446 # The SetupRun*Test functions have populated TARGETS.
ihse@2533 447
ihse@2533 448 TEST_FAILURE := false
ihse@2533 449
ihse@2533 450 run-test: $(TARGETS)
ihse@2533 451 # Print a table of the result of all tests run and their result
ihse@2533 452 $(ECHO)
ihse@2533 453 $(ECHO) ==============================
ihse@2533 454 $(ECHO) Test summary
ihse@2533 455 $(ECHO) ==============================
ihse@2533 456 $(PRINTF) "%2s %-49s %5s %5s %5s %5s %2s\n" " " TEST \
ihse@2533 457 TOTAL PASS FAIL ERROR " "
ihse@2533 458 $(foreach test, $(TESTS_TO_RUN), \
ihse@2533 459 $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
ihse@2533 460 $(TR) -cs '[a-z][A-Z][0-9]\n' '_')) \
ihse@2533 461 $(if $(filter $($(TEST_ID)_PASSED), $($(TEST_ID)_TOTAL)), \
ihse@2533 462 $(PRINTF) "%2s %-49s %5d %5d %5d %5d %2s\n" " " "$(test)" \
ihse@2533 463 $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) $($(TEST_ID)_FAILED) \
ihse@2533 464 $($(TEST_ID)_ERROR) " " $(NEWLINE) \
ihse@2533 465 , \
ihse@2533 466 $(PRINTF) "%2s %-49s %5d %5d %5d %5d %2s\n" ">>" "$(test)" \
ihse@2533 467 $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) $($(TEST_ID)_FAILED) \
ihse@2533 468 $($(TEST_ID)_ERROR) "<<" $(NEWLINE) \
ihse@2533 469 $(eval TEST_FAILURE := true) \
ihse@2533 470 ) \
ihse@2533 471 )
ihse@2533 472 $(ECHO) ==============================
ihse@2533 473 $(if $(filter true, $(TEST_FAILURE)), \
ihse@2533 474 $(ECHO) TEST FAILURE $(NEWLINE) \
ihse@2533 475 $(TOUCH) $(MAKESUPPORT_OUTPUTDIR)/exit-with-error \
ihse@2533 476 , \
ihse@2533 477 $(ECHO) TEST SUCCESS \
ihse@2533 478 )
ihse@2533 479 $(ECHO)
ihse@2533 480
ihse@2533 481 ################################################################################
ihse@2533 482
ihse@2533 483 all: run-test
ihse@2533 484
ihse@2533 485 .PHONY: default all run-test $(TARGETS)