gbuild: Framework for performance unit tests using callgrind

Run them with "make perfcheck" (toplevel or in a module)

To add a new performance test:
- Add it as a perfcheck target
- Use gb_CppunitTest_set_performance_test
- Use CALLGRIND_ macros from valgrind/callgrind.h
http://valgrind.org/docs/manual/cl-manual.html#cl-manual.clientrequests

Change-Id: I67c776dbe4db0a686607efeee7a7e4f3aeae0e5c

5 files changed, 64 insertions, 4 deletions

diff --git a/solenv/gbuild/CppunitTest.mk b/solenv/gbuild/CppunitTest.mk
index 4599dae58d5b..a7d1afa382f5 100644
--- a/solenv/gbuild/CppunitTest.mk
+++ b/solenv/gbuild/CppunitTest.mk
@@ -94,7 +94,10 @@ $(call gb_CppunitTest_get_target,%) :| $(gb_CppunitTest_CPPTESTDEPS)
 			|| ($(if $(value gb_CppunitTest_postprocess), \
 					RET=$$?; \
 					$(call gb_CppunitTest_postprocess,$(gb_CppunitTest_CPPTESTCOMMAND),$@.core,$$RET) >> $@.log 2>&1;) \
-				cat $@.log; $(SRCDIR)/solenv/bin/unittest-failed.sh Cppunit $*))))
+				cat $@.log; $(SRCDIR)/solenv/bin/unittest-failed.sh Cppunit $*))) \
+		$(if $(PERFTEST), && VAL=$$(grep '^==.*== Collected : ' $@.log | sed "s/==.*== Collected : //") && \
+			$(if $(filter 0,$(PERFTEST)), expr "$$VAL" "*" "101" "/" "100", test $$VAL -le $(PERFTEST) || (echo "Unit test is slow! $$VAL instructions detected." && false))) \
+	)
 
 define gb_CppunitTest_CppunitTest
 $(call gb_CppunitTest__CppunitTest_impl,$(1),$(call gb_CppunitTest_get_linktarget,$(1)))
@@ -144,6 +147,21 @@ $(call gb_CppunitTest_get_target,$(1)) : ARGS += $(2)
 
 endef
 
+# Run this unit test with callgrind tool to measure performance. If you are
+# creating a new test, first set the number to 0, run the test, and it will
+# print an number for you to use as a second parameter here. The test will
+# fail if reported number of instructions will be bigger than this parameter.
+#
+# call gb_CppunitTest_set_performance_test,name,instructions_number
+define gb_CppunitTest_set_performance_test
+$(if $(ENABLE_VALGRIND),,$(call gb_Output_error,gb_CppunitTest_set_performance_test used with empty $$(ENABLE_VALGRIND)))
+
+$(call gb_CppunitTest_get_target,$(1)) : PERFTEST := $(2)
+$(call gb_CppunitTest_get_target,$(1)) : gb_CppunitTest_VALGRINDTOOL := valgrind --tool=callgrind --dump-instr=yes --instr-atstart=no
+$(call gb_CppunitTest_use_external,$(1),valgrind)
+
+endef
+
 define gb_CppunitTest_use_ure
 $(call gb_CppunitTest_use_rdb,$(1),ure/services)
 $(call gb_CppunitTest_get_target,$(1)) : URE := $(true)
diff --git a/solenv/gbuild/Module.mk b/solenv/gbuild/Module.mk
index 59a3d25ad938..253564ce5957 100644
--- a/solenv/gbuild/Module.mk
+++ b/solenv/gbuild/Module.mk
@@ -32,6 +32,7 @@
 # unitcheck (global)          run unit tests               top-level Module/unitcheck
 # slowcheck (global)          run slow unit tests          top-level Module/slowcheck
 # subsequentcheck (global)    run system tests             top-level Module/subsequentcheck
+# perfcheck (global)          run performance unit tests   top-level Module/perfcheck
 # all (global)                default goal                 build unitcheck
 
 
@@ -44,6 +45,7 @@ gb_Module_L10NTARGETSTACK :=
 gb_Module_CHECKTARGETSTACK :=
 gb_Module_SLOWCHECKTARGETSTACK :=
 gb_Module_SUBSEQUENTCHECKTARGETSTACK :=
+gb_Module_PERFCHECKTARGETSTACK :=
 gb_Module_CLEANTARGETSTACK :=
 
 # The currently read gbuild makefile.
@@ -65,7 +67,7 @@ $(call gb_Module_get_clean_target,%) :
 	$(call gb_Output_announce,$*,$(false),MOD,5)
 	$(call gb_Output_announce_title,module $* cleared.)
 	-$(call gb_Helper_abbreviate_dirs,\
-		rm -f $(call gb_Module_get_target,$*) $(call gb_Module_get_nonl10n_target,$*) $(call gb_Module_get_l10n_target,$*) $(call gb_Module_get_check_target,$*) $(call gb_Module_get_slowcheck_target,$*) $(call gb_Module_get_subsequentcheck_target,$*))
+		rm -f $(call gb_Module_get_target,$*) $(call gb_Module_get_nonl10n_target,$*) $(call gb_Module_get_l10n_target,$*) $(call gb_Module_get_check_target,$*) $(call gb_Module_get_slowcheck_target,$*) $(call gb_Module_get_subsequentcheck_target,$*) $(call gb_Module_get_perfcheck_target,$*))
 
 $(call gb_Module_get_l10n_target,%) :
 	$(call gb_Output_announce,$*,$(true),LOC,5)
@@ -95,6 +97,13 @@ $(call gb_Module_get_subsequentcheck_target,%) :
 		mkdir -p $(dir $@) && \
 		touch $@)
 
+$(call gb_Module_get_perfcheck_target,%) :
+	$(call gb_Output_announce,$*,$(true),PFC,5)
+	$(call gb_Output_announce_title,module $* perfchecks done.)
+	-$(call gb_Helper_abbreviate_dirs,\
+		mkdir -p $(dir $@) && \
+		touch $@)
+
 $(call gb_Module_get_target,%) :
 	$(call gb_Output_announce,$*,$(true),MOD,5)
 	$(call gb_Output_announce_title,module $* done.)
@@ -102,7 +111,7 @@ $(call gb_Module_get_target,%) :
 		mkdir -p $(dir $@) && \
 		touch $@)
 
-.PHONY : all build build-l10n-only build-non-l10n-only unitcheck slowcheck subsequentcheck clean check debugrun help showmodules translations
+.PHONY : all build build-l10n-only build-non-l10n-only unitcheck slowcheck subsequentcheck perfcheck clean check debugrun help showmodules translations
 .DEFAULT_GOAL := all
 
 all : build $(if $(CROSS_COMPILING),,unitcheck $(if $(gb_PARTIAL_BUILD),,slowcheck))
@@ -147,6 +156,11 @@ subsequentcheck :
 	$(call gb_Output_announce_title,all subsequent tests checked.)
 	$(call gb_Output_announce_bell)
 
+perfcheck :
+	$(call gb_Output_announce,loaded modules: $(sort $(gb_Module_ALLMODULES)),$(true),PFC,6)
+	$(call gb_Output_announce_title,all perftests checked.)
+	$(call gb_Output_announce_bell)
+
 clean :
 	$(call gb_Output_announce,top level modules: $(foreach module,$^,$(notdir $(module))),$(false),ALL,6)
 	$(call gb_Output_announce,loaded modules: $(sort $(gb_Module_ALLMODULES)),$(false),ALL,6)
@@ -191,6 +205,7 @@ gb_Module_L10NTARGETSTACK := $(call gb_Module_get_l10n_target,$(1)) $(gb_Module_
 gb_Module_CHECKTARGETSTACK := $(call gb_Module_get_check_target,$(1)) $(gb_Module_CHECKTARGETSTACK)
 gb_Module_SLOWCHECKTARGETSTACK := $(call gb_Module_get_slowcheck_target,$(1)) $(gb_Module_SLOWCHECKTARGETSTACK)
 gb_Module_SUBSEQUENTCHECKTARGETSTACK := $(call gb_Module_get_subsequentcheck_target,$(1)) $(gb_Module_SUBSEQUENTCHECKTARGETSTACK)
+gb_Module_PERFCHECKTARGETSTACK := $(call gb_Module_get_perfcheck_target,$(1)) $(gb_Module_PERFCHECKTARGETSTACK)
 gb_Module_CLEANTARGETSTACK := $(call gb_Module_get_clean_target,$(1)) $(gb_Module_CLEANTARGETSTACK)
 gb_Module_CURRENTMODULE_DEBUG_ENABLED := $(call gb_Module__debug_enabled,$(1))
 gb_Module_CURRENTMODULE_NAME := $(1)
@@ -272,6 +287,14 @@ $(call gb_Module_get_clean_target,$(1)) : $$(gb_Module_CURRENTCLEANTARGET)
 
 endef
 
+define gb_Module_add_perfcheck_target
+$(call gb_Module__read_targetfile,$(1),$(2),perfcheck target)
+
+$(call gb_Module_get_perfcheck_target,$(1)) : $$(gb_Module_CURRENTTARGET)
+$(call gb_Module_get_clean_target,$(1)) : $$(gb_Module_CURRENTCLEANTARGET)
+
+endef
+
 define gb_Module_add_moduledir
 include $(patsubst $(1):%,%,$(filter $(1):%,$(gb_Module_MODULELOCATIONS)))/$(2)/Module_$(2).mk
 $(call gb_Module_get_target,$(1)) : $$(firstword $$(gb_Module_TARGETSTACK))
@@ -279,12 +302,14 @@ $(call gb_Module_get_l10n_target,$(1)) : $$(firstword $$(gb_Module_L10NTARGETSTA
 $(call gb_Module_get_check_target,$(1)) : $$(firstword $$(gb_Module_CHECKTARGETSTACK))
 $(call gb_Module_get_slowcheck_target,$(1)) : $$(firstword $$(gb_Module_SLOWCHECKTARGETSTACK))
 $(call gb_Module_get_subsequentcheck_target,$(1)) : $$(firstword $$(gb_Module_SUBSEQUENTCHECKTARGETSTACK))
+$(call gb_Module_get_perfcheck_target,$(1)) : $$(firstword $$(gb_Module_PERFCHECKTARGETSTACK))
 $(call gb_Module_get_clean_target,$(1)) : $$(firstword $$(gb_Module_CLEANTARGETSTACK))
 gb_Module_TARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_TARGETSTACK)),$$(gb_Module_TARGETSTACK))
 gb_Module_L10NTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_L10NTARGETSTACK)),$$(gb_Module_L10NTARGETSTACK))
 gb_Module_CHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_CHECKTARGETSTACK)),$$(gb_Module_CHECKTARGETSTACK))
 gb_Module_SLOWCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_SLOWCHECKTARGETSTACK)),$$(gb_Module_SLOWCHECKTARGETSTACK))
 gb_Module_SUBSEQUENTCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_SUBSEQUENTCHECKTARGETSTACK)),$$(gb_Module_SUBSEQUENTCHECKTARGETSTACK))
+gb_Module_PERFCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_PERFCHECKTARGETSTACK)),$$(gb_Module_PERFCHECKTARGETSTACK))
 gb_Module_CLEANTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_CLEANTARGETSTACK)),$$(gb_Module_CLEANTARGETSTACK))
 
 endef
@@ -318,6 +343,11 @@ $(foreach target,$(2),$(call gb_Module_add_subsequentcheck_target,$(1),$(target)
 
 endef
 
+define gb_Module_add_perfcheck_targets
+$(foreach target,$(2),$(call gb_Module_add_perfcheck_target,$(1),$(target)))
+
+endef
+
 define gb_Module_add_moduledirs
 $(foreach target,$(sort $(2)),$(call gb_Module_add_moduledir,$(1),$(target)))
 
@@ -336,6 +366,7 @@ build-l10n-only : $$(firstword $$(gb_Module_L10NTARGETSTACK))
 unitcheck : $$(firstword $$(gb_Module_CHECKTARGETSTACK))
 slowcheck : $$(firstword $$(gb_Module_SLOWCHECKTARGETSTACK))
 subsequentcheck : $$(firstword $$(gb_Module_SUBSEQUENTCHECKTARGETSTACK))
+perfcheck : $$(firstword $$(gb_Module_PERFCHECKTARGETSTACK))
 clean : $$(firstword $$(gb_Module_CLEANTARGETSTACK))
 
 ifneq ($$(words $$(gb_Module_TARGETSTACK)),1)
@@ -347,9 +378,10 @@ gb_Module_L10NTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_L10NTARGETSTACK
 gb_Module_CHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_CHECKTARGETSTACK)),$$(gb_Module_CHECKTARGETSTACK))
 gb_Module_SLOWCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_SLOWCHECKTARGETSTACK)),$$(gb_Module_SLOWCHECKTARGETSTACK))
 gb_Module_SUBSEQUENTCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_SUBSEQUENTCHECKTARGETSTACK)),$$(gb_Module_SUBSEQUENTCHECKTARGETSTACK))
+gb_Module_PERFCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_PERFCHECKTARGETSTACK)),$$(gb_Module_PERFCHECKTARGETSTACK))
 gb_Module_CLEANTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_CLEANTARGETSTACK)),$$(gb_Module_CLEANTARGETSTACK))
 
-ifneq ($$(and $$(gb_Module_TARGETSTACK),$$(gb_Module_CHECKTARGETSTACK),$$(gb_Module_SLOWCHECKTARGETSTACK),$$(gb_Module_SUBSEQUENTCHECKTARGETSTACK),$$(gb_Module_L10NTARGETSTACK)),)
+ifneq ($$(and $$(gb_Module_TARGETSTACK),$$(gb_Module_CHECKTARGETSTACK),$$(gb_Module_SLOWCHECKTARGETSTACK),$$(gb_Module_SUBSEQUENTCHECKTARGETSTACK),$$(gb_Module_PERFCHECKTARGETSTACK),$$(gb_Module_L10NTARGETSTACK)),)
 $$(eval $$(call gb_Output_error,Corrupted module target stack!3))
 endif
 
diff --git a/solenv/gbuild/TargetLocations.mk b/solenv/gbuild/TargetLocations.mk
index 68b9e16d0273..a36e629c6ff7 100644
--- a/solenv/gbuild/TargetLocations.mk
+++ b/solenv/gbuild/TargetLocations.mk
@@ -119,6 +119,7 @@ gb_Module_get_l10n_target = $(WORKDIR)/Module/l10n/$(1)
 gb_Module_get_check_target = $(WORKDIR)/Module/check/$(1)
 gb_Module_get_slowcheck_target = $(WORKDIR)/Module/slowcheck/$(1)
 gb_Module_get_subsequentcheck_target = $(WORKDIR)/Module/subsequentcheck/$(1)
+gb_Module_get_perfcheck_target = $(WORKDIR)/Module/perfcheck/$(1)
 gb_Module_get_target = $(WORKDIR)/Module/$(1)
 gb_ObjCxxObject_get_target = $(WORKDIR)/ObjCxxObject/$(1).o
 gb_ObjCObject_get_target = $(WORKDIR)/ObjCObject/$(1).o
diff --git a/solenv/gbuild/extensions/post_SpeedUpTargets.mk b/solenv/gbuild/extensions/post_SpeedUpTargets.mk
index 8967113d905b..8f3280f3c5aa 100644
--- a/solenv/gbuild/extensions/post_SpeedUpTargets.mk
+++ b/solenv/gbuild/extensions/post_SpeedUpTargets.mk
@@ -20,6 +20,10 @@ ifeq ($(MAKECMDGOALS),build)
 gb_Module_SKIPTARGETS := check slowcheck subsequentcheck
 endif
 
+ifeq (,$(filter perfcheck,$(MAKECMDGOALS)))
+gb_Module_SKIPTARGETS += perfcheck
+endif
+
 ifneq ($(strip $(MAKECMDGOALS)),)
 # speed up depending on the target
 gb_SpeedUpTargets_LEVEL_4 := debugrun help translations install-package-% packageinfo
@@ -63,6 +67,10 @@ ifneq (,$(filter subsequentcheck,$(gb_Module_SKIPTARGETS)))
 gb_Module_add_subsequentcheck_target =
 endif
 
+ifneq (,$(filter perfcheck,$(gb_Module_SKIPTARGETS)))
+gb_Module_add_perfcheck_target =
+endif
+
 ifneq (,$(filter module,$(gb_Module_SKIPTARGETS)))
 gb_Module_add_moduledir =
 endif
diff --git a/solenv/gbuild/gbuild.help.txt b/solenv/gbuild/gbuild.help.txt
index a4396f1957b2..4831e719a36f 100644
--- a/solenv/gbuild/gbuild.help.txt
+++ b/solenv/gbuild/gbuild.help.txt
@@ -32,6 +32,7 @@ AVAILABLE TARGETS
        unitcheck        run unit tests
        slowcheck        run slow unit tests
        subsequentcheck  run system tests (requires full installation)
+       perfcheck        run performance/callgrind unit tests
        check            run unit tests and if in toplevel subsequentcheck
        clean            remove all generated files
        debugrun         starts the INSTDIR instance and allows tests to be run