runtime(syntax-tests): Apply stronger synchronisation between buffers

The current lightweight synchronisation with ":redraw" needs further
reinforcement in the light of v9.1.1110.  And, with v9.1.0820, make
another synchronisation point _before_ the first (or only) screenful is
dumped.

Also add a script to regenerate all screendumps.

closes: #16632

Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

Filelist

@@ -878,6 +878,7 @@ RT_SCRIPTS =	\
 		runtime/syntax/testdir/input/setup/*.* \
 		runtime/syntax/testdir/dumps/*.dump \
 		runtime/syntax/testdir/dumps/*.vim \
+		runtime/syntax/testdir/tools/* \
 		runtime/syntax/generator/Makefile \
 		runtime/syntax/generator/README.md \
 		runtime/syntax/generator/gen_syntax_vim.vim \

runtime/syntax/Makefile

@@ -3,7 +3,7 @@
 # To run the test manually:
 # ../../src/vim -u 'testdir/runtest.vim' --cmd 'breakadd func RunTest'
 
-# Override this if needed, the default assumes Vim was build in the src dir.
+# Override this if needed, the default assumes Vim was built in the src dir.
 #VIMPROG = vim
 VIMPROG = ../../src/vim
 
@@ -13,6 +13,10 @@ VIMRUNTIME = ../..
 # Uncomment this line to use valgrind for memory leaks and extra warnings.
 # VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=45 --log-file=valgrind.$*
 
+# Trace ruler liveness on demand.
+# VIM_SYNTAX_TEST_LOG = `pwd`/testdir/failed/00-TRACE_LOG
+
+# ENVVARS = LC_ALL=C VIM_SYNTAX_TEST_LOG="$(VIM_SYNTAX_TEST_LOG)"
 # ENVVARS = LC_ALL=C LANG=C LANGUAGE=C
 # Run the syntax tests with a C locale
 ENVVARS = LC_ALL=C
@@ -31,6 +35,9 @@ test:
 	@# the "vimcmd" file is used by the screendump utils
 	@echo "../$(VIMPROG)" > testdir/vimcmd
 	@echo "$(RUN_VIMTEST)" >> testdir/vimcmd
+	@# Trace ruler liveness on demand.
+	@#mkdir -p testdir/failed
+	@#touch "$(VIM_SYNTAX_TEST_LOG)"
 	VIMRUNTIME=$(VIMRUNTIME) $(ENVVARS) $(VIMPROG) --clean --not-a-term $(DEBUGLOG) -u testdir/runtest.vim > /dev/null 
 	@rm -f testdir/Xfilter
 	@# FIXME: Temporarily show the whole file to find out what goes wrong

runtime/syntax/testdir/runtest.vim

@@ -15,6 +15,17 @@ let s:messagesFname = fnameescape(syntaxDir .. '/testdir/messages')
 
 let s:messages = []
 
+" Erase the cursor line and do not advance the cursor.
+def EraseLineAndReturnCarriage(rname: string)
+  const full_width: number = winwidth(0)
+  const half_width: number = full_width - (full_width + 1) / 2
+  if (strlen(rname) + strlen('Test' .. "\x20\x20" .. 'FAILED')) > half_width
+    echon "\r" .. repeat("\x20", full_width) .. "\r"
+  else
+    echon repeat("\x20", half_width) .. "\r"
+  endif
+enddef
+
 " Add one message to the list of messages
 func Message(msg)
   echomsg a:msg
@@ -30,22 +41,23 @@ endfunc
 
 " Append s:messages to the messages file and make it empty.
 func AppendMessages(header)
-  exe 'split ' .. s:messagesFname
+  silent exe 'split ' .. s:messagesFname
   call append(line('$'), '')
   call append(line('$'), a:header)
   call append(line('$'), s:messages)
   let s:messages = []
-  wq
+  silent wq
 endfunc
 
 " Relevant messages are written to the "messages" file.
 " If the file already exists it is appended to.
-exe 'split ' .. s:messagesFname
+silent exe 'split ' .. s:messagesFname
 call append(line('$'), repeat('=-', 70))
 call append(line('$'), '')
 let s:test_run_message = 'Test run on ' .. strftime("%Y %b %d %H:%M:%S")
 call append(line('$'), s:test_run_message)
-wq
+silent wq
+echo "\n"
 
 if syntaxDir !~ '[/\\]runtime[/\\]syntax\>'
   call Fatal('Current directory must be "runtime/syntax"')
@@ -88,26 +100,127 @@ func HandleSwapExists()
   endif
 endfunc
 
-def IsWinNumOneAtEOF(in_name_and_out_name: string): bool
+" Trace ruler liveness on demand.
-  # Expect defaults from term_util#RunVimInTerminal().
+if !empty($VIM_SYNTAX_TEST_LOG) && filewritable($VIM_SYNTAX_TEST_LOG)
+  def s:TraceRulerLiveness(context: string, times: number, tail: string)
+    writefile([printf('%s: %4d: %s', context, times, tail)],
+	$VIM_SYNTAX_TEST_LOG,
+	'a')
+  enddef
+else
+  def s:TraceRulerLiveness(_: string, _: number, _: string)
+  enddef
+endif
+
+" See ":help 'ruler'".
+def s:CannotSeeLastLine(ruler: list<string>): bool
+  return !(get(ruler, -1, '') ==# 'All' || get(ruler, -1, '') ==# 'Bot')
+enddef
+
+def s:CannotDumpNextPage(buf: number, prev_ruler: list<string>, ruler: list<string>): bool
+  return !(ruler !=# prev_ruler &&
+      len(ruler) == 2 &&
+      ruler[1] =~# '\%(\d%\|\<Bot\)$' &&
+      get(term_getcursor(buf), 0) != 20)
+enddef
+
+def s:CannotDumpFirstPage(buf: number, _: list<string>, ruler: list<string>): bool
+  return !(len(ruler) == 2 &&
+      ruler[1] =~# '\%(\<All\|\<Top\)$' &&
+      get(term_getcursor(buf), 0) != 20)
+enddef
+
+def s:CannotDumpShellFirstPage(buf: number, _: list<string>, ruler: list<string>): bool
+  return !(len(ruler) > 3 &&
+      get(ruler, -1, '') =~# '\%(\<All\|\<Top\)$' &&
+      get(term_getcursor(buf), 0) != 20)
+enddef
+
+" Poll for updates of the cursor position in the terminal buffer occupying the
+" first window.  (ALWAYS call the function or its equivalent before calling
+" "VerifyScreenDump()" *and* after calling any number of "term_sendkeys()".)
+def s:TermPollRuler(
+	CannotDumpPage: func,	# (TYPE FOR LEGACY CONTEXT CALL SITES.)
+	buf: number,
+	in_name_and_out_name: string): list<string>
+  # Expect defaults from "term_util#RunVimInTerminal()".
   if winwidth(1) != 75 || winheight(1) != 20
     ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)',
       in_name_and_out_name,
       winwidth(1),
       winheight(1)))
-    return true
+    return ['0,0-1', 'All']
   endif
-  # A two-fold role: (1) redraw whenever the first test file is of 19 lines or
+  # A two-fold role for redrawing:
-  # less long (not applicable to c.c); (2) redraw in case the terminal buffer
+  # (*) in case the terminal buffer cannot redraw itself just yet;
-  # cannot redraw itself just yet (else expect extra files generated).
+  # (*) to avoid extra "real estate" checks.
   redraw
-  const pos: string = join([
+  # The contents of "ruler".
-    screenstring(20, 71),
+  var ruler: list<string> = []
-    screenstring(20, 72),
+  # Attempts at most, targeting ASan-instrumented Vim builds.
-    screenstring(20, 73),
+  var times: number = 2048
-    screenstring(20, 74),
+  # Check "real estate" of the terminal buffer.  Read and compare its ruler
-    screenstring(20, 75)], '')
+  # line and let "Xtestscript#s:AssertCursorForwardProgress()" do the rest.
-  return (pos == ' All ' || pos == ' Bot ')
+  # Note that the cursor ought to be advanced after each successive call of
+  # this function yet its relative position need not be changed (e.g. "0%").
+  while CannotDumpPage(ruler) && times > 0
+    ruler = split(term_getline(buf, 20))
+    sleep 1m
+    times -= 1
+    if times % 8 == 0
+      redraw
+    endif
+  endwhile
+  TraceRulerLiveness('P', (2048 - times), in_name_and_out_name)
+  return ruler
+enddef
+
+" Prevent "s:TermPollRuler()" from prematurely reading the cursor position,
+" which is available at ":edit", after outracing the loading of syntax etc. in
+" the terminal buffer.  (Call the function before calling "VerifyScreenDump()"
+" for the first time.)
+def s:TermWaitAndPollRuler(buf: number, in_name_and_out_name: string): list<string>
+  # Expect defaults from "term_util#RunVimInTerminal()".
+  if winwidth(1) != 75 || winheight(1) != 20
+    ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)',
+      in_name_and_out_name,
+      winwidth(1),
+      winheight(1)))
+    return ['0,0-1', 'All']
+  endif
+  # The contents of "ruler".
+  var ruler: string = ''
+  # Attempts at most, targeting ASan-instrumented Vim builds.
+  var times: number = 32768
+  # Check "real estate" of the terminal buffer.  Expect a known token to be
+  # rendered in the terminal buffer; its prefix must be "is_" so that buffer
+  # variables from "sh.vim" can be matched (see "Xtestscript#ShellInfo()").
+  # Verify that the whole line is available!
+  while ruler !~# '^is_.\+\s\%(All\|Top\)$' && times > 0
+    ruler = term_getline(buf, 20)
+    sleep 1m
+    times -= 1
+    if times % 16 == 0
+      redraw
+    endif
+  endwhile
+  TraceRulerLiveness('W', (32768 - times), in_name_and_out_name)
+  if strpart(ruler, 0, 8) !=# 'is_nonce'
+    # Retain any of "b:is_(bash|dash|kornshell|posix|sh)" entries and let
+    # "CannotDumpShellFirstPage()" win the cursor race.
+    return TermPollRuler(
+	function(CannotDumpShellFirstPage, [buf, []]),
+	buf,
+	in_name_and_out_name)
+  else
+    # Clear the "is_nonce" token and let "CannotDumpFirstPage()" win any
+    # race.
+    term_sendkeys(buf, ":redraw!\<CR>")
+  endif
+  return TermPollRuler(
+      function(CannotDumpFirstPage, [buf, []]),
+      buf,
+      in_name_and_out_name)
 enddef
 
 func RunTest()
@@ -337,41 +450,44 @@ func RunTest()
       " load filetype specific settings
       call term_sendkeys(buf, ":call LoadFiletype('" .. filetype .. "')\<CR>")
 
+      " Make a synchronisation point between buffers by requesting to echo
+      " a known token in the terminal buffer and asserting its availability
+      " with "s:TermWaitAndPollRuler()".
       if filetype == 'sh'
 	call term_sendkeys(buf, ":call ShellInfo()\<CR>")
+      else
+	call term_sendkeys(buf, ":echo 'is_nonce'\<CR>")
       endif
 
-      " Screendump at the start of the file: failed/root_00.dump
       let root_00 = root .. '_00'
       let in_name_and_out_name = fname .. ': failed/' .. root_00 .. '.dump'
+      " Queue up all "term_sendkeys()"es and let them finish before returning
+      " from "s:TermWaitAndPollRuler()".
+      let ruler = s:TermWaitAndPollRuler(buf, in_name_and_out_name)
       call ch_log('First screendump for ' .. in_name_and_out_name)
+      " Make a screendump at the start of the file: failed/root_00.dump
       let fail = VerifyScreenDump(buf, root_00, {})
 
-      " Make a Screendump every 18 lines of the file: failed/root_NN.dump
-      let nr = 1
-      let root_next = printf('%s_%02d', root, nr)
-      let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump'
-
       " Accommodate the next code block to "buf"'s contingency for self
       " wipe-out.
       try
-	if !IsWinNumOneAtEOF(in_name_and_out_name)
+	let nr = 0
-	  call term_sendkeys(buf, ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\<CR>")
+	let keys_a = ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
-	  call ch_log('Next screendump for ' .. in_name_and_out_name)
+	let keys_b = ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
-	  let fail += VerifyScreenDump(buf, root_next, {})
+	while s:CannotSeeLastLine(ruler)
+	  call term_sendkeys(buf, keys_a)
+	  let keys_a = keys_b
 	  let nr += 1
 	  let root_next = printf('%s_%02d', root, nr)
 	  let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump'
-
+	  let ruler = s:TermPollRuler(
-	  while !IsWinNumOneAtEOF(in_name_and_out_name)
+	      \ function('s:CannotDumpNextPage', [buf, ruler]),
-	    call term_sendkeys(buf, ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\<CR>")
+	      \ buf,
-	    call ch_log('Next screendump for ' .. in_name_and_out_name)
+	      \ in_name_and_out_name)
-	    let fail += VerifyScreenDump(buf, root_next, {})
+	  call ch_log('Next screendump for ' .. in_name_and_out_name)
-	    let nr += 1
+	  " Make a screendump of every 18 lines of the file: failed/root_NN.dump
-	    let root_next = printf('%s_%02d', root, nr)
+	  let fail += VerifyScreenDump(buf, root_next, {})
-	    let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump'
+	endwhile
-	  endwhile
-	endif
 	call StopVimInTerminal(buf)
       finally
 	call delete('Xtestscript')
@@ -413,6 +529,8 @@ func RunTest()
       let skipped_count += 1
     endif
 
+    call EraseLineAndReturnCarriage(root)
+
     " Append messages to the file "testdir/messages"
     call AppendMessages('Input file ' .. fname .. ':')
 
@@ -421,6 +539,7 @@ func RunTest()
     endif
   endfor
 
+  call EraseLineAndReturnCarriage('')
   call Message(s:test_run_message)
   call Message('OK: ' .. ok_count)
   call Message('FAILED: ' .. len(failed_tests) .. ': ' .. string(failed_tests))
@@ -446,4 +565,4 @@ endif
 
 qall!
 
-" vim:ts=8
+" vim:sw=2:ts=8:noet:

runtime/syntax/testdir/tools/regenerate_screendumps.sh

@@ -0,0 +1,126 @@
+#!/bin/sh -e
+#
+# The following steps are to be taken by this script:
+# 1) Remove all files from the "dumps" directory.
+# 2) Generate screendumps for each syntax test and each self-test.
+# 3) Unconditionally move each batch of screendumps to "dumps"; if generated
+#	files differ on repeated runs, always remove these files from "dumps".
+# 4) Repeat steps 2) and 3) once or as many times as requested with the "$1"
+#	argument.
+# 5) Summarise any differences.
+#
+# Provided that "git difftool" is set up (see src/testdir/commondumps.vim),
+# run "git difftool HEAD -- '**/*.dump'" to collate tracked and generated
+# screendumps.
+
+case "$1" in
+-h | --help)
+	printf >&2 "Usage: [time VIM_SYNTAX_TEST_LOG=/tmp/log] $0 [1 | 2 | ...]\n"
+	exit 0
+	;;
+esac
+
+tries="${1:-1}"
+shift $#
+
+case "$tries" in
+0* | *[!0-9]*)
+	exit 80
+	;;
+esac
+
+test -x "$(command -v make)"	|| exit 81
+test -x "$(command -v git)"	|| exit 82
+
+case "$(git status --porcelain=v1)" in
+'')	;;
+*)	printf >&2 'Resolve ALL changes before proceeding.\n'
+	exit 83
+	;;
+esac
+
+templet=$(printf "\t\t\t\t$(tput rev)%%s$(tput sgr0)") || exit 84
+cd "$(dirname "$0")/../../../syntax" || exit 85
+set +f
+rm testdir/dumps/*.dump || exit 86
+spuriosities=''
+
+# Because the clean target of Make will be executed before each syntax test,
+# this environment variable needs to be pointed to an existing file that is
+# created in a directory not affectable by the target.
+if test -w "$VIM_SYNTAX_TEST_LOG"
+then
+	log=-e VIM_SYNTAX_TEST_LOG="$VIM_SYNTAX_TEST_LOG"
+else
+	log=
+fi
+
+for f in testdir/input/*.*
+do
+	test ! -d "$f" || continue
+	b=$(basename "$f")
+	i=0
+	printf "$templet\n\n" "$b"
+
+	while test "$i" -le "$tries"
+	do
+		make $log clean "$b" test || :
+
+		case "$i" in
+		0)	mv testdir/failed/*.dump testdir/dumps/
+			;;
+		*)	case "$(printf '%s' testdir/failed/*.dump)" in
+			testdir/failed/\*.dump)
+				# (Repeatable) success.
+				;;
+			*)	spuriosities="${spuriosities}${b} "
+				p=${b%.*}
+				rm -f testdir/dumps/"$p"_[0-9][0-9].dump \
+					testdir/dumps/"$p"_[0-9][0-9][0-9].dump \
+					testdir/dumps/"$p"_[0-9][0-9][0-9][0-9].dump
+				;;
+			esac
+			;;
+		esac
+
+		i=$(($i + 1))
+		sleep 1
+	done
+done
+
+# For a 20-file set, initially fail for a series of: 1-6, 7-12, 13-18, 19-20.
+tries=$(($tries + 3))
+i=0
+
+while test "$i" -le "$tries"
+do
+	make $log clean self-testing test || :
+
+	case "$i" in
+	[0-3])	mv testdir/failed/dots_*.dump testdir/dumps/
+		;;
+	*)	case "$(printf '%s' testdir/failed/*.dump)" in
+		testdir/failed/\*.dump)
+			# (Repeatable) success.
+			;;
+		*)	spuriosities="${spuriosities}dots_xy "
+			rm -f testdir/dumps/dots_*.dump
+			;;
+		esac
+		;;
+	esac
+
+	sleep 1
+	i=$(($i + 1))
+done
+
+make clean
+git diff --compact-summary
+
+if test -n "$spuriosities"
+then
+	printf '\n%s\n' "$spuriosities"
+	exit 87
+fi
+
+# vim:sw=8:ts=8:noet:nosta: