From 9cc95aa0d8f52bb0c906ca13297ce3a2ba615f2b Mon Sep 17 00:00:00 2001
From: Peter Simonyi <the.peter.simonyi@gmail.com>
Date: Fri, 1 Dec 2023 12:07:42 -0500
Subject: [PATCH] CI: check that all files are listed in Filelist (#13601)

Sometimes patches add files that should be included in tarballs for
distribution, but are not added to Filelist (used by Makefile to build
the tar archive).  This can break the build, or it can be silently
ignored as runtime files are simply not included in the distribution.

Add a CI check to ensure all files tracked in the repository are
assigned to a variable in Filelist.  A few files were not listed because
they do not need to be included in builds and tarballs, so add an IGNORE
variable for these exceptions.

Co-authored-by: Peter Simonyi <pts@petersimonyi.ca>
Signed-off-by: Christian Brabandt <cb@256bit.org>
---
 .github/workflows/ci.yml |  6 +++++
 Filelist                 | 14 ++++++++++++
 ci/unlisted.make         | 49 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 69 insertions(+)
 create mode 100644 ci/unlisted.make

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cd1615f9f4..2042b8ed7e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -81,6 +81,12 @@ jobs:
       - name: Checkout repository from github
         uses: actions/checkout@v4
 
+      - name: Check Filelist (for packaging)
+        run: |
+          # If any files in the repository are not listed in Filelist this will
+          # exit with an error code and list the missing entries.
+          make -f ci/unlisted.make
+
       - run: sudo dpkg --add-architecture i386
         if: matrix.architecture == 'i386'
 
diff --git a/Filelist b/Filelist
index 21abf4b8a2..efb2daa12c 100644
--- a/Filelist
+++ b/Filelist
@@ -213,6 +213,7 @@ SRC_ALL =	\
 		src/testdir/dumps/*.dump \
 		src/testdir/dumps/*.vim \
 		src/testdir/samples/*.txt \
+		src/testdir/samples/*.vim \
 		src/testdir/samples/test000 \
 		src/testdir/color_ramp.vim \
 		src/testdir/silent.wav \
@@ -1074,4 +1075,17 @@ LANG_DOS = \
 		src/po/*.mo \
 		runtime/lang/Make_mvc.mak \
 
+# Files in the repository that are deliberately not listed above, and will thus
+# be excluded from distribution tarballs and the like.
+# This excludes them from the CI check for unlisted files.
+IGNORE = \
+		.appveyor.yml \
+		.github/FUNDING.yml \
+		.github/labeler.yml \
+		.github/workflows/label.yml \
+		SECURITY.md \
+		ci/unlisted.make \
+		src/libvterm/CODE-MAP \
+		runtime/syntax/testdir/input/html_html \
+
 # vim: set ft=make:
diff --git a/ci/unlisted.make b/ci/unlisted.make
new file mode 100644
index 0000000000..04dfcb53f3
--- /dev/null
+++ b/ci/unlisted.make
@@ -0,0 +1,49 @@
+# vim: ft=make
+SHELL = /bin/bash
+
+# List all files that are tracked in git but not listed in Filelist.
+# Exit code is 2 ("Make encountered an error") if any such files exist.
+
+# Filelist is a Makefile that defines many variables, so we use Make itself to
+# query which variables it defines, then expand them all by wrapping each name
+# in $(...), importing Filelist and using $(eval).
+
+include Filelist
+$(eval all_patterns := $(shell \
+	make -f Filelist --question --print-data-base --no-builtin-rules \
+		--no-builtin-variables 2>/dev/null \
+	| sed -nre \
+		'/^# makefile .from \x27Filelist\x27,/ { \
+			n; \
+			s/ = .*//; \
+			T; \
+			s/.*/$$(\0)/; \
+			p; \
+		}'))
+
+# In Makefile's `prepeare` target, all the IN_README_DIR files are moved from
+# READMEdir to the root, so add those files in their Git-tracked location:
+all_patterns := $(all_patterns) \
+	$(foreach readme, $(IN_README_DIR), READMEdir/$(readme))
+
+# The result 'all_patterns' is a list of patterns (globs), which we expand with
+# wildcard to get actual filenames.  Note this means Filelist can list a file
+# that does not exist, and it will be omitted at this step.
+listed_files := $(wildcard $(all_patterns))
+
+# Default target to actually run the comparison:
+.PHONY: check
+check:
+	@# There are too many files to list on the command line, so we write
+	@# that to a temporary file, one per line.
+	$(file > Filelist-listed-files)
+	$(foreach filename, $(listed_files),\
+		$(file >> Filelist-listed-files,$(filename)))
+	@# Compare the sorted lists.  Delete that temporary file on both
+	@# success and failure, but exit with diff's exit code.
+	diff -u0 --label files-in-git <(git ls-files | sort) \
+		--label Filelist <(sort --unique Filelist-listed-files); \
+	RV=$$?; \
+	rm Filelist-listed-files; \
+	(($$RV != 0)) && echo "Add files to the right variable in Filelist."; \
+	exit $$RV