patch 9.1.0173: msgfmt ver. 0.22 forcibly converts text to UTF-8

Problem:  msgfmt ver. 0.22 forcibly converts text to UTF-8
Solution: use '--no-convert' if msgfmt supports it. Add a configure
          check for the msgfmt version (RestorerZ).

closes: #14163

Co-authored-by: Christian Brabandt <cb@256bit.org>
Signed-off-by: RestorerZ <restorer@mail2k.ru>
Signed-off-by: Christian Brabandt <cb@256bit.org>

src/auto/configure

@@ -650,6 +650,7 @@ ac_subst_vars='LTLIBOBJS
 LIBOBJS
 LINK_AS_NEEDED
 DEPEND_CFLAGS_FILTER
+MSGFMTCMD
 MSGFMT_DESKTOP
 MAKEMO
 MSGFMT
@@ -15964,6 +15965,18 @@ printf "%s\n" "yes" >&6; }
 printf "%s\n" "no" >&6; }
       fi
 
+      { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if msgfmt supports --no-convert" >&5
+printf %s "checking if msgfmt supports --no-convert... " >&6; }
+      if "$MSGFMT" --help | grep -q --  '--no-convert' >/dev/null; then
+        { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+        MSGFMTCMD="OLD_PO_FILE_INPUT=yes msgfmt --no-convert -v"
+      else
+        { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+        MSGFMTCMD="OLD_PO_FILE_INPUT=yes msgfmt -v"
+      fi
+
     fi
   else
     { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no \"po/Makefile\" - disabled" >&5

src/config.mk.in

@@ -175,6 +175,7 @@ OS_EXTRA_OBJ	= @OS_EXTRA_OBJ@
 MAKEMO		= @MAKEMO@
 
 MSGFMT		= @MSGFMT@
+MSGFMTCMD	= @MSGFMTCMD@
 MSGFMT_DESKTOP	= @MSGFMT_DESKTOP@
 
 ### set if $SOURCE_DATE_EPOCH was set when running configure

src/configure.ac

@@ -4523,6 +4523,15 @@ if test "$enable_nls" = "yes"; then
 	AC_MSG_RESULT([no])
       fi
       AC_SUBST(MSGFMT_DESKTOP)
+      AC_MSG_CHECKING([if msgfmt supports --no-convert])
+      if "$MSGFMT" --help | grep -q --  '--no-convert' >/dev/null; then
+        AC_MSG_RESULT([yes])
+        MSGFMTCMD="OLD_PO_FILE_INPUT=yes msgfmt --no-convert -v"
+      else
+        AC_MSG_RESULT([no])
+        MSGFMTCMD="OLD_PO_FILE_INPUT=yes msgfmt -v"
+      fi
+      AC_SUBST(MSGFMTCMD)
     fi
   else
     AC_MSG_RESULT([no "po/Makefile" - disabled]);

src/po/Make_mvc.mak

@@ -12,8 +12,8 @@
 !ENDIF
 
 !IFNDEF LANGUAGE
-! IF [powershell.exe -nologo -noprofile $$lng=(Get-UICulture).TwoLetterISOLanguageName;$$Env:LANGUAGE=$$lng;Set-Content -Path .\lng.tmp -Value "LANGUAGE=$$lng"]
+! IF [powershell.exe -nologo -noprofile $$lng=(Get-UICulture).TwoLetterISOLanguageName; \
-#! IF [powershell.exe -nologo -noprofile -command $$Env:LANGUAGE=(Get-UICulture).TwoLetterISOLanguageName]
+	$$Env:LANGUAGE=$$lng;Set-Content -Path .\lng.tmp -Value "LANGUAGE=$$lng"]
 ! ENDIF
 # In order for the "install" and "cleanup-po" rule to work.
 # The others work with just setting the environment variable.
@@ -30,7 +30,7 @@
 ! MESSAGE LANGUAGE is already set "$(LANGUAGE)"
 !ENDIF
 
-# get LANGUAGES, MOFILES, MOCONVERTED and others
+# Get LANGUAGES, MOFILES, MOCONVERTED and others.
 !INCLUDE Make_all.mak
 
 !IFNDEF VIMRUNTIME
@@ -46,7 +46,15 @@ VIM = ..\vim.exe
 # installed.  Please do not put the path in quotes.
 GETTEXT_PATH = D:\Programs\GetText\bin
 
+# Starting from version 0.22, msgfmt forcibly converts text to UTF-8 regardless
+# of the value of the "charset" field.
+!IF [%comspec% /v:on /e:on /c "for /F "tokens=4 delims= " %G in \
+	('"$(GETTEXT_PATH)\msgfmt.exe" --version^|findstr /rc:[0-9^]\.[0-9^][0-9^]') do \
+		@(set "v=%G" && if !v:~2^,2! GEQ 22 exit /b 1)"]
+MSGFMT = "$(GETTEXT_PATH)\msgfmt.exe" -v --no-convert
+!ELSE
 MSGFMT = "$(GETTEXT_PATH)\msgfmt.exe" -v
+!ENDIF
 XGETTEXT = "$(GETTEXT_PATH)\xgettext.exe"
 MSGMERGE = "$(GETTEXT_PATH)\msgmerge.exe"
 
@@ -55,9 +63,9 @@ MSGMERGE = "$(GETTEXT_PATH)\msgmerge.exe"
 # If the "iconv" program is installed on the system, but it is not registered
 # in the %PATH% environment variable, then specify the full path to this file.
 !IF EXIST ("iconv.exe")
-ICONV = "iconv.exe"
+ICONV = iconv.exe
 !ELSEIF EXIST ("$(GETTEXT_PATH)\iconv.exe")
-ICONV="$(GETTEXT_PATH)\iconv.exe"
+ICONV = "$(GETTEXT_PATH)\iconv.exe"
 !ENDIF
 
 # In case some package like GnuWin32, UnixUtils
@@ -133,7 +141,7 @@ ja.sjis.po: ja.po
 sjiscorr: sjiscorr.c
 	$(CC) sjiscorr.c
 
-# Convert ja.po to create ja.euc-jp.po
+# Convert ja.po to create ja.euc-jp.po.
 ja.euc-jp.po: ja.po
 	-$(RM) $@
 !IF EXIST ("$(GETTEXT_PATH)\msgconv.exe")
@@ -313,7 +321,7 @@ zh_CN.cp936.po: zh_CN.UTF-8.po
 		[System.IO.File]::WriteAllText(\"$@\", $$out, \
 		[System.Text.Encoding]::GetEncoding(20936))
 
-# Convert zh_TW.UTF-8.po to create zh_TW.po
+# Convert zh_TW.UTF-8.po to create zh_TW.po.
 zh_TW.po: zh_TW.UTF-8.po
 	-$(RM) $@
 !IF EXIST ("$(GETTEXT_PATH)\msgconv.exe")
@@ -341,7 +349,7 @@ zh_TW.po: zh_TW.UTF-8.po
 		[System.IO.File]::WriteAllText(\"$@\", $$out, \
 		[System.Text.Encoding]::GetEncoding(950))
 
-# Convert zh_TW.UTF-8.po to create zh_TW.po with backslash characters
+# Convert zh_TW.UTF-8.po to create zh_TW.po with backslash characters.
 # Requires doubling backslashes in the second byte.  Don't depend on big5corr,
 # it should only be compiled when zh_TW.po is outdated.
 
@@ -379,7 +387,7 @@ zh_TW.po: zh_TW.UTF-8.po
 #		[System.IO.File]::WriteAllText(\"$@\", $$out, \
 #		[System.Text.Encoding]::GetEncoding(950))
 
-# see above in the zh_TW.po conversion section for backslashes.
+# See above in the zh_TW.po conversion section for backslashes.
 #big5corr: big5corr.c
 #	$(CC) big5corr.c
 
@@ -512,7 +520,7 @@ $(PACKAGE).pot: files
 # The files that are converted to a different encoding clearly state "DO NOT EDIT".
 update-po: $(MOFILES:.mo=)
 
-# Don't add a dependency here, we only want to update the .po files manually
+# Don't add a dependency here, we only want to update the .po files manually.
 $(LANGUAGES):
 	@$(MAKE) -nologo -f Make_mvc.mak GETTEXT_PATH="$(GETTEXT_PATH)" $(PACKAGE).pot
 	$(CP) $@.po $@.po.orig
@@ -521,14 +529,14 @@ $(LANGUAGES):
 	$(RM) $@.po.old
 
 install: $(LANGUAGE).mo
-	if not exist $(INSTALLDIR) $(MKD) $(INSTALLDIR)
+	if not exist "$(INSTALLDIR)" $(MKD) "$(INSTALLDIR)"
-	$(CP) $(LANGUAGE).mo $(INSTALLDIR)\$(PACKAGE).mo
+	$(CP) $(LANGUAGE).mo "$(INSTALLDIR)\$(PACKAGE).mo"
 
 install-all: all
-	for %%l in ($(LANGUAGES)) do @if not exist $(VIMRUNTIME)\lang\%%l\LC_MESSAGES \
+	for %%l in ($(LANGUAGES)) do @if not exist "$(VIMRUNTIME)\lang\%%l\LC_MESSAGES" \
-		$(MKD) $(VIMRUNTIME)\lang\%%l\LC_MESSAGES
+		$(MKD) "$(VIMRUNTIME)\lang\%%l\LC_MESSAGES"
 	for %%l in ($(LANGUAGES)) do @$(CP) %%l.mo \
-		$(VIMRUNTIME)\lang\%%l\LC_MESSAGES\$(PACKAGE).mo
+		"$(VIMRUNTIME)\lang\%%l\LC_MESSAGES\$(PACKAGE).mo"
 
 cleanup-po: $(LANGUAGE).po
 	"$(VIM)" -u NONE -e -X -S cleanup.vim -c wq $(LANGUAGE).po

src/po/Makefile

@@ -3,24 +3,25 @@
 # Include stuff found by configure.
 include ../auto/config.mk
 
-# get LANGUAGES, MOFILES, MOCONVERTED and others
+# Get LANGUAGES, MOFILES, MOCONVERTED and others.
 include Make_all.mak
 
 # Note: ja.sjis, *.cp1250 and zh_CN.cp936 are only for MS-Windows, they are
-# not installed on Unix
+# not installed on Unix.
 
 PACKAGE = vim
 SHELL = /bin/sh
 VIM = ../vim
 
-# MacOS sed is locale aware, set $LANG to avoid problems
+# MacOS sed is locale aware, set $LANG to avoid problems.
 SED = LANG=C sed
 
 # The OLD_PO_FILE_INPUT and OLD_PO_FILE_OUTPUT are for the new GNU gettext
 # tools 0.10.37, which use a slightly different .po file format that is not
 # compatible with Solaris (and old gettext implementations) unless these are
 # set.  gettext 0.10.36 will not work!
-MSGFMTCMD = OLD_PO_FILE_INPUT=yes $(MSGFMT) -v
+
+# MSGFMTCMD is defined by Configure in ../auto/config.mk
 XGETTEXT = OLD_PO_FILE_INPUT=yes OLD_PO_FILE_OUTPUT=yes xgettext
 MSGMERGE = OLD_PO_FILE_INPUT=yes OLD_PO_FILE_OUTPUT=yes msgmerge
 
@@ -45,7 +46,7 @@ converted: $(MOCONVERTED)
 
 check: $(CHECKFILES)
 
-# installing for real
+# Installing for real.
 install: $(MOFILES) $(MOCONVERTED)
 	@$(MAKE) prefixcheck
 	for lang in $(LANGUAGES); do \
@@ -71,7 +72,7 @@ uninstall:
 	  rm -f $(LOCALEDIR)/$$lang/LC_MESSAGES/$(PACKAGE).mo; \
 	done
 
-# installing for local tryout into ../../runtime/lang
+# Installing for local tryout into ../../runtime/lang.
 tryoutinstall: $(MOFILES) $(MOCONVERTED)
 	@$(MAKE) prefixcheck
 	for lang in $(LANGUAGES); do \
@@ -167,7 +168,7 @@ zh_CN.cp936.po: zh_CN.UTF-8.po
 			-e 's/# Original translations/# Generated from $<, DO NOT EDIT/' \
 			> $@
 
-# Convert zh_TW.UTF-8.po to create zh_TW.po
+# Convert zh_TW.UTF-8.po to create zh_TW.po.
 zh_TW.po: zh_TW.UTF-8.po
 	rm -f $@
 	iconv -f UTF-8 -t BIG5 $< | \
@@ -176,7 +177,7 @@ zh_TW.po: zh_TW.UTF-8.po
 			> $@
 
 
-# Convert zh_TW.UTF-8.po to create zh_TW.po with backslash characters
+# Convert zh_TW.UTF-8.po to create zh_TW.po with backslash characters.
 # Requires doubling backslashes in the second byte.  Don't depend on big5corr,
 # it should only be compiled when zh_TW.po is outdated.
 
@@ -200,7 +201,7 @@ zh_TW.po: zh_TW.UTF-8.po
 
 
 # 06.11.23, added by Restorer
-# see above in the zh_tw.po conversion section for backslashes.
+# See above in the zh_tw.po conversion section for backslashes.
 #big5corr: big5corr.c
 #	$(CC) -o big5corr big5corr.c
 
@@ -259,15 +260,15 @@ PO_INPUTLIST = \
 	vim.desktop.in
 
 $(PACKAGE).pot: $(PO_INPUTLIST) $(PO_VIM_INPUTLIST)
-	# Convert the Vim scripts to (what looks like) Javascript
+	# Convert the Vim scripts to (what looks like) Javascript.
 	$(VIM) -u NONE --not-a-term -S tojavascript.vim $(PACKAGE).pot $(PO_VIM_INPUTLIST)
-	# create vim.pot
+	# Create vim.pot.
 	$(XGETTEXT) --default-domain=$(PACKAGE) --add-comments \
 		$(XGETTEXT_KEYWORDS) $(PO_INPUTLIST) $(PO_VIM_JSLIST)
 	mv -f $(PACKAGE).po $(PACKAGE).pot
-	# Fix Vim scripts names, so that "gf" works
+	# Fix Vim scripts names, so that "gf" works.
 	$(VIM) -u NONE --not-a-term -S fixfilenames.vim $(PACKAGE).pot $(PO_VIM_INPUTLIST)
-	# Delete the temporary files
+	# Delete the temporary files.
 	rm *.js
 
 vim.desktop: vim.desktop.in $(POFILES)
@@ -290,7 +291,7 @@ gvim.desktop: gvim.desktop.in $(POFILES) vim.desktop
 # The files that are converted to a different encoding clearly state "DO NOT EDIT".
 update-po: $(MOFILES:.mo=)
 
-# Don't add a dependency here, we only want to update the .po files manually
+# Don't add a dependency here, we only want to update the .po files manually.
 $(LANGUAGES):
 	@$(MAKE) $(PACKAGE).pot
 	if test ! -f $@.po.orig; then cp $@.po $@.po.orig; fi

src/version.c

@@ -704,6 +704,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    173,
 /**/
     172,
 /**/