patch 9.1.1339: missing out-of-memory checks for enc_to_utf16()/utf16_to_enc()

Problem:  missing out-of-memory checks for enc_to_utf16() and
          utf16_to_enc()
Solution: Add out-of-memory checks and fix a few other minor issues
          (John Marriott)

This change does:
-  add missing out-of-memory checks for enc_to_utf16() and
   utf16_to_enc()
-  add a small optimisation in mch_errmsg_c() and mch_msg_c() (in
   message.c) to only call STRLEN() if needed.
-  fix a memory leak in winpty_term_and_job_init() (in terminal.c).

closes: #17191

Signed-off-by: John Marriott <basilisk@internode.on.net>
Signed-off-by: Christian Brabandt <cb@256bit.org>

src/fileio.c

@@ -5476,6 +5476,8 @@ vim_tempname(
     // "sh".  NOTE: This also checks 'shellcmdflag' to help those people who
     // didn't set 'shellslash' but only if not using PowerShell.
     retval = utf16_to_enc(itmp, NULL);
+    if (retval == NULL)
+	return NULL;
     shname = gettail(p_sh);
     if ((*p_shcf == '-' && !(strstr((char *)shname, "powershell") != NULL
 			     || strstr((char *)shname, "pwsh") != NULL ))

src/gui_w32.c

@@ -7127,8 +7127,11 @@ dialog_callback(
 
 	    GetDlgItemTextW(hwnd, DLG_NONBUTTON_CONTROL + 2, wp, IOSIZE);
 	    p = utf16_to_enc(wp, NULL);
-	    vim_strncpy(s_textfield, p, IOSIZE);
+	    if (p != NULL)
-	    vim_free(p);
+	    {
+		vim_strncpy(s_textfield, p, IOSIZE);
+		vim_free(p);
+	    }
 	    vim_free(wp);
 	}
 

src/message.c

@@ -3508,7 +3508,6 @@ do_more_prompt(int typed_char)
     static void
 mch_errmsg_c(char *str)
 {
-    int	    len = (int)STRLEN(str);
     DWORD   nwrite = 0;
     DWORD   mode = 0;
     HANDLE  h = GetStdHandle(STD_ERROR_HANDLE);
@@ -3516,10 +3515,14 @@ mch_errmsg_c(char *str)
     if (GetConsoleMode(h, &mode) && enc_codepage >= 0
 	    && (int)GetConsoleCP() != enc_codepage)
     {
+	int	len = (int)STRLEN(str);
 	WCHAR	*w = enc_to_utf16((char_u *)str, &len);
 
-	WriteConsoleW(h, w, len, &nwrite, NULL);
+	if (w != NULL)
-	vim_free(w);
+	{
+	    WriteConsoleW(h, w, len, &nwrite, NULL);
+	    vim_free(w);
+	}
     }
     else
     {
@@ -3614,19 +3617,21 @@ mch_errmsg(char *str)
     static void
 mch_msg_c(char *str)
 {
-    int	    len = (int)STRLEN(str);
     DWORD   nwrite = 0;
     DWORD   mode;
     HANDLE  h = GetStdHandle(STD_OUTPUT_HANDLE);
 
-
     if (GetConsoleMode(h, &mode) && enc_codepage >= 0
 	    && (int)GetConsoleCP() != enc_codepage)
     {
+	int	len = (int)STRLEN(str);
 	WCHAR	*w = enc_to_utf16((char_u *)str, &len);
 
-	WriteConsoleW(h, w, len, &nwrite, NULL);
+	if (w != NULL)
-	vim_free(w);
+	{
+	    WriteConsoleW(h, w, len, &nwrite, NULL);
+	    vim_free(w);
+	}
     }
     else
     {

src/os_mswin.c

@@ -3180,10 +3180,12 @@ get_logfont(
 		    if (cp->name == NULL && verbose)
 		    {
 			char_u *s = utf16_to_enc(p, NULL);
-
+			if (s != NULL)
-			semsg(_(e_illegal_str_name_str_in_font_name_str),
+			{
-							   "charset", s, name);
+			    semsg(_(e_illegal_str_name_str_in_font_name_str),
-			vim_free(s);
+							       "charset", s, name);
+			    vim_free(s);
+			}
 			break;
 		    }
 		    break;
@@ -3202,9 +3204,12 @@ get_logfont(
 		    if (qp->name == NULL && verbose)
 		    {
 			char_u *s = utf16_to_enc(p, NULL);
-			semsg(_(e_illegal_str_name_str_in_font_name_str),
+			if (s != NULL)
-							   "quality", s, name);
+			{
-			vim_free(s);
+			    semsg(_(e_illegal_str_name_str_in_font_name_str),
+							       "quality", s, name);
+			    vim_free(s);
+			}
 			break;
 		    }
 		    break;

src/os_win32.c

@@ -3810,10 +3810,13 @@ mch_dirname(
     if (GetLongPathNameW(wbuf, wcbuf, _MAX_PATH) != 0)
     {
 	p = utf16_to_enc(wcbuf, NULL);
-	if (STRLEN(p) >= (size_t)len)
+	if (p != NULL)
 	{
-	    // long path name is too long, fall back to short one
+	    if (STRLEN(p) >= (size_t)len)
-	    VIM_CLEAR(p);
+	    {
+		// long path name is too long, fall back to short one
+		VIM_CLEAR(p);
+	    }
 	}
     }
     if (p == NULL)

src/terminal.c

@@ -7083,7 +7083,11 @@ conpty_term_and_job_init(
     if (cmd_wchar == NULL)
 	goto failed;
     if (opt->jo_cwd != NULL)
+    {
 	cwd_wchar = enc_to_utf16(opt->jo_cwd, NULL);
+	if (cwd_wchar == NULL)
+	    goto failed;
+    }
 
     win32_build_env(opt->jo_env, &ga_env, TRUE);
     env_wchar = ga_env.ga_data;
@@ -7425,7 +7429,11 @@ winpty_term_and_job_init(
     if (cmd_wchar == NULL)
 	goto failed;
     if (opt->jo_cwd != NULL)
+    {
 	cwd_wchar = enc_to_utf16(opt->jo_cwd, NULL);
+	if (cwd_wchar == NULL)
+	    goto failed;
+    }
 
     win32_build_env(opt->jo_env, &ga_env, TRUE);
     env_wchar = ga_env.ga_data;
@@ -7585,7 +7593,11 @@ failed:
 	char *msg = (char *)utf16_to_enc(
 				(short_u *)winpty_error_msg(winpty_err), NULL);
 
-	emsg(msg);
+	if (msg != NULL)
+	{
+	    emsg(msg);
+	    vim_free(msg);
+	}
 	winpty_error_free(winpty_err);
     }
     return FAIL;

src/version.c

@@ -704,6 +704,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    1339,
 /**/
     1338,
 /**/