Unicode support.

git-svn-id: svn://svn.code.sf.net/p/sc2/code/trunk@1551 8092fc87-c524-0410-9efc-e669fe64eaf9

sc2/src/sc2code/comm.c

@@ -49,6 +49,8 @@ static void xform_PLUT_step (SIZE TDelta);
 static void FlushPLUTXForms (void);
 static int ambient_anim_task (void *data);
 
+static BOOLEAN getLineWithinWidth(TEXT *pText,
+		 const unsigned char **startNext, SIZE maxWidth, COUNT maxChars);
 
 #define MAX_RESPONSES 8
 #define BACKGROUND_VOL speechVolumeScale == 0.0f ? MAX_VOLUME : (MAX_VOLUME >> 1)
@@ -125,61 +127,30 @@ static Mutex subtitle_mutex;
 static const UNICODE * volatile last_subtitle;
 static TFB_Image *subtitle_cache;
 
-/* _count_lines - mostly stolen from add_text, just sees how many lines
-                  a given input string would take to display given the
-                  line wrapping information */
-static int _count_lines (PTEXT pText) 
+/* _count_lines - sees how many lines a given input string would take to
+ * display given the line wrapping information
+ */
+static int _count_lines (PTEXT pText)
 {
-	COUNT maxchars = (COUNT)~0;
-	UNICODE ch;
-	const UNICODE *pStr;
 	SIZE text_width;
-	int num = 0;
+	COUNT maxchars = (COUNT)~0;
+	const unsigned char *pStr;
+	int numLines = 0;
+	BOOLEAN eol;
 
 	text_width = CommData.AlienTextWidth;
 	SetContextFont (CommData.AlienFont);
 
 	pStr = pText->pStr;
-
 	do
 	{
-		++num;
+		++numLines;
 		pText->pStr = pStr;
-		pText->CharCount = 1;
-
-		{
-			BOOLEAN eot;
-			RECT r, old_r;
-			COUNT OldCount;
-
-			GetContextClipRect (&r);
-			eot = FALSE;
-			do
-			{
-				old_r = r;
-				OldCount = pText->CharCount;
-				while (!(eot = (BOOLEAN)(
-						(ch = *++pStr) == '\0'
-						|| ch == '\n'
-						|| ch == '\r'
-						|| (COUNT)(pStr - pText->pStr) >= maxchars
-						)) && ch != ' ')
-					;
-				pText->CharCount = pStr - pText->pStr;
-				TextRect (pText, &r, NULL_PTR);
-			} while (!eot && r.extent.width < text_width);
-
-			if (r.extent.width >= text_width)
-			{
-				pText->CharCount = OldCount;
-				r = old_r;
-			}
-		}
-		pStr = pText->pStr + pText->CharCount;
-	} while ((ch = *pStr++) != '\0' && ch != '\n' && ch != '\r' && maxchars);
+		eol = getLineWithinWidth(pText, &pStr, text_width, maxchars);
+	} while (!eol && maxchars);
 	pText->pStr = pStr;
 
-	return (num);
+	return numLines;
 }
 
 static COORD
@@ -189,11 +160,11 @@ add_text (int status, PTEXT pTextIn)
 	TEXT locText;
 	PTEXT pText;
 	SIZE leading;
-	wchar_t ch;
 	const unsigned char *pStr;
 	SIZE text_width;
 	int num_lines = 0;
 	static COORD last_baseline;
+	BOOLEAN eol;
 	
 	BatchGraphics ();
 
@@ -255,7 +226,7 @@ add_text (int status, PTEXT pTextIn)
 		maxchars = pTextIn->CharCount;
 		locText = *pTextIn;
 		locText.baseline.x -= 8;
-		locText.CharCount = 1;
+		locText.CharCount = (COUNT)~0;
 		locText.pStr = "*";
 		font_DrawText (&locText);
 
@@ -274,7 +245,8 @@ add_text (int status, PTEXT pTextIn)
 			pText->baseline.y -= (leading * num_lines);
 		else if (CommData.AlienTextTemplate.valign == VALIGN_MIDDLE)
 			pText->baseline.y -= ((leading * num_lines) / 2);
-		if (pText->baseline.y < 0) pText->baseline.y = 0;
+		if (pText->baseline.y < 0)
+			pText->baseline.y = 0;
 	}
 
 	do
@@ -282,93 +254,49 @@ add_text (int status, PTEXT pTextIn)
 		pText->pStr = pStr;
 		pText->baseline.y += leading;
 
+		eol = getLineWithinWidth(pText, &pStr, text_width, maxchars);
+
+		maxchars -= pText->CharCount;
+		if (maxchars != 0)
+			--maxchars;
+		numchars += pText->CharCount;
+		
+		if (status <= 0)
 		{
-			BOOLEAN eot;
-			RECT r, old_r;
-			COUNT oldCount;
-			const unsigned char *oldPStr;
-			wchar_t oldCh;
-			COUNT charCount;
+			if (pText->baseline.y < SIS_SCREEN_HEIGHT)
+				font_DrawText (pText);
 
-			GetContextClipRect (&r);
-			oldCount = 1;
-			charCount = 0;
-			ch = '\0';
-			for (;;)
+			if (status < -4 && pText->baseline.y >= -status - 10)
 			{
-				old_r = r;
-				oldPStr = pStr;
-				oldCh = ch;
-			
-				for (;;)
-				{
-					ch = getCharFromString (&pStr);
-					eot = ch == '\0' || ch == '\n' || ch == '\r'
-							|| charCount >= maxchars;
-					if (eot || ch == ' ')
-						break;
-					charCount++;
-				}
-
-				oldCount = pText->CharCount;
-				pText->CharCount = charCount;
-				TextRect (pText, &r, NULL_PTR);
-				if (eot || r.extent.width >= text_width)
-					break;
-				charCount++;
-						// For the space in between words.
-			}
-
-			if (r.extent.width >= text_width)
-			{
-				pText->CharCount = oldCount;
-				pStr = oldPStr;
-				r = old_r;
-				ch = oldCh;
-			}
-
-			maxchars -= pText->CharCount;
-			if (maxchars != 0)
-				--maxchars;
-			numchars += pText->CharCount;
-			
-			if (status <= 0)
-			{
-				if (pText->baseline.y < SIS_SCREEN_HEIGHT)
-					font_DrawText (pText);
-
-				if (status < -4 && pText->baseline.y >= -status - 10)
-				{
-					++pStr;
-					break;
-				}
-			}
-			else
-			{
-				SetContextForeGroundColor (CommData.AlienTextBColor);
-
-				--pText->baseline.x;
-				font_DrawText (pText);
-
-				++pText->baseline.x;
-				--pText->baseline.y;
-				font_DrawText (pText);
-
-				++pText->baseline.x;
-				++pText->baseline.y;
-				font_DrawText (pText);
-
-				--pText->baseline.x;
-				++pText->baseline.y;
-				font_DrawText (pText);
-
-				SetContextForeGroundColor (CommData.AlienTextFColor);
-
-				--pText->baseline.y;
-				font_DrawText (pText);
+				++pStr;
+				break;
 			}
 		}
-	} while (ch != '\0' && ch != '\n' && ch != '\r' && maxchars);
+		else
+		{
+			SetContextForeGroundColor (CommData.AlienTextBColor);
+
+			--pText->baseline.x;
+			font_DrawText (pText);
+
+			++pText->baseline.x;
+			--pText->baseline.y;
+			font_DrawText (pText);
+
+			++pText->baseline.x;
+			++pText->baseline.y;
+			font_DrawText (pText);
+
+			--pText->baseline.x;
+			++pText->baseline.y;
+			font_DrawText (pText);
+
+			SetContextForeGroundColor (CommData.AlienTextFColor);
+
+			--pText->baseline.y;
+			font_DrawText (pText);
+		}
+	} while (!eol && maxchars);
 	pText->pStr = pStr;
 
 	if (status == 1)
@@ -391,6 +319,70 @@ add_text (int status, PTEXT pTextIn)
 	return (pText->baseline.y);
 }
 
+static BOOLEAN
+getLineWithinWidth(TEXT *pText, const unsigned char **startNext,
+		SIZE maxWidth, COUNT maxChars) {
+	BOOLEAN eol;
+			// The end of the line of text has been reached.
+	BOOLEAN done;
+			// We cannot add any more words.
+	RECT rect;
+	COUNT oldCount;
+	const unsigned char *ptr;
+	const unsigned char *wordStart;
+	wchar_t ch;
+	COUNT charCount;
+
+	//GetContextClipRect (&rect);
+
+	eol = FALSE;	
+	done = FALSE;
+	oldCount = 1;
+	charCount = 0;
+	ch = '\0';
+	ptr = pText->pStr;
+	for (;;)
+	{
+		wordStart = ptr;
+
+		// Scan one word.
+		for (;;)
+		{
+			if (*ptr == '\0')
+			{
+				eol = TRUE;
+				done = TRUE;
+				break;
+			}
+			ch = getCharFromString (&ptr);
+			eol = ch == '\0' || ch == '\n' || ch == '\r';
+			done = eol || charCount >= maxChars;
+			if (done || ch == ' ')
+				break;
+			charCount++;
+		}
+
+		oldCount = pText->CharCount;
+		pText->CharCount = charCount;
+		TextRect (pText, &rect, NULL_PTR);
+		
+		if (rect.extent.width >= maxWidth)
+		{
+			pText->CharCount = oldCount;
+			*startNext = wordStart;
+			return FALSE;
+		}
+
+		if (done)
+		{
+			*startNext = ptr;
+			return eol;
+		}
+		charCount++;
+				// For the space in between words.
+	}
+}
+
 static void
 DrawSISComWindow (void)
 {
@@ -1641,11 +1633,14 @@ DoCommunication (PENCOUNTER_STATE pES)
 
 		if (PulsedInputState.key[KEY_MENU_SELECT])
 		{
-			pES->phrase_buf_index =
-					pES->response_list[pES->cur_response].response_text.CharCount;
-			wstrncpy (pES->phrase_buf,
-					pES->response_list[pES->cur_response].response_text.pStr,
-					pES->phrase_buf_index);
+			const unsigned char *end;
+			PTEXT response_text =
+					&pES->response_list[pES->cur_response].response_text;
+			end = skipUTF8Chars(response_text->pStr,
+					 response_text->CharCount);
+			pES->phrase_buf_index = end - response_text->pStr;
+			memcpy(pES->phrase_buf, response_text->pStr,
+					 pES->phrase_buf_index);
 			pES->phrase_buf[pES->phrase_buf_index++] = '\0';
 
 			LockMutex (GraphicsLock);
@@ -1943,7 +1938,7 @@ DoResponsePhrase (RESPONSE_REF R, RESPONSE_FUNC response_func,
 	pEntry->response_ref = R;
 	pEntry->response_text.pStr = ConstructStr;
 	if (pEntry->response_text.pStr)
-		pEntry->response_text.CharCount = wstrlen (ConstructStr);
+		pEntry->response_text.CharCount = (COUNT)~0;
 	else
 	{
 		STRING locString;
@@ -2395,7 +2390,9 @@ do_subtitles (UNICODE *pStr)
 			CommData.AlienTextTemplate.pStr = pStr;
 			t = CommData.AlienTextTemplate;
 
-			CommData.AlienTextTemplate.CharCount = t.pStr - CommData.AlienTextTemplate.pStr;
+			CommData.AlienTextTemplate.CharCount =
+					utf8StringCountN(CommData.AlienTextTemplate.pStr,
+					t.pStr);
 			subtitle_state = WAIT_SUBTITLE;
 			break;
 		}

sc2/src/sc2code/confirm.c

@@ -58,7 +58,7 @@ DrawConfirmationWindow (BOOLEAN answer)
 	t.pStr = "Really Quit?";
 	t.align = ALIGN_CENTER;
 	t.valign = VALIGN_BOTTOM;
-	t.CharCount = ~0;
+	t.CharCount = (COUNT)~0;
 	font_DrawText (&t);
 	t.baseline.y += 10;
 	t.baseline.x = r.corner.x + (r.extent.width >> 2);

sc2/src/sc2code/libs/graphics/font.c

@@ -324,7 +324,7 @@ _text_blt (PRECT pClipRect, PRIMITIVEPTR PrimPtr)
 
 static inline FRAME_DESC *
 getCharFrame (FONT_DESC *fontPtr, wchar_t ch) {
-	wchar_t pageStart = ch & 0xff00;
+	wchar_t pageStart = ch & CHARACTER_PAGE_MASK;
 
 	FONT_PAGE *page = fontPtr->fontPages;
 	for (;;)

sc2/src/sc2code/libs/graphics/font.h

@@ -25,6 +25,7 @@ typedef struct FontPage
 {
 	struct FontPage *next;
 	wchar_t pageStart;
+#define CHARACTER_PAGE_MASK 0xff800
 	wchar_t firstChar;
 	size_t numChars;
 	FRAME_DESC *charDesc;

sc2/src/sc2code/libs/graphics/sdl/3do_getbody.c

@@ -687,11 +687,11 @@ _GetFontData (uio_Stream *fp, DWORD length)
 		{
 			// Process one character page.
 			size_t endBCD;
-			pageStart = bcds[startBCD].index & 0xff00;
+			pageStart = bcds[startBCD].index & CHARACTER_PAGE_MASK;
 
 			endBCD = startBCD;
 			while (endBCD < numBCDs &&
-					(bcds[endBCD].index & 0xff00) == pageStart)
+					(bcds[endBCD].index & CHARACTER_PAGE_MASK) == pageStart)
 				endBCD++;
 
 			{

sc2/src/sc2code/libs/strings/strings.c

@@ -144,10 +144,23 @@ GetStringLength (STRING String)
 
 		StringIndex = STRING_INDEX (String);
 		LockStringTable (StringTable, &StringTablePtr);
+
+		{
+			STRINGPTR start;
+			STRINGPTR end;
+
+			start = (STRINGPTR) ((BYTE *) StringTablePtr +
+					StringTablePtr->StringOffsets[StringIndex]);
+			end = (STRINGPTR) ((BYTE *) StringTablePtr +
+					StringTablePtr->StringOffsets[StringIndex + 1]);
+			StringLength = utf8StringCountN(start, end);
+		}
+
+#if 0
 		StringLength = (COUNT)(
 				StringTablePtr->StringOffsets[StringIndex + 1]
-				- StringTablePtr->StringOffsets[StringIndex]
-				);
+				- StringTablePtr->StringOffsets[StringIndex]);
+#endif
 		UnlockStringTable (StringTable);
 	}
 

sc2/src/sc2code/libs/strings/unicode.c

@@ -17,10 +17,18 @@
 #include <stddef.h>
 #include <stdio.h>
 
+// Resynchronise (skip everything starting with 0x10xxxxxx):
+static inline void
+resyncUTF8(const unsigned char **ptr) {
+	while ((**ptr & 0xc0) == 0x80)
+		*ptr++;
+}
+
 // Get one character from a UTF-8 encoded string.
 // *ptr will point to the start of the next character.
 // Returns 0 if the encoding is bad. This can be distinguished from the
-// '\0' character by checking whether **ptr == '\0'.
+// '\0' character by checking whether **ptr == '\0' before calling this
+// function.
 wchar_t
 getCharFromString(const unsigned char **ptr) {
 	wchar_t result;
@@ -106,11 +114,132 @@ err:
 	fprintf(stderr, "Warning: Invalid UTF8 sequence.\n");
 	
 	// Resynchronise (skip everything starting with 0x10xxxxxx):
-	while ((**ptr & 0xc0) == 0x80)
-		*ptr++;
+	resyncUTF8(ptr);
 	
 	return 0;
 }
 
+wchar_t
+getCharFromStringN(const unsigned char **ptr, const unsigned char *end) {
+	int numBytes;
+
+	if (**ptr < 0x80) {
+		numBytes = 1;
+	} else if ((**ptr & 0xe0) == 0xc0) {
+		numBytes = 2;
+	} else if ((**ptr & 0xf0) == 0xe0) {
+		numBytes = 3;
+	} else if ((**ptr & 0xf8) == 0xf0) {
+		numBytes = 4;
+	} else
+		goto err;
+
+	if (*ptr + numBytes > end)
+		goto err;
+
+	return getCharFromString(ptr);
+
+err:
+	*ptr = end;
+	return 0;
+}
+
+// Get one line from a string.
+// A line is terminated with either CRLF (DOS/Windows),
+// LF (Unix, MacOS X), or CR (old MacOS).
+// The end of the string is reached when **startNext == '\0'.
+// NULL is returned if the string is not valid UTF8. In this case
+// *end points to the first invalid character (or the character before if
+// it was a LF), and *startNext to the start of the next (possibly invalid
+// too) character.
+unsigned char *
+getLineFromString(const unsigned char *start, const unsigned char **end,
+		const unsigned char **startNext) {
+	const unsigned char *ptr = start;
+	const unsigned char *lastPtr;
+	wchar_t ch;
+
+	// Search for the first newline.
+	for (;;) {
+		if (*ptr == '\0') {
+			*end = ptr;
+			*startNext = ptr;
+			return (unsigned char *) start;
+		}
+		lastPtr = ptr;
+		ch = getCharFromString(&ptr);
+		if (ch == '\0') {
+			// Bad string
+			*end = lastPtr;
+			*startNext = ptr;
+			return NULL;
+		}
+		if (ch == '\n') {
+			*end = lastPtr;
+			if (*ptr == '\0'){
+				// LF at the end of the string.
+				*startNext = ptr;
+				return (unsigned char *) start;
+			}
+			ch = getCharFromString(&ptr);
+			if (ch == '\0') {
+				// Bad string
+				return NULL;
+			}
+			if (ch == '\r') {
+				// LFCR
+				*startNext = ptr;
+			} else {
+				// LF
+				*startNext = *end;
+			}
+			return (unsigned char *) start;
+		} else if (ch == '\r') {
+			*end = lastPtr;
+			*startNext = ptr;
+			return (unsigned char *) start;
+		} // else: a normal character
+	}
+}
+
+size_t
+utf8StringCount(const unsigned char *start) {
+	size_t count = 0;
+	wchar_t ch;
+
+	for (;;) {
+		ch = getCharFromString(&start);
+		if (ch == '\0')
+			return count;
+		count++;
+	}
+}
+
+size_t
+utf8StringCountN(const unsigned char *start, const unsigned char *end) {
+	size_t count = 0;
+	wchar_t ch;
+
+	for (;;) {
+		ch = getCharFromStringN(&start, end);
+		if (ch == '\0')
+			return count;
+		count++;
+	}
+}
+
+unsigned char *
+skipUTF8Chars(const unsigned char *ptr, size_t num) {
+	wchar_t ch;
+	const unsigned char *oldPtr;
+
+	while (num--) {
+		oldPtr = ptr;
+		ch = getCharFromString(&ptr);
+		if (ch == '\0')
+			return (unsigned char *) oldPtr;
+	}
+	return (unsigned char *) ptr;
+}
 
 

sc2/src/sc2code/libs/strlib.h

@@ -51,6 +51,14 @@ extern BOOLEAN GetStringContents (STRING String, STRINGPTR StringBuf,
 		BOOLEAN AppendSpace);
 
 wchar_t getCharFromString(const unsigned char **ptr);
+wchar_t getCharFromStringN(const unsigned char **ptr,
+		const unsigned char *end);
+unsigned char *getLineFromString(const unsigned char *start,
+		const unsigned char **end, const unsigned char **startNext);
+size_t utf8StringCount(const unsigned char *start);
+size_t utf8StringCountN(const unsigned char *start,
+		const unsigned char *end);
+unsigned char *skipUTF8Chars(const unsigned char *ptr, size_t num);
 
 #endif /* _STRLIB_H */