[LyX/2.3.x] Check encodability of math user macro names (#11855)

[Juergen Spitzmueller]

commit 8900b5a3aa00393c6fcc6429c16ef6e59e3d1617
Author: Juergen Spitzmueller <spitz at lyx.org>
Date:   Fri Dec 2 11:23:32 2022 +0100

    Check encodability of math user macro names (#11855)
    
    (cherry picked from commit 14b108fc222ef9e7891d90ce364ceb227f8182dc)
---
 src/mathed/InsetMathMacro.cpp |   51 ++++++++++++++++++++++++++++++++++++++--
 status.23x                    |    2 +
 2 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/src/mathed/InsetMathMacro.cpp b/src/mathed/InsetMathMacro.cpp
index 1a5e220..5bed4a0 100644
--- a/src/mathed/InsetMathMacro.cpp
+++ b/src/mathed/InsetMathMacro.cpp
@@ -26,6 +26,7 @@
 #include "BufferView.h"
 #include "CoordCache.h"
 #include "Cursor.h"
+#include "Encoding.h"
 #include "FuncStatus.h"
 #include "FuncRequest.h"
 #include "LaTeXFeatures.h"
@@ -33,6 +34,7 @@
 #include "LyXRC.h"
 #include "MetricsInfo.h"
 
+#include "frontends/alert.h"
 #include "frontends/Painter.h"
 
 #include "support/debug.h"
@@ -1074,10 +1076,53 @@ void InsetMathMacro::write(WriteStream & os) const
 	mode_type mode = currentMode();
 	MathEnsurer ensurer(os, mode == MATH_MODE, true, mode == TEXT_MODE);
 
+	// Check if the macro name is encodable. Otherwise we might crash (#11855).
+	docstring const name_in = name();
+	docstring name_recoded;
+	docstring uncodable;
+	for (char_type c : name_in) {
+		if (!os.encoding())
+			break;
+		if (os.encoding()->encodable(c))
+			name_recoded += c;
+		else {
+			switch (os.output()) {
+			case WriteStream::wsDryrun: {
+				os << "<" << _("LyX Warning: ")
+				   << _("uncodable character") << " '";
+				os << docstring(1, c);
+				os << "'>";
+				break;
+			}
+			case WriteStream::wsPreview: {
+				// indicate the encoding error by a boxed '?'
+				os << "{\\fboxsep=1pt\\fbox{?}}";
+				LYXERR0("Uncodable character" << " '"
+					<< docstring(1, c)
+					<< "'");
+				break;
+			}
+			case WriteStream::wsDefault:
+			default:
+				// record for error message
+				uncodable += c;
+				break;
+			}
+		}
+	}
+	
+	if (!uncodable.empty()) {
+		frontend::Alert::warning(
+			_("Uncodable characters in math macro"),
+			support::bformat(_("The macro name '%1$s' contains a character.\n"
+					   "that is not encodable in the current encoding (%2$s).\n"
+					   "Please fix this macro."), name_in, uncodable));
+	}
+
 	// non-normal mode
 	if (d->displayMode_ != DISPLAY_NORMAL) {
-		os << "\\" << name();
-		if (name().size() != 1 || isAlphaASCII(name()[0]))
+		os << "\\" << name_recoded;
+		if (name_recoded.size() != 1 || isAlphaASCII(name_recoded[0]))
 			os.pendingSpace(true);
 		return;
 	}
@@ -1090,7 +1135,7 @@ void InsetMathMacro::write(WriteStream & os) const
 	if (os.fragile())
 		os << "\\protect";
 
-	os << "\\" << name();
+	os << "\\" << name_recoded;
 	bool first = true;
 
 	// Optional arguments:
diff --git a/status.23x b/status.23x
index 076bb4e..2b96fd2 100644
--- a/status.23x
+++ b/status.23x
@@ -83,6 +83,8 @@ What's new
 
 - Fix validating a macro definitions (bug 12524).
 
+- Fix crashing due to uncodable character in math macro name (bug 11855).
+
 
 
 * USER INTERFACE