From eb2d596d72247d90a89fc3b1c702b5b54aaecf6b Mon Sep 17 00:00:00 2001
From: Solomon Peachy <pizza@shaftnet.org>
Date: Sun, 13 Oct 2024 09:01:20 -0400
Subject: updatelang:  Normalize all strings in our lang files to NFC form.

Now no matter how [de]normalized the input strings are, we will
normalize them to the best of our ability in what we use.

This adds a dependencey for Perl's Unicode::Normalize.

Change-Id: I13e275692ea33a463b19f3a499ea06ce1acbb44a
---
 tools/updatelang | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

(limited to 'tools/updatelang')

diff --git a/tools/updatelang b/tools/updatelang
index ff4ce340e0..94f6fd1e8c 100755
--- a/tools/updatelang
+++ b/tools/updatelang
@@ -6,11 +6,15 @@
 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
 #                     \/            \/     \/    \/            \/
 #
-# Copyright (C) 2020 Solomon Peachy
+# Copyright (C) 2020-2024 Solomon Peachy
 #
 
 use utf8;
 use File::Basename;
+use Unicode::Normalize;
+
+use open qw( :std :encoding(UTF-8) );
+binmode(STDOUT, ":encoding(UTF-8)");
 
 sub trim {
     my ($string) = @_;
@@ -72,6 +76,7 @@ sub parselangfile {
 #		    $l = "*";
 		}
 
+		$w = NFD($w); # Unicode decompose
 		$thisphrase{$pos}->{$l} = $w;
 	    }
 	}
@@ -538,10 +543,11 @@ foreach my $id (@finalorder) {
     %lp = combinetgts(%{$lang{$id}{'source'}});
     print $fh "  <source>\n";
     foreach my $tgt (sort(keys(%lp))) {
-	if ($lp{$tgt} eq 'none') {
-	    print $fh "    $tgt: $lp{$tgt}\n";
+	my $w = NFC($lp{$tgt});
+	if ($w eq 'none') {
+	    print $fh "    $tgt: $w\n";
 	} else {
-	    print $fh "    $tgt: \"$lp{$tgt}\"\n";
+	    print $fh "    $tgt: \"$w\"\n";
 	}
     }
     print $fh "  </source>\n";
@@ -550,10 +556,11 @@ foreach my $id (@finalorder) {
     %lp = combinetgts(%{$lang{$id}{'dest'}});
     print $fh "  <dest>\n";
     foreach my $tgt (sort(keys(%lp))) {
-	if ($lp{$tgt} eq 'none') {
-	    print $fh "    $tgt: $lp{$tgt}\n";
+	my $w = NFC($lp{$tgt});
+	if ($w eq 'none') {
+	    print $fh "    $tgt: $w\n";
 	} else {
-	    print $fh "    $tgt: \"$lp{$tgt}\"\n";
+	    print $fh "    $tgt: \"$w\"\n";
 	}
     }
     print $fh "  </dest>\n";
@@ -562,10 +569,11 @@ foreach my $id (@finalorder) {
     %lp = combinetgts(%{$lang{$id}{'voice'}});
     print $fh "  <voice>\n";
     foreach my $tgt (sort(keys(%lp))) {
-	if ($lp{$tgt} eq 'none') {
-	    print $fh "    $tgt: $lp{$tgt}\n";
+	my $w = NFC($lp{$tgt});
+	if ($w eq 'none') {
+	    print $fh "    $tgt: $w\n";
 	} else {
-	    print $fh "    $tgt: \"$lp{$tgt}\"\n";
+	    print $fh "    $tgt: \"$w\"\n";
 	}
     }
     print $fh "  </voice>\n";
-- 
cgit v1.2.3