Create binary voice string archive on target.

Extend genlang to create files holding the voice strings in a binary format
similar to lng. Create such files for all languages during a normal build, put
them in a zip file and add them to rockbox.zip so they are installed with a
main build.

Creating a voice file requires access to the voice strings, which are currently
not distributed with Rockbox. Therefore it is necessary to either have access
to the source code of the installed build or to use a build the genlang service
on rockbox.org can retrieve the strings for, making it impossible for normal
users to generate a voice file for custom builds, test builds and older builds
that aren't supported by the rockbox.org script. It also requires a network
connection when creating voice files with Rockbox Utility even if creating a
voice file by itself isn't network related.

Furthermore, putting the voice strings on the target is also a prerequisite for
eventually supporting a TTS on the player.

Strip voice strings path.

Change-Id: Ic2f86130909b3b257986a348e5cfcf618c4500c3

3 files changed, 57 insertions, 15 deletions

diff --git a/apps/lang/lang.make b/apps/lang/lang.make
index d0dac71e95..a1351cee67 100644
--- a/apps/lang/lang.make
+++ b/apps/lang/lang.make
@@ -9,6 +9,7 @@
 
 LANGS := $(call preprocess, $(APPSDIR)/lang/SOURCES)
 LANGOBJ := $(LANGS:$(ROOTDIR)/%.lang=$(BUILDDIR)/%.lng)
+VOICEOBJ := $(LANGS:$(ROOTDIR)/%.lang=$(BUILDDIR)/%.vstrings)
 LANG_O = $(BUILDDIR)/lang/lang_core.o
 
 CLEANOBJS += $(BUILDDIR)/lang/max_language_size.h $(BUILDDIR)/lang/lang*
@@ -23,7 +24,7 @@ CLEANOBJS += $(BUILDDIR)/lang/max_language_size.h $(BUILDDIR)/lang/lang*
 # SUBHEADER_SIZE.
 # TODO: In the future generate this file within genlang or another script
 # in order to only calculate the maximum size based on the core strings.
-$(BUILDDIR)/lang/max_language_size.h: $(LANGOBJ)
+$(BUILDDIR)/lang/max_language_size.h: $(LANGOBJ) $(BUILDDIR)/apps/lang/voicestrings.zip
         $(call PRINTS,GEN $(subst $(BUILDDIR)/,,$@))
         $(SILENT)echo "#define MAX_LANGUAGE_SIZE `ls -ln $(BUILDDIR)/apps/lang/* | awk '{print $$5-10}' | sort -n | tail -1`" > $@
 
@@ -34,7 +35,11 @@ $(BUILDDIR)/lang/lang_core.o: $(APPSDIR)/lang/$(LANGUAGE).lang $(BUILDDIR)/apps/
 
 $(BUILDDIR)/lang/lang.h: $(BUILDDIR)/lang/lang_core.o
 
-$(BUILDDIR)/%.lng : $(ROOTDIR)/%.lang $(BUILDDIR)/apps/genlang-features
+$(BUILDDIR)/%.lng $(BUILDDIR)/%.vstrings: $(ROOTDIR)/%.lang $(BUILDDIR)/apps/genlang-features
         $(call PRINTS,GENLANG $(subst $(ROOTDIR)/,,$<))
         $(SILENT)mkdir -p $(dir $@)
-        $(SILENT)$(TOOLSDIR)/genlang -e=$(APPSDIR)/lang/english.lang -t=$(MODELNAME):`cat $(BUILDDIR)/apps/genlang-features` -i=$(TARGET_ID) -b=$@ $<
+        $(SILENT)$(TOOLSDIR)/genlang -e=$(APPSDIR)/lang/english.lang -t=$(MODELNAME):`cat $(BUILDDIR)/apps/genlang-features` -i=$(TARGET_ID) -b=$*.lng -c=$*.vstrings $<
+
+$(BUILDDIR)/apps/lang/voicestrings.zip: $(VOICEOBJ)
+        $(call PRINTS,ZIP $(subst $(BUILDDIR)/,,$@))
+        $(SILENT)zip -9 -q $@ $(subst $(BUILDDIR)/,,$^)
diff --git a/tools/buildzip.pl b/tools/buildzip.pl
index e2341f4252..70445570f8 100755
--- a/tools/buildzip.pl
+++ b/tools/buildzip.pl
@@ -683,6 +683,7 @@ STOP
 
     # copy the already built lng files
     glob_copy('apps/lang/*lng', "$temp_dir/langs/");
+    glob_copy('apps/lang/*.zip', "$temp_dir/langs/");
 
     # copy the .lua files
     glob_mkdir("$temp_dir/rocks/viewers/lua/");
diff --git a/tools/genlang b/tools/genlang
index bfd9fac7b5..ac49e594d3 100755
--- a/tools/genlang
+++ b/tools/genlang
@@ -13,6 +13,7 @@
 # See apps/language.c (TODO: Use common include for both)
 # Cookie and binary version for the binary lang file
 my $LANGUAGE_COOKIE   = 0x1a;
+my $VOICE_COOKIE      = 0x9a;
 my $LANGUAGE_VERSION  = 0x06;
 my $LANGUAGE_FLAG_RTL = 0x01;
 
@@ -38,6 +39,10 @@ Usage: genlang [options] <langv2 file>
     Make the tool create a binary language (.lng) file named [outfile].
     The use of this option requires that you also use -e, -t and -i.
 
+ -c=<outfile>
+    Create binary voicestring file named [outfile]. Works like -b and can be
+    used the same time.
+
  -u
     Update language file. Given the translated file and the most recent english
     file, you\'ll get an updated version sent to stdout. Suitable action to do
@@ -89,18 +94,19 @@ my $prefix = $p;
 my $binary = $b;
 my $update = $u;
 my $sortfile = $s;
+my $binvoice = $c;
 
 my $english = $e;
 my $voiceout = $o;
 
-my $check = ($binary?1:0) + ($prefix?1:0) + ($update?1:0) + ($voiceout?1:0) + ($sortfile?1:0);
+my $check = ($binary?.5:0) + ($prefix?1:0) + ($update?1:0) + ($voiceout?1:0) + ($sortfile?1:0) + ($binvoice?.5:0);
 
 if($check > 1) {
-    print STDERR "Please use only one of -p, -u, -o, -b and -s\n";
+    print STDERR "Please use only one of -p, -u, -o, -b, -c and -s\n";
     exit;
 }
 if(!$check) {
-    print STDERR "Please use at least one of -p, -u, -o, -b and -s\n";
+    print STDERR "Please use at least one of -p, -u, -o, -c, -e and -s\n";
     exit;
 }
 
@@ -724,7 +730,7 @@ MOO
     close(HFILE_CORE);
     close(CFILE_CORE);
 } # end of the c/h file generation
-elsif($binary) {
+elsif($binary || $binvoice) {
     # Creation of a binary lang file was requested
 
     # We must first scan the english file to get the correct order of the id
@@ -732,10 +738,18 @@ elsif($binary) {
     # files. The english file is scanned before the translated file was
     # scanned.
 
-    open(OUTF, ">$binary") or die "Error: Can't create $binary";
-    binmode OUTF;
-    printf OUTF ("%c%c%c%c", $LANGUAGE_COOKIE, $LANGUAGE_VERSION, $target_id,
-      $langoptions); # magic lang file header
+    if($binary) {
+        open(OUTF, ">$binary") or die "Error: Can't create $binary";
+        binmode OUTF;
+        printf OUTF ("%c%c%c%c", $LANGUAGE_COOKIE, $LANGUAGE_VERSION, $target_id,
+            $langoptions); # magic lang file header
+    }
+    if($binvoice) {
+        open(OUTV, ">$binvoice") or die "Error: Can't create $binary";
+        binmode OUTV;
+        printf OUTV ("%c%c%c%c", $VOICE_COOKIE, $LANGUAGE_VERSION, $target_id,
+            $langoptions); # magic lang file header
+    }
 
     # output the number of strings for each user
     my $foffset = $HEADER_SIZE + $SUBHEADER_SIZE * keys(%users);
@@ -744,18 +758,28 @@ elsif($binary) {
         for $n (0 .. $idcount[$_]-1) {
             $size += length(trim($dest{$idnum[$_][$n]})) + 1;
         }
-        printf OUTF ("%c%c%c%c%c%c", ($idcount[$_] >> 8), ($idcount[$_] & 0xff),
-                     ($size >> 8), ($size & 0xff), ($foffset >> 8), ($foffset & 0xff));
+        if($binary) {
+            printf OUTF ("%c%c%c%c%c%c", ($idcount[$_] >> 8), ($idcount[$_] & 0xff),
+                ($size >> 8), ($size & 0xff), ($foffset >> 8), ($foffset & 0xff));
+        }
+        if($binvoice) {
+            printf OUTV ("%c%c%c%c%c%c", ($idcount[$_] >> 8), ($idcount[$_] & 0xff),
+                ($size >> 8), ($size & 0xff), ($foffset >> 8), ($foffset & 0xff));
+        }
         $foffset += $size;
     }
 
     for (keys %users) {
         # loop over the target phrases
-        for $n (0 .. $idcount[$_]-1) {
+        # This loops over the strings in the translated language file order
+        my @ids = ((0 .. ($idcount[$_]-1)));
+        push @ids, (0x8000 .. ($voiceid[$_]-1));
+        for $n (@ids) {
             my $name=$idnum[$_][$n]; # get the ID
             my $dest = $dest{$name}; # get the destination phrase
+            my $voice = $voice{$name}; # get the destination voice string
 
-            if($dest) {
+            if($dest && $n < 0x8000 && $binary) {
                 $dest =~ s/^\"(.*)\"\s*$/$1/g; # cut off quotes
 
                 # Now, make sure we get the number from the english sort order:
@@ -763,8 +787,20 @@ elsif($binary) {
 
                 printf OUTF ("%c%c%s\x00", ($idnum>>8), ($idnum&0xff), $dest);
             }
+            if($voice && $binvoice) {
+                $voice =~ s/^\"(.*)\"\s*$/$1/g; # cut off quotes
+                # Now, make sure we get the number from the english sort order:
+                $idnum = $idmap[$_]{$name};
+                printf OUTV ("%c%c%s\x00", ($idnum>>8), ($idnum&0xff), $voice);
+            }
         }
     }
+    if($binary) {
+        close(OUTF);
+    }
+    if($binvoice) {
+        close(OUTV);
+    }
 }
 elsif($voiceout) {
     # voice output requested, display id: and voice: strings in a v1-like