nwztools/upg: move upg handling to its own file, completely rework kas handling

This was a huge mess, the new is much cleaner hopefully.

Change-Id: I43663d021dc8bc31662d3923e1c3da22d987ebf9

6 files changed, 361 insertions, 386 deletions

diff --git a/utils/nwztools/upgtools/Makefile b/utils/nwztools/upgtools/Makefile
index 35d5bfabff..1030b1b849 100644
--- a/utils/nwztools/upgtools/Makefile
+++ b/utils/nwztools/upgtools/Makefile
@@ -16,7 +16,7 @@ all: $(BINS)
 %.o: %.cpp
         $(CXX) $(CXXFLAGS) -c -o $@ $<
 
-upgtool: upgtool.o misc.o fwp.o mg.o keysig_search.o
+upgtool: upgtool.o upg.o misc.o fwp.o mg.o keysig_search.o
         $(LD) -o $@ $^ $(LDFLAGS)
 
 clean:
diff --git a/utils/nwztools/upgtools/fwp.c b/utils/nwztools/upgtools/fwp.c
index c300f42f34..34c55f6e5a 100644
--- a/utils/nwztools/upgtools/fwp.c
+++ b/utils/nwztools/upgtools/fwp.c
@@ -19,6 +19,7 @@
  *
  ****************************************************************************/
 #include <stdio.h>
+#include <stdlib.h>
 #include "fwp.h"
 #include "misc.h"
 #include "mg.h"
@@ -53,9 +54,6 @@ int fwp_crypt(void *buf, int size, int mode)
         size -= NWZ_KEY_SIZE;
     }
     if(size != 0)
-    {
-        cprintf(GREY, "Cannot fwp_crypt non-multiple of 8!\n");
-        return -1;
-    }
+        abort();
     return 0;
-}
\ No newline at end of file
+}
diff --git a/utils/nwztools/upgtools/misc.c b/utils/nwztools/upgtools/misc.c
index 108235e7fd..ed1ba6c788 100644
--- a/utils/nwztools/upgtools/misc.c
+++ b/utils/nwztools/upgtools/misc.c
@@ -22,25 +22,19 @@
 #include <stdio.h>
 #include <time.h>
 #include <ctype.h>
+#include <stdarg.h>
 #include "misc.h"
 
-char OFF[] = { 0x1b, 0x5b, 0x31, 0x3b, '0', '0', 0x6d, '\0' };
+const char OFF[] = { 0x1b, 0x5b, 0x31, 0x3b, '0', '0', 0x6d, '\0' };
 
-char GREY[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '0', 0x6d, '\0' };
-char RED[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '1', 0x6d, '\0' };
-char GREEN[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '2', 0x6d, '\0' };
-char YELLOW[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '3', 0x6d, '\0' };
-char BLUE[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '4', 0x6d, '\0' };
+const char GREY[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '0', 0x6d, '\0' };
+const char RED[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '1', 0x6d, '\0' };
+const char GREEN[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '2', 0x6d, '\0' };
+const char YELLOW[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '3', 0x6d, '\0' };
+const char BLUE[] = { 0x1b, 0x5b, 0x31, 0x3b, '3', '4', 0x6d, '\0' };
 
 static bool g_color_enable = true;
 
-void *xmalloc(size_t s)
-{
-    void * r = malloc(s);
-    if(!r) bugp("malloc");
-    return r;
-}
-
 void enable_color(bool enable)
 {
     g_color_enable = enable;
@@ -51,3 +45,14 @@ void color(color_t c)
     if(g_color_enable)
         printf("%s", (char *)c);
 }
+
+void generic_std_printf(void *u, bool err, color_t c, const char *f, ...)
+{
+    (void)u;
+    (void)err;
+    va_list args;
+    va_start(args, f);
+    color(c);
+    vprintf(f, args);
+    va_end(args);
+}
diff --git a/utils/nwztools/upgtools/misc.h b/utils/nwztools/upgtools/misc.h
index 96666a2eff..e493cc2019 100644
--- a/utils/nwztools/upgtools/misc.h
+++ b/utils/nwztools/upgtools/misc.h
@@ -27,20 +27,16 @@
 #define _STR(a) #a
 #define STR(a) _STR(a)
 
-#define bug(...) do { fprintf(stderr,"["__FILE__":"STR(__LINE__)"]ERROR: "__VA_ARGS__); exit(1); } while(0)
-#define bugp(...) do { fprintf(stderr, __VA_ARGS__); perror(" "); exit(1); } while(0)
-
 #define ROUND_UP(val, round) ((((val) + (round) - 1) / (round)) * (round))
 
-typedef char color_t[];
+typedef const char color_t[];
 
 extern color_t OFF, GREY, RED, GREEN, YELLOW, BLUE;
-void *xmalloc(size_t s);
 void color(color_t c);
 void enable_color(bool enable);
 
-#define cprintf(col, ...) do {color(col); printf(__VA_ARGS__); }while(0)
+typedef void (*generic_printf_t)(void *u, bool err, color_t c, const char *f, ...);
 
-#define cprintf_field(str1, ...) do{ cprintf(GREEN, str1); cprintf(YELLOW, __VA_ARGS__); }while(0)
+void generic_std_printf(void *u, bool err, color_t c, const char *f, ...);
 
 #endif /* __MISC_H__ */
diff --git a/utils/nwztools/upgtools/upg.c b/utils/nwztools/upgtools/upg.c
new file mode 100644
index 0000000000..44d3eca789
--- /dev/null
+++ b/utils/nwztools/upgtools/upg.c
@@ -0,0 +1,260 @@
+/***************************************************************************
+ *             __________               __   ___.
+ *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
+ *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
+ *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
+ *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
+ *                     \/            \/     \/    \/            \/
+ * $Id$
+ *
+ * Copyright (C) 2016 Amaury Pouly
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ****************************************************************************/
+#include "upg.h"
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <openssl/md5.h>
+
+struct nwz_model_t g_model_list[] =
+{
+    { "nwz-e450", true, "8a01b624bfbfde4a1662a1772220e3c5" },
+    { "nwz-e460", true, "89d813f8f966efdebd9c9e0ea98156d2" },
+    { "nwz-a860", true, "a7c4af6c28b8900a783f307c1ba538c5" },
+    { "nwz-a850", true, "a2efb9168616c2e84d78291295c1aa5d" },
+    { "nwz-e470", true, "e4144baaa2707913f17b5634034262c4" },
+    { "nwz-e580", true, "6e25f79812eca7ceed04819d833e80af" },
+    /* The following keys were obtained by brute forcing firmware upgrades,
+     * someone with a device needs to confirm that they work */
+    { "nw-a820", false, "0c9869c268e0eaa6d1ba62daab09cebc" },
+    { "nwz-a10", false, "a4605e0628c9c3baeb5142ce9cb834d6" },
+    { "nwz-a20", false, "e9d7185e5ac183bf26e9a5b66f983c0b" },
+    { "nwz-zx100", false, "2c0bf029804f73e073154388743f84d2" },
+    { 0 }
+};
+
+static int digit_value(char c)
+{
+    if(c >= '0' && c <= '9') return c - '0';
+    if(c >= 'a' && c <= 'f') return c - 'a' + 10;
+    if(c >= 'A' && c <= 'F') return c - 'A' + 10;
+    return -1;
+}
+
+static char hex_digit(unsigned v)
+{
+    return (v < 10) ? v + '0' : (v < 16) ? v - 10 + 'a' : 'x';
+}
+
+int decrypt_keysig(const char kas[NWZ_KAS_SIZE], char key[NWZ_KEY_SIZE],
+    char sig[NWZ_SIG_SIZE])
+{
+    uint8_t src[NWZ_KAS_SIZE / 2];
+    for(int index = 0; index < NWZ_KAS_SIZE / 2; index++)
+    {
+        int a = digit_value(kas[index * 2]);
+        int b = digit_value(kas[index * 2 + 1]);
+        if(a < 0 || b < 0)
+            return -1;
+        src[index] = a << 4 | b;
+    }
+    fwp_setkey("ed295076");
+    fwp_crypt(src, sizeof(src), 1);
+    memcpy(key, src, NWZ_KEY_SIZE);
+    memcpy(sig, src + NWZ_KEY_SIZE, NWZ_SIG_SIZE);
+    return 0;
+}
+
+void encrypt_keysig(char kas[NWZ_KEY_SIZE],
+    const char key[NWZ_SIG_SIZE], const char sig[NWZ_KAS_SIZE])
+{
+    uint8_t src[NWZ_KAS_SIZE / 2];
+    fwp_setkey("ed295076");
+    memcpy(src, key, NWZ_KEY_SIZE);
+    memcpy(src + NWZ_KEY_SIZE, sig, NWZ_SIG_SIZE);
+    fwp_crypt(src, sizeof(src), 0);
+    for(int i = 0; i < NWZ_KAS_SIZE / 2; i++)
+    {
+        kas[2 * i] = hex_digit((src[i] >> 4) & 0xf);
+        kas[2 * i + 1] = hex_digit(src[i] & 0xf);
+    }
+}
+
+struct upg_file_t *upg_read_memory(void *buf, size_t size, char key[NWZ_KEY_SIZE],
+    char *sig, void *u, generic_printf_t printf)
+{
+#define cprintf(col, ...) printf(u, false, col, __VA_ARGS__)
+#define cprintf_field(str1, ...) do{ cprintf(GREEN, str1); cprintf(YELLOW, __VA_ARGS__); }while(0)
+#define err_printf(col, ...) printf(u, true, col, __VA_ARGS__)
+    struct upg_md5_t *md5 = buf;
+    cprintf(BLUE, "Preliminary\n");
+    cprintf(GREEN, "  MD5: ");
+    for(int i = 0; i < MD5_DIGEST_LENGTH; i++)
+        cprintf(YELLOW, "%02x", md5->md5[i]);
+    cprintf(OFF, " ");
+
+    /* check MD5 */
+    uint8_t actual_md5[MD5_DIGEST_LENGTH];
+    {
+        MD5_CTX c;
+        MD5_Init(&c);
+        MD5_Update(&c, md5 + 1, size - sizeof(struct upg_header_t));
+        MD5_Final(actual_md5, &c);
+    }
+    if(memcmp(actual_md5, md5->md5, MD5_DIGEST_LENGTH) != 0)
+    {
+        cprintf(RED, "Mismatch\n");
+        err_printf(GREY, "MD5 Mismatch\n");
+        return NULL;
+    }
+    cprintf(RED, "Ok\n");
+
+    struct upg_header_t *hdr = (void *)(md5 + 1);
+    /* decrypt the whole file at once */
+    fwp_read(hdr, size - sizeof(*md5), hdr, (void *)key);
+
+    cprintf(BLUE, "Header\n");
+    cprintf_field("  Signature:", " ");
+    for(int i = 0; i < NWZ_SIG_SIZE; i++)
+        cprintf(YELLOW, "%c", isprint(hdr->sig[i]) ? hdr->sig[i] : '.');
+    if(sig)
+    {
+        if(memcmp(hdr->sig, sig, NWZ_SIG_SIZE) != 0)
+        {
+            cprintf(RED, "Mismatch\n");
+            err_printf(GREY, "Signature Mismatch\n");
+            return NULL;
+        }
+        cprintf(RED, " Ok\n");
+    }
+    else
+        cprintf(RED, " Can't check\n");
+    cprintf_field("  Files: ", "%d\n", hdr->nr_files);
+    cprintf_field("  Pad: ", "0x%x\n", hdr->pad);
+
+
+    /* Do a first pass to decrypt in-place */
+    cprintf(BLUE, "Files\n");
+    struct upg_entry_t *entry = (void *)(hdr + 1);
+    for(unsigned i = 0; i < hdr->nr_files; i++, entry++)
+    {
+        cprintf(GREY, "  File");
+        cprintf(RED, " %d\n", i);
+        cprintf_field("    Offset: ", "0x%x\n", entry->offset);
+        cprintf_field("    Size: ", "0x%x\n", entry->size);
+    }
+    /* Do a second pass to create the file structure */
+    /* create file */
+    struct upg_file_t *file = malloc(sizeof(struct upg_file_t));
+    memset(file, 0, sizeof(struct upg_file_t));
+    file->nr_files = hdr->nr_files;
+    file->files = malloc(sizeof(struct upg_file_entry_t) * file->nr_files);
+
+    entry = (void *)(hdr + 1);
+    for(unsigned i = 0; i < hdr->nr_files; i++, entry++)
+    {
+        memset(&file->files[i], 0, sizeof(struct upg_file_entry_t));
+        file->files[i].size = entry->size;
+        file->files[i].data = malloc(file->files[i].size);
+        memcpy(file->files[i].data, buf + entry->offset, entry->size);
+    }
+
+    return file;
+}
+
+void *upg_write_memory(struct upg_file_t *file, char key[NWZ_KEY_SIZE],
+    char sig[NWZ_SIG_SIZE], size_t *out_size, void *u, generic_printf_t printf)
+{
+    if(file->nr_files == 0)
+    {
+        err_printf(GREY, "A UPG file must have at least one file\n");
+        return NULL;
+    }
+    /* compute total size and create buffer */
+    size_t tot_size = sizeof(struct upg_md5_t) + sizeof(struct upg_header_t)
+        + file->nr_files * sizeof(struct upg_entry_t);
+    for(int i = 0; i < file->nr_files; i++)
+        tot_size += ROUND_UP(file->files[i].size, 8);
+    /* allocate buffer */
+    void *buf = malloc(tot_size);
+
+    /* create md5 context, we push data to the context as we create it */
+    struct upg_md5_t *md5 = buf;
+    memset(md5, 0, sizeof(*md5));
+    /* create the encrypted signature and header */
+    struct upg_header_t *hdr = (void *)(md5 + 1);
+    memcpy(hdr->sig, sig, NWZ_SIG_SIZE);
+    hdr->nr_files = file->nr_files;
+    hdr->pad = 0;
+
+    /* create file headers */
+    size_t offset = sizeof(*md5) + sizeof(*hdr) + file->nr_files * sizeof(struct upg_entry_t);
+    struct upg_entry_t *entry = (void *)(hdr + 1);
+    cprintf(BLUE, "Files\n");
+    for(int i = 0; i < file->nr_files; i++)
+    {
+        entry[i].offset = offset;
+        entry[i].size = file->files[i].size;
+        offset += ROUND_UP(entry[i].size, 8); /* pad each file to a multiple of 8 for encryption */
+
+        cprintf(GREY, "  File");
+        cprintf(RED, " %d\n", i);
+        cprintf_field("    Offset: ", "0x%lx\n", entry[i].offset);
+        cprintf_field("    Size: ", "0x%lx\n", entry[i].size);
+    }
+
+    /* add file data */
+    for(int i = 0; i < file->nr_files; i++)
+    {
+        /* copy data to buffer, and then encrypt in-place */
+        size_t r_size = ROUND_UP(file->files[i].size, 8);
+        void *data_ptr = (uint8_t *)buf + entry[i].offset;
+        memset(data_ptr, 0, r_size); /* the padding will be zero 0 */
+        memcpy(data_ptr, file->files[i].data, file->files[i].size);
+    }
+    /* encrypt everything and hash everything */
+    fwp_write(hdr, tot_size - sizeof(*md5), hdr, (void *)key);
+    /* write final MD5 */
+    {
+        MD5_CTX c;
+        MD5_Init(&c);
+        MD5_Update(&c, (void *)hdr, tot_size - sizeof(*md5));
+        MD5_Final(md5->md5, &c);
+    }
+    *out_size = tot_size;
+    return buf;
+}
+
+struct upg_file_t *upg_new(void)
+{
+    struct upg_file_t *f = malloc(sizeof(struct upg_file_t));
+    memset(f, 0, sizeof(struct upg_file_t));
+    return f;
+}
+
+void upg_append(struct upg_file_t *file, void *data, size_t size)
+{
+    file->files = realloc(file->files, (file->nr_files + 1) * sizeof(struct upg_file_entry_t));
+    file->files[file->nr_files].data = data;
+    file->files[file->nr_files].size = size;
+    file->nr_files++;
+}
+
+void upg_free(struct upg_file_t *file)
+{
+    if(file)
+    {
+        for(int i = 0; i < file->nr_files; i++)
+            free(file->files[i].data);
+        free(file->files);
+    }
+    free(file);
+}
diff --git a/utils/nwztools/upgtools/upgtool.c b/utils/nwztools/upgtools/upgtool.c
index 833321ec12..0de46a4260 100644
--- a/utils/nwztools/upgtools/upgtool.c
+++ b/utils/nwztools/upgtools/upgtool.c
@@ -33,6 +33,7 @@
 #include "crypt.h"
 #include "fwp.h"
 #include "keysig_search.h"
+#include "upg.h"
 
 #ifndef MIN
 #define MIN(a,b) ((a) < (b) ? (a) : (b))
@@ -59,183 +60,11 @@ enum keysig_search_method_t g_keysig_search = KEYSIG_SEARCH_NONE;
     { cprintf(RED, str_bad); let_the_force_flow(__LINE__); } \
     else { cprintf(RED, str_ok); }
 
-static void usage(void);
-
-#define HAS_KAS     (1 << 0)
-#define HAS_KEY     (1 << 1)
-#define HAS_SIG     (1 << 2)
-#define CONFIRMED   (1 << 3)
-
-struct nwz_model_t
-{
-    const char *model;
-    unsigned flags;
-    char *kas;
-    char *key;
-    char *sig;
-};
-
-/** Firmware format
- *
- * The firmware starts with the MD5 hash of the entire file (except the MD5 hash
- * itself of course). This is used to check that the file was not corrupted.
- * The remaining of the file is encrypted (using DES) with the model key. The
- * encrypted part starts with a header containing the model signature and the
- * number of files. Since the header is encrypted, decrypting the header with
- * the key and finding the right signature serves to authenticate the firmware.
- * The header is followed by N entries (where N is the number of files) giving
- * the offset, within the file, and size of each file. Note that the files in
- * the firmware have no name. */
-
-struct upg_md5_t
-{
-    uint8_t md5[16];
-}__attribute__((packed));
-
-struct upg_header_t
-{
-    uint8_t sig[NWZ_SIG_SIZE];
-    uint32_t nr_files;
-    uint32_t pad; // make sure structure size is a multiple of 8
-} __attribute__((packed));
-
-struct upg_entry_t
-{
-    uint32_t offset;
-    uint32_t size;
-} __attribute__((packed));
-
-/** KAS / Key / Signature
- *
- * Since this is all very confusing, we need some terminology and notations:
- * - [X, Y, Z] is a sequence of bytes, for example:
- *     [8, 0x89, 42]
- *   is a sequence of three bytes.
- * - "abcdef" is a string: it is a sequences of bytes where each byte happens to
- *   be the ASCII encoding of a letter. So for example:
- *     "abc" = [97, 98, 99]
- *   because 'a' has ASCII encoding 97 and so one
- * - HexString(Seq) refers to the string where each byte of the original sequence
- *   is represented in hexadecimal by two ASCII characters. For example:
- *     HexString([8, 0x89, 42]) = "08892a"
- *   because 8 = 0x08 so it represented by "08" and 42 = 0x2a. Note that the length
- *   of HexString(Seq) is always exactly twice the length of Seq.
- * - DES(Seq,Pass) is the result of encrypting Seq with Pass using the DES cipher.
- *   Seq must be a sequence of 8 bytes (known as a block) and Pass must be a
- *   sequence of 8 bytes. The result is also a 8-byte sequence.
- * - ECB_DES([Block0, Block1, ..., BlockN], Pass)
- *     = [DES(Block0,Pass), DES(Block1,Pass), ..., DES(BlockN,Pass)]
- *   where Blocki is a block (8 byte).
- *
- *
- * A firmware upgrade file is always encrypted using a Key. To authenticate it,
- * the upgrade file (before encryption) contains a Sig(nature). The pair (Key,Sig)
- * is refered to as KeySig and is specific to each series. For example all
- * NWZ-E46x use the same KeySig but the NWZ-E46x and NWZ-A86x use different KeySig.
- * In the details, a Key is a sequence of 8 bytes and a Sig is also a sequence
- * of 8 bytes. A KeySig is a simply the concatenation of the Key followed by
- * the Sig, so it is a sequence of 16 bytes. Probably in an attempt to obfuscate
- * things a little further, Sony never provides the KeySig directly but instead
- * encrypts it using DES in ECB mode using a hardcoded password and provides
- * the hexadecimal string of the result, known as the KAS, which is thus a string
- * of 32 ASCII characters.
- * Note that since DES works on blocks of 8 bytes and ECB encrypts blocks
- * independently, it is the same to encrypt the KeySig as once or encrypt the Key
- * and Sig separately.
- *
- * To summarize:
- *   Key = [K0, K1, K2, ..., K7] (8 bytes) (model specific)
- *   Sig = [S0, S1, S2, ..., S7] (8 bytes) (model specific)
- *   KeySig = [Key, Sig] = [K0, ... K7, S0, ..., S7] (16 bytes)
- *   FwpPass = "ed295076" (8 bytes) (never changes)
- *   EncKeySig = ECB_DES(KeySig, FwpPass) = [DES(Key, FwpPass), DES(Sig, FwpPass)]
- *   KAS = HexString(EncKeySig) (32 characters)
- *
- * In theory, the Key and Sig can be any 8-byte sequence. In practice, they always
- * are strings, probably to make it easier to write them down. In many cases, the
- * Key and Sig are even the hexadecimal string of 4-byte sequences but it is
- * unclear if this is the result of pure luck, confused engineers, lazyness on
- * Sony's part or by design. The following code assumes that Key and Sig are
- * strings (though it could easily be fixed to work with anything if this is
- * really needed).
- *
- *
- * Here is a real example, from the NWZ-E46x Series:
- *   Key = "6173819e" (note that this is a string and even a hex string in this case)
- *   Sig = "30b82e5c"
- *   KeySig = [Key, Sig] = "6173819e30b82e5c"
- *   FwpPass = "ed295076" (never changes)
- *   EncKeySig = ECB_DES(KeySig, FwpPass)
- *             = [0x8a, 0x01, 0xb6, ..., 0xc5] (16 bytes)
- *   KAS = HexString(EncKeySig) = "8a01b624bfbfde4a1662a1772220e3c5"
- *
- */
+#define cprintf(col, ...) do {color(col); printf(__VA_ARGS__); }while(0)
 
-struct nwz_model_t g_model_list[] =
-{
-    { "nwz-e450", HAS_KAS | CONFIRMED, "8a01b624bfbfde4a1662a1772220e3c5", "", "" },
-    { "nwz-e460", HAS_KAS | CONFIRMED, "89d813f8f966efdebd9c9e0ea98156d2", "", "" },
-    { "nwz-a860", HAS_KAS | CONFIRMED, "a7c4af6c28b8900a783f307c1ba538c5", "", "" },
-    { "nwz-a850", HAS_KAS | CONFIRMED, "a2efb9168616c2e84d78291295c1aa5d", "", "" },
-    { "nwz-e470", HAS_KAS | CONFIRMED, "e4144baaa2707913f17b5634034262c4", "", "" },
-    /* The following keys were obtained by brute forcing firmware upgrades,
-     * someone with a device needs to confirm that they work */
-    { "nw-a820", HAS_KEY | HAS_SIG, "", "4df06482", "07fa0b6e" },
-    { "nwz-a10", HAS_KEY | HAS_SIG, "", "ec2888e2", "f62ced8a" },
-    { "nwz-a20", HAS_KEY | HAS_SIG, "", "e8e204ee", "577614df" },
-    { "nwz-zx100", HAS_KEY | HAS_SIG, "", "22e44606", "a9f95e90" },
-    { "nwz-e580", HAS_KEY | HAS_SIG, "", "a60806ea", "97e8ce46" },
-};
-
-static int digit_value(char c)
-{
-    if(c >= '0' && c <= '9') return c - '0';
-    if(c >= 'a' && c <= 'f') return c - 'a' + 10;
-    if(c >= 'A' && c <= 'F') return c - 'A' + 10;
-    return -1;
-}
+#define cprintf_field(str1, ...) do{ cprintf(GREEN, str1); cprintf(YELLOW, __VA_ARGS__); }while(0)
 
-static char hex_digit(unsigned v)
-{
-    return (v < 10) ? v + '0' : (v < 16) ? v - 10 + 'a' : 'x';
-}
-
-static int decrypt_keysig(const char kas[NWZ_KAS_SIZE], char key[NWZ_KEY_SIZE],
-    char sig[NWZ_SIG_SIZE])
-{
-    uint8_t src[NWZ_KAS_SIZE / 2];
-    for(int index = 0; index < NWZ_KAS_SIZE / 2; index++)
-    {
-        int a = digit_value(kas[index * 2]);
-        int b = digit_value(kas[index * 2 + 1]);
-        if(a < 0 || b < 0)
-        {
-            cprintf(GREY, "Invalid KAS !\n");
-            return -1;
-        }
-        src[index] = a << 4 | b;
-    }
-    fwp_setkey("ed295076");
-    fwp_crypt(src, sizeof(src), 1);
-    memcpy(key, src, NWZ_KEY_SIZE);
-    memcpy(sig, src + NWZ_KEY_SIZE, NWZ_SIG_SIZE);
-    return 0;
-}
-
-static void encrypt_keysig(char kas[NWZ_KEY_SIZE],
-    const char key[NWZ_SIG_SIZE], const char sig[NWZ_KAS_SIZE])
-{
-    uint8_t src[NWZ_KAS_SIZE / 2];
-    fwp_setkey("ed295076");
-    memcpy(src, key, NWZ_KEY_SIZE);
-    memcpy(src + NWZ_KEY_SIZE, sig, NWZ_SIG_SIZE);
-    fwp_crypt(src, sizeof(src), 0);
-    for(int i = 0; i < NWZ_KAS_SIZE / 2; i++)
-    {
-        kas[2 * i] = hex_digit((src[i] >> 4) & 0xf);
-        kas[2 * i + 1] = hex_digit(src[i] & 0xf);
-    }
-}
+static void usage(void);
 
 /* user needs to be pointer to a NWZ_KEYSIG_SIZE-byte buffer, on success g_key
  * and g_sig are updated to point to the key and sig in the buffer */
@@ -249,20 +78,13 @@ static bool upg_notify_keysig(void *user, uint8_t key[NWZ_KEY_SIZE],
     return true;
 }
 
-static int get_key_and_sig(bool is_extract, void *encrypted_hdr)
+static int get_key_and_sig(bool is_extract, void *buf)
 {
     static char keysig[NWZ_KEYSIG_SIZE];
     static char kas[NWZ_KAS_SIZE];
     /* database lookup */
     if(g_model_index != -1)
-    {
-        if(g_model_list[g_model_index].flags & HAS_KAS)
-            g_kas = g_model_list[g_model_index].kas;
-        if(g_model_list[g_model_index].flags & HAS_KEY)
-            g_key = g_model_list[g_model_index].key;
-        if(g_model_list[g_model_index].flags & HAS_SIG)
-            g_sig = g_model_list[g_model_index].sig;
-    }
+        g_kas = g_model_list[g_model_index].kas;
 
     /* always prefer KAS because it contains everything */
     if(g_kas)
@@ -276,33 +98,26 @@ static int get_key_and_sig(bool is_extract, void *encrypted_hdr)
         g_sig = keysig + NWZ_KEY_SIZE;
         decrypt_keysig(g_kas, g_key, g_sig);
     }
-    /* fall back to key and signature otherwise. The signature is not required
-     * when extracting but prevents from checking decryption */
-    else if(g_key && (is_extract || g_sig))
+    /* Otherwise require key and signature */
+    else if(g_key && g_sig)
     {
+        /* check key and signature size */
         if(strlen(g_key) != 8)
         {
             cprintf(GREY, "The specified key has wrong length (must be 8 hex digits)\n");
             return 4;
         }
-
-        /* if there is a signature, it must have the correct size */
-        if(g_sig)
+        if(strlen(g_sig) != 8)
         {
-            if(strlen(g_sig) != 8)
-            {
-                cprintf(GREY, "The specified sig has wrong length (must be 8 hex digits)\n");
-                return 5;
-            }
-        }
-        else
-        {
-            cprintf(GREY, "Warning: you have specified a key but no sig, I won't be able to do any checks\n");
+            cprintf(GREY, "The specified sig has wrong length (must be 8 hex digits)\n");
+            return 5;
         }
     }
     /* for extraction, we offer a brute force search method from the MD5 */
     else if(is_extract && g_keysig_search != KEYSIG_SEARCH_NONE)
     {
+        struct upg_md5_t *md5 = (void *)buf;
+        void *encrypted_hdr = (md5 + 1);
         cprintf(BLUE, "keysig Search\n");
         cprintf_field("  Method: ", "%s\n", keysig_search_desc[g_keysig_search].name);
         bool ok = keysig_search(g_keysig_search, encrypted_hdr, 8,
@@ -329,13 +144,8 @@ static int get_key_and_sig(bool is_extract, void *encrypted_hdr)
      * valid files anyway */
     if(!g_kas)
     {
-        if(!g_sig)
-        {
-            /* if we extract and don't have a signature, just use a random
-             * one, we cannot check it anyway */
-            g_sig = keysig;
-            memset(g_sig, '?', NWZ_SIG_SIZE);
-        }
+        /* This is useful to print the KAS for the user when brute-forcing since
+         * the process will produce a key+sig and the database requires a KAS */
         g_kas = kas;
         encrypt_keysig(g_kas, g_key, g_sig);
     }
@@ -343,86 +153,38 @@ static int get_key_and_sig(bool is_extract, void *encrypted_hdr)
     cprintf(BLUE, "Keys\n");
     cprintf_field("  KAS: ", "%."STR(NWZ_KAS_SIZE)"s\n", g_kas);
     cprintf_field("  Key: ", "%."STR(NWZ_KEY_SIZE)"s\n", g_key);
-    if(g_sig)
-        cprintf_field("  Sig: ", "%."STR(NWZ_SIG_SIZE)"s\n", g_sig);
+    cprintf_field("  Sig: ", "%."STR(NWZ_SIG_SIZE)"s\n", g_sig);
 
     return 0;
 }
 
 static int do_upg(void *buf, long size)
 {
-    struct upg_md5_t *md5 = buf;
-    cprintf(BLUE, "Preliminary\n");
-    cprintf(GREEN, "  MD5: ");
-    for(int i = 0; i < 16; i++)
-        cprintf(YELLOW, "%02x", md5->md5[i]);
-    printf(" ");
-
-    uint8_t actual_md5[MD5_DIGEST_LENGTH];
-    {
-        MD5_CTX c;
-        MD5_Init(&c);
-        MD5_Update(&c, md5 + 1, size - sizeof(struct upg_header_t));
-        MD5_Final(actual_md5, &c);
-    }
-    check_field(memcmp(actual_md5, md5->md5, 16), 0, "Ok\n", "Mismatch\n");
-
-    int ret = get_key_and_sig(true, md5 + 1);
+    int ret = get_key_and_sig(true, buf);
     if(ret != 0)
         return ret;
-
-    struct upg_header_t *hdr = (void *)(md5 + 1);
-    ret = fwp_read(hdr, sizeof(struct upg_header_t), hdr, (void *)g_key);
-    if(ret)
+    struct upg_file_t *file = upg_read_memory(buf, size, g_key, g_sig, NULL,
+        generic_std_printf);
+    if(file == NULL)
         return ret;
-
-    cprintf(BLUE, "Header\n");
-    cprintf_field("  Signature:", " ");
-    for(int i = 0; i < 8; i++)
-        cprintf(YELLOW, "%c", isprint(hdr->sig[i]) ? hdr->sig[i] : '.');
-    if(g_sig)
+    for(int i = 0; i < file->nr_files; i++)
     {
-        check_field(memcmp(hdr->sig, g_sig, 8), 0, " OK\n", " Mismatch\n");
-    }
-    else
-        cprintf(RED, " Can't check\n");
-    cprintf_field("  Files: ", "%d\n", hdr->nr_files);
-    cprintf_field("  Pad: ", "0x%x\n", hdr->pad);
-
-    cprintf(BLUE, "Files\n");
-    struct upg_entry_t *entry = (void *)(hdr + 1);
-    for(unsigned i = 0; i < hdr->nr_files; i++, entry++)
-    {
-        int ret = fwp_read(entry, sizeof(struct upg_entry_t), entry, (void *)g_key);
-        if(ret)
-            return ret;
-        cprintf(GREY, "  File");
-        cprintf(RED, " %d\n", i);
-        cprintf_field("    Offset: ", "0x%x\n", entry->offset);
-        cprintf_field("    Size: ", "0x%x\n", entry->size);
-
-        if(g_out_prefix)
+        if(!g_out_prefix)
+            continue;
+        char *str = malloc(strlen(g_out_prefix) + 32);
+        sprintf(str, "%s%d.bin", g_out_prefix, i);
+        FILE *f = fopen(str, "wb");
+        if(!f)
         {
-            char *str = malloc(strlen(g_out_prefix) + 32);
-            sprintf(str, "%s%d.bin", g_out_prefix, i);
-            FILE *f = fopen(str, "wb");
-            if(f)
-            {
-                // round up size, there is some padding done with random data
-                int crypt_size = ROUND_UP(entry->size, 8);
-                int ret = fwp_read(buf + entry->offset, crypt_size,
-                    buf + entry->offset, (void *)g_key);
-                if(ret)
-                    return ret;
-                // but write the *good* amount of data
-                fwrite(buf + entry->offset, 1, entry->size, f);
-                fclose(f);
-            }
-            else
-                cprintf(GREY, "Cannot open '%s' for writing\n", str);
+            cprintf(GREY, "Cannot open '%s' for writing\n", str);
+            free(str);
+            continue;
         }
+        free(str);
+        fwrite(file->files[i].data, 1, file->files[i].size, f);
+        fclose(f);
     }
-
+    upg_free(file);
     return 0;
 }
 
@@ -490,7 +252,7 @@ static int create_upg(int argc, char **argv)
 {
     if(argc == 0)
     {
-        printf("You must specify a firmware filename\n");
+        cprintf(GREY, "You must specify a firmware filename\n");
         usage();
     }
 
@@ -498,89 +260,55 @@ static int create_upg(int argc, char **argv)
     if(ret != 0)
         return ret;
 
-    FILE *fout = fopen(argv[0], "wb");
-    if(fout == NULL)
-    {
-        printf("Cannot open output firmware file: %m\n");
-        return 1;
-    }
-
+    struct upg_file_t *upg = upg_new();
     int nr_files = argc - 1;
-    FILE **files = malloc(nr_files * sizeof(FILE *));
 
     for(int i = 0; i < nr_files; i++)
     {
-        files[i] = fopen(argv[1 + i], "rb");
-        if(files[i] == NULL)
+        FILE *f = fopen(argv[1 + i], "rb");
+        if(f == NULL)
+        {
+            upg_free(upg);
+            printf(GREY, "Cannot open input file '%s': %m\n", argv[i + 1]);
+            return 1;
+        }
+        size_t size = filesize(f);
+        void *buf = malloc(size);
+        if(fread(buf, 1, size, f) != size)
         {
-            printf("Cannot open input file '%s': %m\n", argv[i + 1]);
+            cprintf(GREY, "Cannot read input file '%s': %m\n", argv[i + 1]);
+            fclose(f);
+            upg_free(upg);
             return 1;
         }
+        fclose(f);
+        upg_append(upg, buf, size);
     }
 
-    struct upg_md5_t md5;
-    memset(&md5, 0, sizeof(md5));
-    MD5_CTX c;
-    MD5_Init(&c);
-    // output a dummy md5 sum
-    fwrite(&md5, 1, sizeof(md5), fout);
-    // output the encrypted signature
-    struct upg_header_t hdr;
-    memcpy(hdr.sig, g_sig, 8);
-    hdr.nr_files = nr_files;
-    hdr.pad = 0;
-
-    ret = fwp_write(&hdr, sizeof(hdr), &hdr, (void *)g_key);
-    if(ret)
-        return ret;
-    MD5_Update(&c, &hdr, sizeof(hdr));
-    fwrite(&hdr, 1, sizeof(hdr), fout);
-
-    // output file headers
-    long offset = sizeof(md5) + sizeof(hdr) + nr_files * sizeof(struct upg_entry_t);
-    for(int i = 0; i < nr_files; i++)
+    size_t size = 0;
+    void *buf = upg_write_memory(upg, g_key, g_sig, &size, NULL, generic_std_printf);
+    upg_free(upg);
+    if(buf == NULL)
     {
-        struct upg_entry_t entry;
-        entry.offset = offset;
-        entry.size = filesize(files[i]);
-        offset += ROUND_UP(entry.size, 8); // do it before encryption !!
-
-        ret = fwp_write(&entry, sizeof(entry), &entry, (void *)g_key);
-        if(ret)
-            return ret;
-        MD5_Update(&c, &entry, sizeof(entry));
-        fwrite(&entry, 1, sizeof(entry), fout);
+        cprintf(GREY, "Error creating UPG file\n");
+        return 1;
     }
-
-    cprintf(BLUE, "Files\n");
-    for(int i = 0; i < nr_files; i++)
+    FILE *fout = fopen(argv[0], "wb");
+    if(fout == NULL)
     {
-        long size = filesize(files[i]);
-        long r_size = ROUND_UP(size, 8);
-        cprintf(GREY, "  File");
-        cprintf(RED, " %d\n", i);
-        cprintf_field("    Offset: ", "0x%lx\n", ftell(fout));
-        cprintf_field("    Size: ", "0x%lx\n", size);
-
-        void *buf = malloc(r_size);
-        memset(buf, 0, r_size);
-        fread(buf, 1, size, files[i]);
-        fclose(files[i]);
-
-        ret = fwp_write(buf, r_size, buf, (void *)g_key);
-        if(ret)
-            return ret;
-        MD5_Update(&c, buf, r_size);
-        fwrite(buf, 1, r_size, fout);
-
+        cprintf(GREY, "Cannot open output firmware file: %m\n");
         free(buf);
+        return 1;
+    }
+    if(fwrite(buf, 1, size, fout) != size)
+    {
+        cprintf(GREY, "Cannot write output file: %m\n");
+        fclose(fout);
+        free(buf);
+        return 1;
     }
-
-    fseek(fout, 0, SEEK_SET);
-    MD5_Final(md5.md5, &c);
-    fwrite(&md5, 1, sizeof(md5), fout);
     fclose(fout);
-
+    free(buf);
     return 0;
 }
 
@@ -703,25 +431,13 @@ int main(int argc, char **argv)
     if(g_model && strcmp(g_model, "?") == 0)
     {
         cprintf(BLUE, "Model list:\n");
-        for(unsigned i = 0; i < sizeof(g_model_list) / sizeof(g_model_list[0]); i++)
+        for(unsigned i = 0; g_model_list[i].model; i++)
         {
             cprintf(GREEN, "  %s:", g_model_list[i].model);
-            if(g_model_list[i].flags & HAS_KAS)
-            {
-                cprintf(RED, " kas=");
-                cprintf(YELLOW, "%."STR(NWZ_KAS_SIZE)"s", g_model_list[i].kas);
-            }
-            if(g_model_list[i].flags & HAS_KEY)
-            {
-                cprintf(RED, " key=");
-                cprintf(YELLOW, "%.8s", g_model_list[i].key);
-            }
-            if(g_model_list[i].flags & HAS_SIG)
-            {
-                cprintf(RED, " sig=");
-                cprintf(YELLOW, "%.8s", g_model_list[i].sig);
-            }
-            if(g_model_list[i].flags & CONFIRMED)
+
+            cprintf(RED, " kas=");
+            cprintf(YELLOW, "%."STR(NWZ_KAS_SIZE)"s", g_model_list[i].kas);
+            if(g_model_list[i].confirmed)
                 cprintf(RED, " confirmed");
             else
                 cprintf(RED, " guessed");
@@ -732,7 +448,7 @@ int main(int argc, char **argv)
 
     if(g_model)
     {
-        for(unsigned i = 0; i < sizeof(g_model_list) / sizeof(g_model_list[0]); i++)
+        for(unsigned i = 0; g_model_list[i].model; i++)
             if(strcmp(g_model, g_model_list[i].model) == 0)
                 g_model_index = i;
         if(g_model_index == -1)