Commit 7abd7c7
Index the Unicode data base
Changed files (7)
modifiedbirdfont/Main.valamodifiedinstall.pymodifiedlibbirdfont/Argument.valamodifiedlibbirdfont/BirdFont.valamodifiedlibbirdfont/CharDatabase.valamodifiedlibbirdfont/CharDatabaseParser.valaaddedresources/ucd.sqlite
Diff
@@ -23,7 +23,6 @@ public static int main (string[] arg) { string file; BirdFont.BirdFont birdfont; - Icons.use_high_resolution (true); birdfont = new BirdFont.BirdFont (); birdfont.init (arg, null); Gtk.init (ref arg);@@ -36,50 +35,8 @@ public static int main (string[] arg) { birdfont.load_font_from_command_line (); - load_ucd (); Gtk.main ();- return 0; } -/** Load descriptions from the unicode character database in a - * background thread.- */-void load_ucd () { - CharDatabaseParser db;- unowned Thread<CharDatabaseParser> db_thread;- Mutex database_mutex = new Mutex ();- Cond main_loop_idle = new Cond ();- bool in_idle = false;- - try {- db = new CharDatabaseParser ();- db_thread = Thread.create<CharDatabaseParser> (db.load, false);- - // wait until main loop is done- db.sync.connect (() => {- database_mutex.lock ();- IdleSource idle = new IdleSource ();- in_idle = false;- - idle.set_callback (() => {- database_mutex.lock ();- in_idle = true;- main_loop_idle.broadcast ();- database_mutex.unlock ();- return false;- });- idle.attach (null);- - while (!in_idle) {- main_loop_idle.wait (database_mutex);- }- - database_mutex.unlock ();- });- } catch (GLib.Error e) {- warning (e.message);- }-}- @@ -105,6 +105,7 @@ install ('resources/key_bindings.xml', '/share/birdfont', 644) install ('resources/roboto.bf', '/share/birdfont', 644) install ('resources/linux/birdfont_window_icon.png', '/share/birdfont', 644) install ('resources/linux/birdfont.desktop', '/share/applications', 644)+install ('resources/ucd.sqlite', '/share/birdfont', 644) install ('resources/linux/256x256/birdfont.png', '/share/icons/hicolor/256x256/apps', 644) install ('resources/linux/128x128/birdfont.png', '/share/icons/hicolor/128x128/apps', 644)@@ -1,5 +1,5 @@ /*- Copyright (C) 2012 Johan Mattsson+ Copyright (C) 2012 2015 Johan Mattsson This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as @@ -16,18 +16,18 @@ namespace BirdFont { public class Argument : GLib.Object { - List<string> args;+ Gee.ArrayList<string> args; public Argument (string line) {- args = new List<string> ();+ args = new Gee.ArrayList<string> (); set_argument (line); } public Argument.command_line (string[] arg) { - args = new List<string> ();+ args = new Gee.ArrayList<string> (); foreach (string a in arg) {- args.append (a);+ args.add (a); } } @@ -35,8 +35,9 @@ public class Argument : GLib.Object { public int validate () { string prev = ""; int i = 0;- foreach (string a in args) {- + string[] p;+ + foreach (string a in args) { if (a == "") { continue; }@@ -55,6 +56,11 @@ public class Argument : GLib.Object { continue; } + if (a.index_of ("=") > -1) {+ p = a.split ("=");+ a = p[0];+ }+ // a single character, like -t if (!a.has_prefix ("--") && a.has_prefix ("-")) { a = expand_param (a);@@ -71,8 +77,8 @@ public class Argument : GLib.Object { a == "--mac" || a == "--android" || a == "--log" ||- a == "--no-ucd" ||- a == "--windows") {+ a == "--windows" ||+ a == "--parse-ucd") { prev = a; i++; continue;@@ -96,8 +102,8 @@ public class Argument : GLib.Object { public string get_file () { string f = ""; - if (args.length () >= 2) {- f = args.nth (1).data;+ if (args.size >= 2) {+ f = args.get (1); } if (f.has_prefix ("-")) {@@ -108,7 +114,7 @@ public class Argument : GLib.Object { } public void print_all () {- print (@"$(args.length ()) arguments:\n");+ print (@"$(args.size) arguments:\n"); foreach (string p in args) { print (@"$p\n");@@ -124,7 +130,9 @@ public class Argument : GLib.Object { int i = 0; string? n; string p;-+ string v = "";+ string[] pm;+ if (param.substring (0, 1) != "-") { warning (@"parameters must begin with \"-\" got $param"); return null;@@ -132,10 +140,22 @@ public class Argument : GLib.Object { foreach (string s in args) { + if (s.index_of ("=") > -1) {+ pm = s.split ("=");+ + if (pm.length > 1) {+ v = pm[1];+ }+ + s = pm[0];+ }+ // this is content not a parameter - if (s.substring (0, 1) != "-") continue;+ if (s.substring (0, 1) != "-") {+ continue;+ } - // we might need to expand -t to test fo instance+ // we might need to expand -t to test for instance if (s.substring (0, 2) != "--") { p = expand_param (s); } else { @@ -143,20 +163,24 @@ public class Argument : GLib.Object { } if (param == p) {- if (i + 2 >= args.length ()) {+ if (v != "") {+ return v;+ }+ + if (i + 2 >= args.size) { return ""; } - n = args.nth (i + 2).data;+ n = args.get (i + 2); if (n == null) { return ""; } - if (args.nth (i + 2).data.substring (0, 1) == "-") {+ if (args.get (i + 2).substring (0, 1) == "-") { return ""; } - return args.nth (i + 2).data;+ return args.get (i + 2); } i++;@@ -184,10 +208,14 @@ public class Argument : GLib.Object { private string expand_param (string? param) { if (param == null) return ""; var p = (!) param;++ if (p.get_char (0) != '-') {+ return "";+ } - if (p.length == 0) return "";- if (p.get_char (0) != '-') return "";- if (p.char_count () != 2) return "";+ if (p.char_count () != 2) {+ return "";+ } switch (p.get_char (1)) { case 'c':@@ -235,17 +263,17 @@ public class Argument : GLib.Object { n = arg.substring (i, a - i + 1); } - args.append (n);+ args.add (n); i += n.char_count () + 1; } while (i < arg.char_count ()); } public void print_help () - requires (args.length () > 0)+ requires (args.size > 0) { stdout.printf (t_("Usage") + ": ");- stdout.printf (args.nth (0).data);+ stdout.printf (args.get (0)); stdout.printf (" [" + t_("FILE") + "] [" + t_("OPTION") + " ...]\n"); print_padded ("-a, --android", t_("enable Android customizations"));@@ -398,7 +398,8 @@ public class BirdFont { string theme; int default_theme_version; string theme_version;- + CharDatabaseParser parser;+ args = new Argument.command_line (arg); #if ANDROID@@ -524,6 +525,11 @@ public class BirdFont { if (TestBirdFont.get_singleton ().test_cases_to_run != "All") { TestBirdFont.run_tests (); }+ + if (has_argument ("--parse-ucd")) {+ parser = new CharDatabaseParser ();+ parser.regenerate_database ();+ } } public static Argument get_arguments () {@@ -1,5 +1,5 @@ /*- Copyright (C) 2012 Johan Mattsson+ Copyright (C) 2012 2015 Johan Mattsson This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as @@ -19,7 +19,6 @@ namespace BirdFont { public class CharDatabase { public static GlyphRange full_unicode_range;- public static bool database_is_loaded = false; public static unowned Database db; public static Database? database = null;@@ -28,24 +27,8 @@ public class CharDatabase { File f; full_unicode_range = new GlyphRange ();- f = get_database_file ();- regenerate_database (); - }- - public static void regenerate_database () {- File f = get_database_file ();- - try {- if (f.query_exists ()) {- f.delete ();- }- - open_database ();- create_tables ();- } catch (GLib.Error e) {- warning (e.message);- }+ open_database () ; } public static void open_database () {@@ -59,66 +42,8 @@ public class CharDatabase { } } - public static void create_tables () {- int ec;- string? errmsg;- string description_table = """- CREATE TABLE Description (- unicode INTEGER PRIMARY KEY NOT NULL,- description TEXT NOT NULL- );- """;-- ec = db.exec (description_table, null, out errmsg);- if (ec != Sqlite.OK) {- warning ("Error: %s\n", (!) errmsg);- }-- string index_table = """- CREATE TABLE Words (- unicode INTEGER NOT NULL,- word TEXT NOT NULL- );- """;-- ec = db.exec (index_table, null, out errmsg);- if (ec != Sqlite.OK) {- warning ("Error: %s\n", (!) errmsg);- }-- string create_index = "CREATE INDEX word_index ON Words (word);";-- ec = db.exec (create_index, null, out errmsg);- if (ec != Sqlite.OK) {- warning ("Error: %s\n", (!) errmsg);- }- }- public static File get_database_file () {- return get_child (BirdFont.get_settings_directory (), "ucd.sqlite");- }-- public static void add_lookup (int64 character, string word) {- string? errmsg;- string query = """- INSERT INTO Words (unicode, word)- VALUES (""" + @"$((int64) character)" + """, '""" + word.replace ("'", "\'") + "\"');";- int ec = db.exec (query, null, out errmsg);- if (ec != Sqlite.OK) {- warning ("Error: %s\n", (!) errmsg);- }- }- - public static void add_entry (int64 character, string description) {- string? errmsg;- string query = """- INSERT INTO Description (unicode, description)- VALUES (""" + @"$((int64) character)" + """, '""" + description.replace ("'", "\'") + "\"');";- int ec = db.exec (query, null, out errmsg);- if (ec != Sqlite.OK) {- warning ("Error: %s\n", (!) errmsg);- warning (@"Can't insert description to: $(character)");- }+ return SearchPaths.find_file (null, "ucd.sqlite"); } public static GlyphRange search (string s) {@@ -137,12 +62,16 @@ public class CharDatabase { } } + if (s.char_count () == 1) {+ result.add_single (s.get_char (0));+ }+ select = "SELECT unicode FROM Words "- + "WHERE word = '" + s.replace ("'", "\\'") + "'";+ + "WHERE word = '" + s.replace ("'", "''") + "';"; rc = db.prepare_v2 (select, select.length, out statement, null); - if (rc != 1) {+ if (rc == Sqlite.OK) { cols = statement.column_count(); if (cols != 1) {@@ -156,7 +85,7 @@ public class CharDatabase { if (rc == Sqlite.DONE) { break; } else if (rc == Sqlite.ROW) {- c = (unichar) statement.column_integer (0);+ c = (unichar) statement.column_int64 (0); ucd_result.add_single (c); } else { printerr ("Error: %d, %s\n", rc, db.errmsg ());@@ -207,22 +136,6 @@ public class CharDatabase { return false; } - /** Convert from the U+xx form to the unicode database hex value. */ - static string to_database_hex (unichar c) {- string hex_char = Font.to_hex (c).replace ("U+", "");-- if (hex_char.char_count () == 2) {- hex_char = "00" + hex_char;- }- - if (hex_char.char_count () == 6 && hex_char.has_prefix ("0")) {- hex_char = hex_char.substring (1);- }- - hex_char = hex_char.up (); - return hex_char;- }- public static string get_unicode_database_entry (unichar c) { string description = ""; int rc, cols;@@ -232,7 +145,7 @@ public class CharDatabase { rc = db.prepare_v2 (select, select.length, out statement, null); - if (rc != 1) {+ if (rc == Sqlite.OK) { cols = statement.column_count(); if (cols != 1) {@@ -1,5 +1,5 @@ /*- Copyright (C) 2013 Johan Mattsson+ Copyright (C) 2013 2015 Johan Mattsson This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as @@ -13,18 +13,120 @@ */ using Gee;+using Sqlite; namespace BirdFont { public class CharDatabaseParser : GLib.Object {+ public static unowned Database db;+ public static Database? database = null; - public signal void sync ();- GlyphRange utf8 = new GlyphRange (); public CharDatabaseParser () { } + public File get_database_file () {+ string? fn = BirdFont.get_argument ("--parse-ucd");+ + if (fn != null && ((!) fn) != "") {+ return File.new_for_path ((!) fn);+ }+ + return File.new_for_path ("ucd.sqlite");+ }+ + public void regenerate_database () {+ File f = get_database_file ();+ + stdout.printf ("Generating sqlite database in: %s\n", (!) f.get_path ());+ + try {+ if (f.query_exists ()) {+ f.delete ();+ }+ + open_database ();+ create_tables ();+ parse_all_entries ();+ } catch (GLib.Error e) {+ warning (e.message);+ }+ }+ + public void open_database () {+ File f = get_database_file ();+ int rc = Database.open ((!) f.get_path (), out database);++ db = (!) database;++ if (rc != Sqlite.OK) {+ stderr.printf ("Can't open database: %d, %s\n", rc, db.errmsg ());+ }+ }+ + public void create_tables () {+ int ec;+ string? errmsg;+ string description_table = """+ CREATE TABLE Description (+ unicode INTEGER PRIMARY KEY NOT NULL,+ description TEXT NOT NULL+ );+ """;++ ec = db.exec (description_table, null, out errmsg);+ if (ec != Sqlite.OK) {+ warning ("Error: %s\n", (!) errmsg);+ }++ string index_table = """+ CREATE TABLE Words (+ unicode INTEGER NOT NULL,+ word TEXT NOT NULL+ );+ """;++ ec = db.exec (index_table, null, out errmsg);+ if (ec != Sqlite.OK) {+ warning ("Error: %s\n", (!) errmsg);+ }++ string create_index = "CREATE INDEX word_index ON Words (word);";++ ec = db.exec (create_index, null, out errmsg);+ if (ec != Sqlite.OK) {+ warning ("Error: %s\n", (!) errmsg);+ }+ }++ public void insert_lookup (int64 character, string word) {+ string? errmsg;+ string query = """+ INSERT INTO Words (unicode, word)+ VALUES (""" + @"$((int64) character)" + """, '""" + word.replace ("'", "''") + "');";+ int ec = db.exec (query, null, out errmsg);+ if (ec != Sqlite.OK) {+ stderr.printf (query);+ warning ("Error: %s\n", (!) errmsg);+ }+ }+ + public void insert_entry (int64 character, string description) {+ string? errmsg;+ string query = """+ INSERT INTO Description (unicode, description)+ VALUES (""" + @"$((int64) character)" + """, '""" + description.replace ("'", "''") + "');";+ + int ec = db.exec (query, null, out errmsg);+ + if (ec != Sqlite.OK) {+ stderr.printf (query);+ warning ("Error: %s\n", (!) errmsg);+ warning (@"Can't insert description to: $(character)");+ }+ }+ private void add_entry (string data) { string[] e; string[] r;@@ -61,13 +163,8 @@ public class CharDatabaseParser : GLib.Object { unicode_hex = e[0].up (); ch = Font.to_unichar ("U+" + unicode_hex.down ());- - Idle.add (() => {- CharDatabase.add_entry ((int64) ch, data);- return false;- });- sync ();- + stdout.printf ("Adding " + (!) ch.to_string () + "\n");+ insert_entry ((int64) ch, data); utf8.add_single (ch); foreach (string s in e) {@@ -76,11 +173,7 @@ public class CharDatabaseParser : GLib.Object { d = t.split (" "); foreach (string token in d) { if (token != "") {- Idle.add (() => {- CharDatabase.add_lookup ((int64)ch, token);- return false;- });- sync ();+ insert_lookup ((int64)ch, token); } } }@@ -94,14 +187,17 @@ public class CharDatabaseParser : GLib.Object { string data; string description = ""; File file;+ int ec;+ string? errmsg;+ uint64 transaction_number = 0;+ + file = get_unicode_database (); - if (BirdFont.has_argument ("--no-ucd")) {- warning ("Not loading UCD.");- return;+ ec = db.exec ("BEGIN TRANSACTION", null, out errmsg);+ if (ec != Sqlite.OK) {+ warning ("Error: %s\n", (!) errmsg); }-- file = get_unicode_database ();- + try { fin = file.read (); din = new DataInputStream (fin);@@ -119,6 +215,23 @@ public class CharDatabaseParser : GLib.Object { } else { if (description.index_of ("<not a character>") == -1) { add_entry (description);+ transaction_number++;+ + if (transaction_number >= 1000) {+ print ("Write to database\n");+ + ec = db.exec ("END TRANSACTION", null, out errmsg);+ if (ec != Sqlite.OK) {+ warning ("Error: %s\n", (!) errmsg);+ } ++ ec = db.exec ("BEGIN TRANSACTION", null, out errmsg);+ if (ec != Sqlite.OK) {+ warning ("Error: %s\n", (!) errmsg);+ }+ + transaction_number = 0; + } } break; } @@ -139,23 +252,16 @@ public class CharDatabaseParser : GLib.Object { warning (e.message); warning ("In %s", (!) get_unicode_database ().get_path ()); }- }- - public CharDatabaseParser load () {- parse_all_entries (); - IdleSource idle = new IdleSource ();- idle.set_callback (() => {- CharDatabase.full_unicode_range = utf8;- CharDatabase.database_is_loaded = true;- return false;- });- idle.attach (null);- - return this;+ ec = db.exec ("END TRANSACTION", null, out errmsg);+ if (ec != Sqlite.OK) {+ warning ("Error: %s\n", (!) errmsg);+ }+ + stdout.printf ("Done"); } - static File get_unicode_database () {+ File get_unicode_database () { return SearchPaths.get_char_database (); } }new file mode 100644Binary files /dev/null and b/resources/ucd.sqlite differ