Use UTF-8 internally and respect platform filename encoding.

Added 'textadept.iconv' function to interface with GLib's g_convert() and set
global _CHARSET to be the platform's filename encoding/code page.

3 files changed, 52 insertions, 37 deletions

diff --git a/core/ext/find.lua b/core/ext/find.lua
index 6bbeba0b..064a7ddb 100644
--- a/core/ext/find.lua
+++ b/core/ext/find.lua
@@ -81,7 +81,7 @@ function find.find(text, next, flags, nowrap, wrapped)
     end
 
   else -- find in files
-    local dir =
+    local utf8_dir =
       cocoa_dialog('fileselect', {
         title = locale.FIND_IN_FILES_TITLE,
         text = locale.FIND_IN_FILES_TEXT,
@@ -89,12 +89,13 @@ function find.find(text, next, flags, nowrap, wrapped)
         ['with-directory'] = (buffer.filename or ''):match('^.+[/\\]'),
         ['no-newline'] = true
       })
-    if #dir > 0 then
+    if #utf8_dir > 0 then
       if not find.lua then text = text:gsub('([().*+?^$%%[%]-])', '%%%1') end
       if not find.match_case then text = text:lower() end
       if find.whole_word then text = '[^%W_]'..text..'[^%W_]' end
       local match_case = find.match_case
       local whole_word = find.whole_word
+      local iconv = textadept.iconv
       local format = string.format
       local matches = { 'Find: '..text }
       function search_file(file)
@@ -104,6 +105,7 @@ function find.find(text, next, flags, nowrap, wrapped)
           if not match_case then optimized_line = line:lower() end
           if whole_word then optimized_line = ' '..line..' ' end
           if string.find(optimized_line, text) then
+            file = iconv(file, 'UTF-8', _CHARSET)
             matches[#matches + 1] = format('%s:%s:%s', file, line_num, line)
           end
           line_num = line_num + 1
@@ -122,6 +124,7 @@ function find.find(text, next, flags, nowrap, wrapped)
           end
         end
       end
+      local dir = iconv(utf8_dir, _CHARSET, 'UTF-8')
       search_dir(dir)
       if #matches == 1 then matches[2] = locale.FIND_NO_RESULTS end
       matches[#matches + 1] = ''
@@ -293,7 +296,7 @@ end
 textadept.events.add_handler('double_click', goto_file)
 
 ---
--- [Local function] Goes to the next or previous file found relative to the file
+-- Goes to the next or previous file found relative to the file
 -- on the current line.
 -- @param next Flag indicating whether or not to go to the next file.
 function find.goto_file_in_list(next)
diff --git a/core/ext/pm/file_browser.lua b/core/ext/pm/file_browser.lua
index 21309a21..8bff30f0 100644
--- a/core/ext/pm/file_browser.lua
+++ b/core/ext/pm/file_browser.lua
@@ -25,17 +25,19 @@ function matches(entry_text)
 end
 
 function get_contents_for(full_path)
+  local iconv = textadept.iconv
   local dir = {}
-  local dirpath = table.concat(full_path, '/')
+  local dirpath = iconv(table.concat(full_path, '/'), _CHARSET, 'UTF-8')
   local path = lfs.attributes(dirpath)
-  local invalid_file = show_dot_files and '^%.%.?$' or '^%.'
   if path and path.mode == 'directory' then
-    for name in lfs.dir(dirpath) do
-      if not name:find(invalid_file) then
-        dir[name] = { text = name }
-        if lfs.attributes(dirpath..'/'..name, 'mode') == 'directory' then
-          dir[name].parent = true
-          dir[name].pixbuf = 'gtk-directory'
+    local invalid_file = show_dot_files and '^%.%.?$' or '^%.'
+    for filename in lfs.dir(dirpath) do
+      if not filename:find(invalid_file) then
+        local utf8_filename = iconv(filename, 'UTF-8', _CHARSET)
+        dir[utf8_filename] = { text = utf8_filename }
+        if lfs.attributes(dirpath..'/'..filename, 'mode') == 'directory' then
+          dir[utf8_filename].parent = true
+          dir[utf8_filename].pixbuf = 'gtk-directory'
         end
       end
     end
@@ -44,8 +46,8 @@ function get_contents_for(full_path)
 end
 
 function perform_action(selected_item)
-  local filepath = table.concat(selected_item, '/')
-  textadept.io.open(filepath)
+  local utf8_filepath = table.concat(selected_item, '/')
+  textadept.io.open(utf8_filepath)
   view:focus()
 end
 
@@ -61,9 +63,10 @@ function get_context_menu(selected_item)
 end
 
 function perform_menu_action(menu_id, selected_item)
-  local filepath = table.concat(selected_item, '/'):gsub('[/\\]+', '/')
+  local utf8_filepath = table.concat(selected_item, '/'):gsub('[/\\]+', '/')
+  local filepath = textadept.iconv(utf8_filepath, _CHARSET, 'UTF-8')
   if menu_id == ID.CHANGE_DIR then
-    textadept.pm.entry_text = filepath
+    textadept.pm.entry_text = utf8_filepath
     textadept.pm.activate()
   elseif menu_id == ID.FILE_INFO then
     local date_format = '%D %T'
@@ -76,7 +79,7 @@ function perform_menu_action(menu_id, selected_item)
                     os.date(date_format, attr.change))
     cocoa_dialog('textbox', {
       ['informative-text'] =
-        string.format(locale.PM_BROWSER_FILE_INFO_TEXT, filepath),
+        string.format(locale.PM_BROWSER_FILE_INFO_TEXT, utf8_filepath),
       text = out,
       button1 = locale.PM_BROWSER_FILE_INFO_OK,
       editable = false
@@ -90,15 +93,18 @@ end
 -- load the dropped directory (if any) into the file browser; events.lua's
 -- "uri_dropped" handler already opens dropped files
 textadept.events.add_handler('uri_dropped',
-  function(uris)
-    for uri in uris:gmatch('[^\r\n\f]+') do
-      if uri:find('^file://') then
-        uri = uri:match('^file://([^\r\n\f]+)')
-        uri = uri:gsub('%%20', ' ') -- sub back for spaces
-        if WIN32 then uri = uri:sub(2, -1) end -- ignore leading '/'
+  function(utf8_uris)
+    local lfs = require 'lfs'
+    for utf8_uri in utf8_uris:gmatch('[^\r\n\f]+') do
+      if utf8_uri:find('^file://') then
+        utf8_uri = utf8_uri:match('^file://([^\r\n\f]+)')
+        utf8_uri = utf8_uri:gsub('%%(%x%x)',
+          function(hex) return string.char(tonumber(hex, 16)) end)
+        if WIN32 then utf8_uri = utf8_uri:sub(2, -1) end -- ignore leading '/'
+        local uri = textadept.iconv(utf8_uri, _CHARSET, 'UTF-8')
         if lfs.attributes(uri).mode == 'directory' then
-          textadept.pm.add_browser(uri)
-          textadept.pm.entry_text = uri
+          textadept.pm.add_browser(utf8_uri)
+          textadept.pm.entry_text = utf8_uri
           textadept.pm.activate()
         end
       end
diff --git a/core/ext/pm/modules_browser.lua b/core/ext/pm/modules_browser.lua
index 37e83592..4cbe715c 100644
--- a/core/ext/pm/modules_browser.lua
+++ b/core/ext/pm/modules_browser.lua
@@ -71,7 +71,9 @@ local keys = _G.keys
 if type(keys) == 'table' then
   keys.$1 = {
     al = {
-      m = { textadept.io.open, _HOME..'/modules/$1/init.lua' },
+      m = { textadept.io.open,
+            textadept.iconv(_HOME..'/modules/$1/init.lua',
+                            'UTF-8', _CHARSET) },
     },
   }
 end
@@ -82,20 +84,22 @@ function matches(entry_text)
 end
 
 local function modify_path(path)
-  path[1] = _HOME..'/modules'
+  path[1] = textadept.iconv(_HOME..'/modules', 'UTF-8', _CHARSET)
   return path
 end
 
 function get_contents_for(full_path)
   full_path = modify_path(full_path)
+  local iconv = textadept.iconv
   local dir = {}
-  local dirpath = table.concat(full_path, '/')
-  for name in lfs.dir(dirpath) do
-    if not name:find('^%.') then
-      dir[name] = { text = name }
-      if lfs.attributes(dirpath..'/'..name, 'mode') == 'directory' then
-        dir[name].parent = true
-        dir[name].pixbuf = 'gtk-directory'
+  local dirpath = iconv(table.concat(full_path, '/'), _CHARSET, 'UTF-8')
+  for filename in lfs.dir(dirpath) do
+    if not filename:find('^%.') then
+      local utf8_filename = iconv(filename, 'UTF-8', _CHARSET)
+      dir[utf8_filename] = { text = utf8_filename }
+      if lfs.attributes(dirpath..'/'..filename, 'mode') == 'directory' then
+        dir[utf8_filename].parent = true
+        dir[utf8_filename].pixbuf = 'gtk-directory'
       end
     end
   end
@@ -104,8 +108,8 @@ end
 
 function perform_action(selected_item)
   selected_item = modify_path(selected_item)
-  local filepath = table.concat(selected_item, '/')
-  textadept.io.open(filepath)
+  local utf8_filepath = table.concat(selected_item, '/')
+  textadept.io.open(utf8_filepath)
   view:focus()
 end
 
@@ -183,10 +187,12 @@ function perform_menu_action(menu_id, selected_item)
       return
     end
   elseif menu_id == ID.CONF_MIME_TYPES then
-    textadept.io.open(_HOME..'/core/ext/mime_types.lua')
+    textadept.io.open(
+      textadept.iconv(_HOME..'/core/ext/mime_types.lua', 'UTF-8', _CHARSET))
   elseif menu_id == ID.CONF_KEY_COMMANDS then
     if textadept.key_commands then
-      textadept.io.open(_HOME..'/core/ext/key_commands.lua')
+      textadept.io.open(
+        textadept.iconv(_HOME..'/core/ext/key_commands.lua', 'UTF-8', _CHARSET))
     end
   elseif menu_id == ID.RELOAD then
     textadept.reset()