From 4bc9faf8f8928bddc02a28897f5e51bdd58a8132 Mon Sep 17 00:00:00 2001
From: mitchell <70453897+667e-11@users.noreply.github.com>
Date: Wed, 19 Oct 2016 20:39:04 -0400
Subject: Allow detection of encodings with NUL bytes like UTF-16;
 core/file_io.lua

---
 core/file_io.lua | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

(limited to 'core/file_io.lua')

diff --git a/core/file_io.lua b/core/file_io.lua
index e79da353..cf364305 100644
--- a/core/file_io.lua
+++ b/core/file_io.lua
@@ -120,15 +120,19 @@ function io.open_file(filenames)
     end
     local buffer = buffer.new()
     -- Try to detect character encoding and convert to UTF-8.
-    if not text:sub(1, 65536):find('\0') then
-      for j = 1, #io.encodings do
+    local has_zeroes = text:sub(1, 65536):find('\0')
+    for j = 1, #io.encodings do
+      if not has_zeroes or io.encodings[j]:find('^UTF%-[13][62]') then
         local ok, conv = pcall(string.iconv, text, 'UTF-8', io.encodings[j])
-        if ok then buffer.encoding, text = io.encodings[j], conv break end
+        if ok then
+          buffer.encoding, text = io.encodings[j], conv
+          goto encoding_detected
+        end
       end
-      assert(buffer.encoding, _L['Encoding conversion failed.'])
-    else
-      buffer.encoding = nil -- binary (default was 'UTF-8')
     end
+    assert(has_zeroes, _L['Encoding conversion failed.'])
+    buffer.encoding = nil -- binary (default was 'UTF-8')
+    ::encoding_detected::
     buffer.code_page = buffer.encoding and buffer.CP_UTF8 or 0
     -- Detect EOL mode.
     buffer.eol_mode = text:find('\r\n') and buffer.EOL_CRLF or buffer.EOL_LF
-- 
cgit v1.2.3