fix: unescape unicode properly

Author: mdvorak

fixtures/test-all.properties

@@ -70,3 +70,5 @@ encodedHelloInJapanese = \u3053\u3093\u306b\u3061\u306f
 # Using \u without being followed by four hexadecimal digits will throw an exception.
 # But with more modern file encodings like UTF-8, you can directly use supported characters.
 helloInJapanese = こんにちは
+# Key can be also encoded
+\u3053\u3093\u306b\u3061\u306f = hello

src/properties.spec.ts

@@ -116,7 +116,6 @@ describe('data access', () => {
       expect(result).toBeUndefined()
     })
 
-
     it('should return last value of duplicate key', () => {
       const config: properties.Properties = {
         lines: [
@@ -129,6 +128,25 @@ describe('data access', () => {
       const result = properties.get(config, 'key1')
       expect(result).toBe('foo3')
     })
+
+    it('should throw on invalid unicode sequence in key', () => {
+      const config: properties.Properties = {
+        lines: ['foo\\u23a=bar']
+      }
+
+      expect(() => properties.get(config, 'foo')).toThrowError()
+    })
+
+    it.each([
+      ['foo=bar\\u23a'],
+      ['foo=bar\\u23ax5']
+    ])('should throw on invalid unicode sequence in value %s', (line) => {
+      const config: properties.Properties = {
+        lines: [line]
+      }
+
+      expect(() => properties.get(config, 'foo')).toThrowError()
+    })
   })
 
   describe('set value', () => {
@@ -364,6 +382,7 @@ describe('data access', () => {
         'evenLikeThis\\': '',
         hello: 'hello',
         helloInJapanese: 'こんにちは',
+        'こんにちは': 'hello',
         keyWithBackslashes: 'This has random backslashes',
         'keyWithDelimiters:= ': 'This is the value for the key "keyWithDelimiters:= "',
         'keyWitheven\\': 'this colon is not escaped',
@@ -383,19 +402,6 @@ describe('data access', () => {
   })
 })
 
-describe('unescape', () => {
-  it.each([
-    ['foo', 'foo'],
-    ['\\:\\#\\!\\ ', ':#! '],
-    ['a\\r\\f\\n\\t\\\\\\  ', 'a\r\f\n\t\\  '],
-    ['\\u0000\\u0001', '\0\u0001'],
-    ['\\u3053\\u3093\\u306b\\u3061\\u306f', 'こんにちは']
-  ])('should unescape string "%s" to "%s"', (str: string, expected: string) => {
-    const result = properties.unescape(str)
-    expect(result).toEqual(expected)
-  })
-})
-
 describe('escapeKey', () => {
   it.each([
     ['foo1', 'foo1'],

src/properties.ts

@@ -232,7 +232,8 @@ function* listPairs(lines: string[]): Generator<{
     sep: string,
     value: string,
     skipSpace: boolean,
-    escapedNext: boolean
+    escapedNext: boolean,
+    unicode?: string
   } =>
     ({state: State.START, start: -1, key: '', sep: '', value: '', skipSpace: true, escapedNext: false})
 
@@ -245,6 +246,20 @@ function* listPairs(lines: string[]): Generator<{
     }
     state.skipSpace = false
 
+    // Parse unicode
+    if (state.unicode) {
+      // Handle incomplete sequence
+      if (char === 'EOL') {
+        throw new Error(`Invalid unicode sequence at line ${line}`)
+      }
+
+      // Append and consume until it has correct length
+      state.unicode += char
+      if (state.unicode.length < 6) {
+        continue
+      }
+    }
+
     // First char on the line
     if (state.state === State.START) {
       switch (char) {
@@ -272,6 +287,13 @@ function* listPairs(lines: string[]): Generator<{
 
     // Key
     if (state.state === State.KEY) {
+      // Special unicode handling
+      if (state.unicode) {
+        state.key += parseUnicode(state.unicode, line)
+        state.unicode = undefined
+        continue
+      }
+
       switch (char) {
         case 'EOL':
           if (state.escapedNext) {
@@ -307,10 +329,20 @@ function* listPairs(lines: string[]): Generator<{
           }
           break
         default:
-          // Normal char
-          // TODO handle unicode
-          state.key += state.escapedNext ? unescapeChar(char) : char
-          state.escapedNext = false
+          // Escape sequence
+          if (state.escapedNext) {
+            state.escapedNext = false
+            if (char === 'u') {
+              // Unicode
+              state.unicode = '0x'
+            } else {
+              // Special char
+              state.key += unescapeChar(char)
+            }
+          } else {
+            // Normal char
+            state.key += char
+          }
           break
       }
     }
@@ -346,6 +378,13 @@ function* listPairs(lines: string[]): Generator<{
 
     // Value
     if (state.state === State.VALUE) {
+      // Special unicode handling
+      if (state.unicode) {
+        state.value += parseUnicode(state.unicode, line)
+        state.unicode = undefined
+        continue
+      }
+
       switch (char) {
         case 'EOL':
           if (state.escapedNext) {
@@ -369,10 +408,19 @@ function* listPairs(lines: string[]): Generator<{
           }
           break
         default:
-          // Normal char
-          // TODO handle unicode
-          state.value += state.escapedNext ? unescapeChar(char) : char
-          state.escapedNext = false
+          if (state.escapedNext) {
+            state.escapedNext = false
+            if (char === 'u') {
+              // Unicode
+              state.unicode = '0x'
+            } else {
+              // Special char
+              state.value += unescapeChar(char)
+            }
+          } else {
+            // Normal char
+            state.value += char
+          }
           break
       }
     }
@@ -395,14 +443,12 @@ const unescapeChar = (c: string): string => {
   }
 }
 
-/**
- * Unescape key or value.
- *
- * @param str Escaped string.
- * @return Actual string.
- */
-export const unescape = (str: string): string =>
-  str.replace(/\\(.)/g, s => unescapeChar(s[1]))
+const parseUnicode = (sequence: string, line: number): string => {
+  if (!sequence.match(/^0x[\da-fA-F]{4}$/)) {
+    throw new Error(`Invalid unicode sequence at line ${line}`)
+  }
+  return String.fromCharCode(parseInt(sequence, 16))
+}
 
 /**
  * Escape property key.