RetroLinker
Linker for several 8-bit, 16-bit and 32-bit formats
Loading...
Searching...
No Matches
unicode.cc
1
2#include <cppunit/extensions/HelperMacros.h>
3#include <cppunit/TestFixture.h>
4
5#include "../src/unicode.h"
6
7namespace UnitTests
8{
9
10class TestUnicode : public CppUnit::TestFixture
11{
12 CPPUNIT_TEST_SUITE(TestUnicode);
13 CPPUNIT_TEST(testUTF8ToUTF32);
14 CPPUNIT_TEST(testUTF32ToUTF8);
15 CPPUNIT_TEST(testUTF16ToUTF32);
16 CPPUNIT_TEST(testUTF32ToUTF16);
17 CPPUNIT_TEST_SUITE_END();
18private:
19
20 void testUTF8ToUTF32();
21 void testUTF32ToUTF8();
22 void testUTF16ToUTF32();
23 void testUTF32ToUTF16();
24};
25
26static const char initial_bytes[8] = { 0, 0, '\xC0', '\xE0', '\xF0', '\xF8', '\xFC', '\xFE' };
27
28void TestUnicode::testUTF8ToUTF32()
29{
30 UnicodeConversion::Result result;
31 char const * input_pointer;
32 char32_t * output_pointer;
33
34 // check single byte input sequences
35
36 for(uint16_t i = 0; i < 0x100; i++)
37 {
38 char input_buffer[1];
39 char32_t output_buffer[1];
40
41 input_buffer[0] = i;
42
43 // strict
44 input_pointer = input_buffer;
45 output_pointer = output_buffer;
46
47 result = UTF8ToUTF32(input_pointer, output_pointer, 1, false);
48
49 if(i == 0)
50 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::NullCharacter, result);
51 else if(i < 0x80)
52 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
53 else if(i < 0xC0)
54 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InvalidValue, result);
55 else if(i < 0xF5)
56 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InputTruncated, result);
57 else
58 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InvalidValue, result);
59
60 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(UnicodeConversion::InputAdvanced(result) ? 1 : 0), const_cast<char *>(input_pointer) - input_buffer);
61 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(UnicodeConversion::InputAdvanced(result) ? 1 : 0), output_pointer - output_buffer);
62
63 if(UnicodeConversion::InputAdvanced(result) && i < 0x80)
64 {
65 CPPUNIT_ASSERT_EQUAL(uint32_t(i), uint32_t(output_buffer[0]));
66 }
67
68 // permissive
69 input_pointer = input_buffer;
70 output_pointer = output_buffer;
71
72 result = UTF8ToUTF32(input_pointer, output_pointer, 1, true);
73
74 if(i == 0)
75 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::NullCharacter, result);
76 else if(i < 0x80)
77 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
78 else if(i < 0xC0)
79 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InvalidValue, result);
80 else if(i < 0xFF)
81 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InputTruncated, result);
82 else
83 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InvalidValue, result);
84
85 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(UnicodeConversion::InputAdvanced(result) ? 1 : 0), const_cast<char *>(input_pointer) - input_buffer);
86 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(UnicodeConversion::InputAdvanced(result) ? 1 : 0), output_pointer - output_buffer);
87
88 if(UnicodeConversion::InputAdvanced(result) && i < 0x80)
89 {
90 CPPUNIT_ASSERT_EQUAL(uint32_t(i), uint32_t(output_buffer[0]));
91 }
92 }
93
94 for(size_t sequence_length = 2; sequence_length < 8; sequence_length++)
95 {
96 char input_buffer[7];
97 char32_t output_buffer[1];
98
99 for(int32_t bit_position = -1; uint32_t(bit_position + 1) <= 1 + sequence_length * 5 && uint32_t(bit_position + 1) <= 32; bit_position ++)
100 {
101 input_buffer[0] = initial_bytes[sequence_length];
102 for(size_t i = 1; i < sequence_length; i++)
103 {
104 input_buffer[i] = 0x80;
105 }
106
107 if(bit_position >= 0)
108 input_buffer[sequence_length - 1 - bit_position / 6] |= 1 << (bit_position % 6);
109
110 // permissive
111 input_pointer = input_buffer;
112 output_pointer = output_buffer;
113
114 result = UTF8ToUTF32(input_pointer, output_pointer, sequence_length, true);
115 if(bit_position == -1)
116 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::NullCharacter, result);
117 else if(bit_position < 21)
118 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
119 else
120 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::OutOfRangeCharacter, result);
121
122 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(sequence_length), const_cast<char *>(input_pointer) - input_buffer);
123 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), output_pointer - output_buffer);
124
125 if(bit_position < 0)
126 CPPUNIT_ASSERT_EQUAL(uint32_t(0), uint32_t(output_buffer[0]));
127 else
128 CPPUNIT_ASSERT_EQUAL(uint32_t(1 << bit_position), uint32_t(output_buffer[0]));
129
130 // strict
131 input_pointer = input_buffer;
132 output_pointer = output_buffer;
133
134 result = UTF8ToUTF32(input_pointer, output_pointer, sequence_length, false);
135 if(sequence_length >= 5)
136 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InvalidValue, result);
137 else if(bit_position == -1)
138 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::NullCharacter, result);
139 else
140 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
141
142 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(UnicodeConversion::IsSuccess(result) ? sequence_length : 1), const_cast<char *>(input_pointer) - input_buffer);
143 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), output_pointer - output_buffer);
144
145 if(UnicodeConversion::IsSuccess(result))
146 {
147 if(bit_position < 0)
148 CPPUNIT_ASSERT_EQUAL(uint32_t(0), uint32_t(output_buffer[0]));
149 else
150 CPPUNIT_ASSERT_EQUAL(uint32_t(1 << bit_position), uint32_t(output_buffer[0]));
151 }
152 else
153 {
154 CPPUNIT_ASSERT_EQUAL(uint32_t(input_buffer[0] & 0xFF), uint32_t(output_buffer[0]));
155 }
156 }
157 }
158}
159
160void TestUnicode::testUTF32ToUTF8()
161{
162 UnicodeConversion::Result result;
163 char32_t const * input_pointer;
164 char * output_pointer;
165 size_t required_output;
166
167 for(int32_t bit_position = -1; bit_position < 32; bit_position ++)
168 {
169 char32_t input_buffer[1];
170 char output_buffer[7];
171
172 size_t actual_required_output = bit_position <= 6 ? 1 : (bit_position + 4) / 5;
173 if(bit_position < 0)
174 input_buffer[0] = 0;
175 else
176 input_buffer[0] = 1 << bit_position;
177
178 input_pointer = input_buffer;
179 output_pointer = output_buffer;
180 required_output = size_t(-1);
181
182 result = UTF32ToUTF8(input_pointer, output_pointer, actual_required_output, required_output);
183 if(bit_position == -1)
184 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::NullCharacter, result);
185 else if(bit_position < 21)
186 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
187 else
188 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::OutOfRangeCharacter, result);
189
190 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), const_cast<char32_t *>(input_pointer) - input_buffer);
191 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(actual_required_output), output_pointer - output_buffer);
192 CPPUNIT_ASSERT_EQUAL(actual_required_output, required_output);
193
194 if(actual_required_output > 1)
195 {
196 input_pointer = input_buffer;
197 output_pointer = output_buffer;
198 required_output = size_t(-1);
199
200 result = UTF32ToUTF8(input_pointer, output_pointer, actual_required_output - 1, required_output);
201 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::BufferOverflow, result);
202
203 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(0), const_cast<char32_t *>(input_pointer) - input_buffer);
204 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(0), output_pointer - output_buffer);
205 CPPUNIT_ASSERT_EQUAL(actual_required_output, required_output);
206 }
207 }
208}
209
210void TestUnicode::testUTF16ToUTF32()
211{
212 UnicodeConversion::Result result;
213 char16_t const * input_pointer;
214 char32_t * output_pointer;
215
216 for(uint16_t i = 0; i < 256; i++)
217 {
218 char16_t input_buffer[1];
219 char32_t output_buffer[1];
220
221 input_buffer[0] = i;
222
223 input_pointer = input_buffer;
224 output_pointer = output_buffer;
225
226 result = UTF16ToUTF32(input_pointer, output_pointer, 1);
227
228 if(i == 0)
229 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::NullCharacter, result);
230 else
231 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
232
233 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), const_cast<char16_t *>(input_pointer) - input_buffer);
234 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), output_pointer - output_buffer);
235 CPPUNIT_ASSERT_EQUAL(uint32_t(i), uint32_t(output_buffer[0]));
236
237
238 input_buffer[0] = i << 8;
239
240 input_pointer = input_buffer;
241 output_pointer = output_buffer;
242
243 result = UTF16ToUTF32(input_pointer, output_pointer, 1);
244
245 if(0xD8 <= i && i <= 0xDB)
246 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InputTruncated, result);
247 else if(0xDC <= i && i <= 0xDF)
248 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InvalidValue, result);
249 else if(i == 0)
250 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::NullCharacter, result);
251 else
252 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
253
254 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), const_cast<char16_t *>(input_pointer) - input_buffer);
255 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), output_pointer - output_buffer);
256 if(!(0xD8 <= i && i <= 0xDF))
257 CPPUNIT_ASSERT_EQUAL(uint32_t(i << 8), uint32_t(output_buffer[0]));
258 }
259
260 for(int32_t bit_position = -1; bit_position < 20; bit_position ++)
261 {
262 char16_t input_buffer[2];
263 char32_t output_buffer[1];
264
265 input_buffer[0] = 0xD800;
266 input_buffer[1] = 0xDC00;
267
268 if(bit_position >= 0)
269 input_buffer[1 - bit_position / 10] |= 1 << (bit_position % 10);
270
271 input_pointer = input_buffer;
272 output_pointer = output_buffer;
273
274 result = UTF16ToUTF32(input_pointer, output_pointer, 2);
275
276 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
277 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(2), const_cast<char16_t *>(input_pointer) - input_buffer);
278 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), output_pointer - output_buffer);
279 CPPUNIT_ASSERT_EQUAL(uint32_t(0x00100000 | (bit_position >= 0 ? 1 << bit_position : 0)), uint32_t(output_buffer[0]));
280
281
282 input_pointer = input_buffer;
283 output_pointer = output_buffer;
284
285 result = UTF16ToUTF32(input_pointer, output_pointer, 1);
286
287 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InputTruncated, result);
288 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), const_cast<char16_t *>(input_pointer) - input_buffer);
289 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), output_pointer - output_buffer);
290 CPPUNIT_ASSERT_EQUAL(uint32_t(0x00100000 | (bit_position >= 10 ? 1 << bit_position : 0)), uint32_t(output_buffer[0]));
291 }
292}
293
294void TestUnicode::testUTF32ToUTF16()
295{
296 UnicodeConversion::Result result;
297 char32_t const * input_pointer;
298 char16_t * output_pointer;
299 size_t required_output;
300
301 for(uint16_t i = 0; i < 256; i++)
302 {
303 char32_t input_buffer[1];
304 char16_t output_buffer[1];
305
306 input_buffer[0] = i;
307
308 input_pointer = input_buffer;
309 output_pointer = output_buffer;
310 required_output = size_t(-1);
311
312 result = UTF32ToUTF16(input_pointer, output_pointer, 1, required_output);
313
314 if(i == 0)
315 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::NullCharacter, result);
316 else
317 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
318
319 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), const_cast<char32_t *>(input_pointer) - input_buffer);
320 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), output_pointer - output_buffer);
321 CPPUNIT_ASSERT_EQUAL(uint32_t(i), uint32_t(output_buffer[0]));
322
323
324 input_buffer[0] = i << 8;
325
326 input_pointer = input_buffer;
327 output_pointer = output_buffer;
328 required_output = size_t(-1);
329
330 result = UTF32ToUTF16(input_pointer, output_pointer, 1, required_output);
331
332 if(0xD8 <= i && i <= 0xDF)
333 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::InvalidCharacter, result);
334 else if(i == 0)
335 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::NullCharacter, result);
336 else
337 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
338
339 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), const_cast<char32_t *>(input_pointer) - input_buffer);
340 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), output_pointer - output_buffer);
341 if(!(0xD8 <= i && i <= 0xDF))
342 CPPUNIT_ASSERT_EQUAL(uint32_t(i << 8), uint32_t(output_buffer[0]));
343 }
344
345 for(int32_t bit_position = -1; bit_position < 16; bit_position ++)
346 {
347 char32_t input_buffer[1];
348 char16_t output_buffer[2];
349
350 input_buffer[0] = 0x00100000;
351 if(bit_position >= 0)
352 input_buffer[0] |= 1 << bit_position;
353
354 input_pointer = input_buffer;
355 output_pointer = output_buffer;
356 required_output = size_t(-1);
357
358 result = UTF32ToUTF16(input_pointer, output_pointer, 2, required_output);
359 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::Success, result);
360
361 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), const_cast<char32_t *>(input_pointer) - input_buffer);
362 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(2), output_pointer - output_buffer);
363 CPPUNIT_ASSERT_EQUAL(size_t(2), required_output);
364
365
366 input_pointer = input_buffer;
367 output_pointer = output_buffer;
368 required_output = size_t(-1);
369
370 result = UTF32ToUTF16(input_pointer, output_pointer, 1, required_output);
371 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::BufferOverflow, result);
372
373 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(0), const_cast<char32_t *>(input_pointer) - input_buffer);
374 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(0), output_pointer - output_buffer);
375 }
376
377 for(int32_t bit_position = 16; bit_position < 32; bit_position ++)
378 {
379 char32_t input_buffer[1];
380 char16_t output_buffer[2];
381
382 input_buffer[0] = 0x00100000 + (1 << bit_position);
383
384 input_pointer = input_buffer;
385 output_pointer = output_buffer;
386 required_output = size_t(-1);
387
388 result = UTF32ToUTF16(input_pointer, output_pointer, 2, required_output);
389 CPPUNIT_ASSERT_EQUAL(UnicodeConversion::OutOfRangeCharacter, result);
390
391 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(1), const_cast<char32_t *>(input_pointer) - input_buffer);
392 CPPUNIT_ASSERT_EQUAL(ptrdiff_t(0), output_pointer - output_buffer);
393 }
394}
395
396}
397
398
UnitTests::TestUnicode
Definition unicode.cc:11
Generated by doxygen 1.10.0