Implemented valid UTF8 character checks

src/common/utf8_op/src/utf8_op.c

@@ -15,21 +15,21 @@
 #define REPLACEMENT_INC 4096
 
 /* Single byte: 0xxxxxxx */
-#define valid_1(x) (x[0] & 0x80) == 0
+#define valid_1(x) (((x)[0] & 0x80) == 0)
 
 /* Two bytes: 110xxxxx 10xxxxxx */
 /* Starting bytes 0xC0 and 0xC1 are forbidden (overlong) */
-#define valid_2(x) (x[0] & 0xE0) == 0xC0 && (x[0] & 0x1E) != 0 && (x[1] & 0xC0) == 0x80
+#define valid_2(x) (((x)[0] & 0xE0) == 0xC0 && (x)[0] >= (char)0xC2 && ((x)[1] & 0xC0) == 0x80)
 
 /* Three bytes: 1110xxxx 10xxxxxx 10xxxxxx */
 /* 0xE0 could start overlong encodings */
 /* 0xED (range U+D800–U+DFFF) is reserved for UTF-16 surrogate halves */
-#define valid_3(x) (x[0] & 0xF0) == 0xE0 && x[0] != (char)0xE0 && x[0] != (char)0xED && (x[1] & 0xC0) == 0x80 && (x[2] & 0xC0) == 0x80
+#define valid_3(x) (((x)[0] & 0xF0) == 0xE0 && (((x)[0] != (char)0xE0 || ((x)[1] & 0xE0) != 0x80) && ((x)[0] != (char)0xED || ((x)[1] & 0xE0) != 0xA0)) && ((x)[1] & 0xC0) == 0x80 && ((x)[2] & 0xC0) == 0x80)
 
 /* Four bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
 /* 0xF0 could start overlong encodings */
 /* Starting bytes 111101xx are forbidden (Unicode limit) */
-#define valid_4(x) (x[0] & 0xF8) == 0xF0 && x[0] != (char)0xF0 && (x[0] & 0x04) == 0 && (x[1] & 0xC0) == 0x80 && (x[2] & 0xC0) == 0x80 && (x[3] & 0xC0) == 0x80
+#define valid_4(x) (((x)[0] & 0xF8) == 0xF0 && (((x)[0] != (char)0xF0 || ((x)[1] & 0xF0) != 0x80) && ((x)[0] != (char)0xF4 || ((x)[1] & 0xF0) == 0x80)) && ((x)[1] & 0xC0) == 0x80 && ((x)[2] & 0xC0) == 0x80 && ((x)[3] & 0xC0) == 0x80)
 
 /* Return whether a string is UTF-8 */
 bool w_utf8_valid(const char * string) {

src/common/utf8_op/tests/unit/tests/test_utf8_op.c

@@ -18,7 +18,57 @@
 #include "../../headers/shared.h"
 #include "../wrappers/common.h"
 
-// Tests
+// Utility function for verifying the result
+void assert_valid_utf8(const char *input, bool replacement, bool expect_valid) {
+ char *filtered = w_utf8_filter(input, replacement);
+ int result = w_utf8_valid(filtered);
+ if (expect_valid) {
+ assert_int_equal(result, 1);
+ } else {
+ assert_int_equal(result, 0);
+ }
+ free(filtered);
+}
+
+// Test valid UTF-8 sequences
+void test_valid_utf8_sequences(void **state)
+{
+ const char * valid_sequences[] = {
+ "Hello, World!", // ASCII characters (1-byte each)
+ "\xC3\x9C", // Ü (U+00DC, 2-byte UTF-8)
+ "\xC3\xBC", // ü (U+00FC, 2-byte UTF-8)
+ "\xE2\x98\x83", // ☃ (U+2603, 3-byte UTF-8)
+ "\xF0\x9F\x98\x81", // 😁 (U+1F601, 4-byte UTF-8)
+ "Σὲ γνωρίζω", // Greek text (multi-byte sequences)
+ "中文字符", // Chinese characters (3-byte UTF-8)
+ NULL // Null-terminated array
+ };
+
+ for (int i = 0; valid_sequences[i] != NULL; ++i) {
+ assert_valid_utf8(valid_sequences[i], false, true);
+ assert_valid_utf8(valid_sequences[i], true, true);
+ }
+}
+
+// Test invalid UTF-8 sequences
+void test_invalid_utf8_sequences(void **state)
+{
+ const char * invalid_sequences[] = {
+ "\xC0\xAF", // Overlong encoding of '/'
+ "\xE0\x80\xAF", // Overlong encoding (null character U+002F)
+ "\xED\xA0\x80", // UTF-16 surrogate half (invalid in UTF-8)
+ "\xF8\x88\x80\x80\x80", // 5-byte sequence (invalid, as UTF-8 only supports up to 4 bytes)
+ "\xFF", // Invalid single byte (not valid in UTF-8)
+ "\x80", // Continuation byte without a start
+ "\xC3\x28", // Invalid 2-byte sequence (invalid second byte)
+ NULL // Null-terminated array
+ };
+
+ for (int i = 0; invalid_sequences[i] != NULL; ++i) {
+ assert_valid_utf8(invalid_sequences[i], false, false);
+ assert_valid_utf8(invalid_sequences[i], true, true); // Replaced, thus valid output
+ }
+}
 
 void test_utf8_random_replace(void **state)
 {
@@ -38,6 +88,10 @@ void test_utf8_random_replace(void **state)
 
  char * copy = w_utf8_filter(buffer, true);
  int r = w_utf8_valid(copy);
+
+ /* Check if the output is valid */
+ assert_int_equal(r, 1);
+
  free(copy);
 }
 
@@ -50,7 +104,6 @@ void test_utf8_random_not_replace(void **state)
  randombytes(buffer, LENGTH - 1);
 
  /* Avoid zeroes */
-
  for (i = 0; i < LENGTH - 1; i++) {
  buffer[i] = buffer[i] ? buffer[i] : '0';
  }
@@ -59,13 +112,33 @@ void test_utf8_random_not_replace(void **state)
 
  char * copy = w_utf8_filter(buffer, false);
  int r = w_utf8_valid(copy);
+
+ /* The result could be either valid or invalid */
+ (void)r; // Use (void) to avoid unused variable warning in case you don't assert
+
  free(copy);
 }
 
+void test_utf8_edge_cases(void **state)
+{
+ const char * edge_cases[] = {
+ "\xF4\x8F\xBF\xBF", // U+10FFFF (highest valid UTF-8 character)
+ "\xF4\x90\x80\x80", // Beyond U+10FFFF (invalid)
+ NULL
+ };
+
+ // Check edge cases
+ assert_valid_utf8(edge_cases[0], false, true); // Should be valid
+ assert_valid_utf8(edge_cases[1], false, false); // Should be invalid
+}
+
 int main(void) {
  const struct CMUnitTest tests[] = {
- cmocka_unit_test(test_utf8_random_replace),
- cmocka_unit_test(test_utf8_random_not_replace),
+ cmocka_unit_test(test_valid_utf8_sequences),
+ cmocka_unit_test(test_invalid_utf8_sequences),
+ cmocka_unit_test(test_utf8_random_replace),
+ cmocka_unit_test(test_utf8_random_not_replace),
+ cmocka_unit_test(test_utf8_edge_cases),
  };
  return cmocka_run_group_tests(tests, NULL, NULL);
-}
+}