initialize repository

utf8.c

@@ -0,0 +1,59 @@
+#include "utf8.h"
+
+// This macro tests if a char is a continuation byte in utf8.
+#define IS_CONT(x) (((x) & 0xc0) == 0x80)
+
+// This returns the code point encoded at **s and advances *s to point to the
+// next character. Thus it can easily be used in a loop.
+int decode_code_point(const char **s) {
+  int k = **s ? __builtin_clz(~(**s << 24)) : 0; // Count # of leading 1 bits.
+  int mask = (1 << (8 - k)) - 1;                 // All 1s with k leading 0s.
+  int value = **s & mask;
+  // k = 0 for one-byte code points; otherwise, k = #total bytes.
+  for (++(*s), --k; k > 0 && IS_CONT(**s); --k, ++(*s)) {
+    value <<= 6;
+    value += (**s & 0x3F);
+  }
+  return value;
+}
+
+// This assumes that `code` is <= 0x10FFFF and ensures that nothing will be
+// written at or beyond `end`. It advances *s so it's easy to use in a loop.
+void encode_code_point(char **s, char *end, int code) {
+  char val[4];
+  int lead_byte_max = 0x7F;
+  int val_index = 0;
+  while (code > lead_byte_max) {
+    val[val_index++] = (code & 0x3F) | 0x80;
+    code >>= 6;
+    lead_byte_max >>= (val_index == 1 ? 2 : 1);
+  }
+  val[val_index++] = (code & lead_byte_max) | (~lead_byte_max << 1);
+  while (val_index-- && *s < end) {
+    **s = val[val_index];
+    (*s)++;
+  }
+}
+
+// This returns 0 if no split was needed.
+int split_into_surrogates(int code, int *surr1, int *surr2) {
+  if (code <= 0xFFFF)
+    return 0;
+  *surr2 = 0xDC00 | (code & 0x3FF); // Save the low 10 bits.
+  code >>= 10;                      // Drop the low 10 bits.
+  // If `code` now has low bits "uuu uuxx xxxx", then the bits of *surr are
+  // "1101 10ww wwxx xxxx" where wwww = (uuuuu - 1).
+  *surr1 = 0xD800 | ((code & 0x7FF) - 0x40);
+  return 1;
+}
+
+// This expects to be used in a loop and see all code points in *code. Start
+// *old at 0; this function updates *old for you - don't change it after
+// initialization. This returns 0 when *code is the 1st of a surrogate pair;
+// otherwise use *code as the final code point.
+int join_from_surrogates(int *old, int *code) {
+  if (*old)
+    *code = (((*old & 0x3FF) + 0x40) << 10) + (*code & 0x3FF);
+  *old = ((*code & 0xD800) == 0xD800 ? *code : 0);
+  return !(*old);
+}