48#if ADA_UNICODE_SSSE3 || ADA_NEON
50#ifdef ADA_REGULAR_VISUAL_STUDIO
52 _BitScanForward(&ret, input_num);
53 return static_cast<int>(ret);
55 return __builtin_ctz(input_num);
61 size_t width, std::string& out) {
65 const int zero_run = trailing_zeroes32(
static_cast<uint32_t
>(bits));
67 out.append(p + off,
static_cast<size_t>(zero_run));
69 off +=
static_cast<size_t>(zero_run);
72 bits >>=
static_cast<unsigned>(zero_run + 1);
75 out.append(p + off, width - off);
81 const uint8_t character_set[],
83 for (; p != end; ++p) {
96struct ssse3_percent_tables {
106load_ssse3_percent_tables(
const uint8_t character_set[])
noexcept {
107 ssse3_percent_tables t{};
108 t.cs_lo = _mm_loadu_si128(
reinterpret_cast<const __m128i*
>(character_set));
110 _mm_loadu_si128(
reinterpret_cast<const __m128i*
>(character_set + 16));
113 _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128);
114 t.mask_0f = _mm_set1_epi8(0x0F);
115 t.mask_07 = _mm_set1_epi8(0x07);
116 t.zero = _mm_setzero_si128();
121 __m128i word,
const ssse3_percent_tables& tables)
noexcept {
122 const __m128i idx = _mm_and_si128(_mm_srli_epi16(word, 3), tables.mask_0f);
123 const __m128i lo = _mm_shuffle_epi8(tables.cs_lo, idx);
124 const __m128i hi = _mm_shuffle_epi8(tables.cs_hi, idx);
126 const __m128i high_byte = _mm_cmpgt_epi8(tables.zero, word);
127 const __m128i cs_byte = _mm_or_si128(_mm_and_si128(hi, high_byte),
128 _mm_andnot_si128(high_byte, lo));
130 _mm_shuffle_epi8(tables.pow2, _mm_and_si128(word, tables.mask_07));
131 const __m128i hits = _mm_and_si128(cs_byte, bits);
132 return _mm_movemask_epi8(_mm_cmpeq_epi8(hits, tables.zero)) ^ 0xFFFF;
136 const char* p,
const char* end,
const uint8_t character_set[],
137 const ssse3_percent_tables& tables, std::string& out) {
139 while (p + 32 <= end) {
140 const __m128i word0 = _mm_loadu_si128(
reinterpret_cast<const __m128i*
>(p));
141 const __m128i word1 =
142 _mm_loadu_si128(
reinterpret_cast<const __m128i*
>(p + 16));
143 const int mask0 = ssse3_percent_mask(word0, tables);
144 const int mask1 = ssse3_percent_mask(word1, tables);
145 if ((mask0 | mask1) == 0) {
151 encode_mask_window(p,
static_cast<uint32_t
>(mask0), 16, out);
154 out.append(p + 16, 16);
156 encode_mask_window(p + 16,
static_cast<uint32_t
>(mask1), 16, out);
162 const __m128i word = _mm_loadu_si128(
reinterpret_cast<const __m128i*
>(p));
163 const int mask = ssse3_percent_mask(word, tables);
167 encode_mask_window(p,
static_cast<uint32_t
>(mask), 16, out);
177load_neon_percent_table(
const uint8_t character_set[])
noexcept {
178 uint8x16x2_t table{};
179 table.val[0] = vld1q_u8(character_set);
180 table.val[1] = vld1q_u8(character_set + 16);
185 uint8x16x2_t table)
noexcept {
186 const uint8x16_t cs_bytes = vqtbl2q_u8(table, vshrq_n_u8(word, 3));
187 const uint8x16_t bit_mask = vshlq_u8(
188 vdupq_n_u8(1), vreinterpretq_s8_u8(vandq_u8(word, vdupq_n_u8(7))));
189 return vandq_u8(cs_bytes, bit_mask);
193 const uint8x16_t cmp = vcgtq_u8(hits, vdupq_n_u8(0));
194 const uint8x16_t bit = {1, 2, 4, 8, 16, 32, 64, 128,
195 1, 2, 4, 8, 16, 32, 64, 128};
196 const uint8x16_t masked = vandq_u8(cmp, bit);
197 return static_cast<uint32_t
>(vaddv_u8(vget_low_u8(masked))) |
198 (
static_cast<uint32_t
>(vaddv_u8(vget_high_u8(masked))) << 8);
202 const uint8_t character_set[],
205 while (p + 32 <= end) {
206 const uint8x16_t hits0 =
207 neon_percent_hits(vld1q_u8(
reinterpret_cast<const uint8_t*
>(p)), table);
208 const uint8x16_t hits1 = neon_percent_hits(
209 vld1q_u8(
reinterpret_cast<const uint8_t*
>(p + 16)), table);
210 const bool clean0 = vmaxvq_u32(vreinterpretq_u32_u8(hits0)) == 0;
211 const bool clean1 = vmaxvq_u32(vreinterpretq_u32_u8(hits1)) == 0;
212 if (clean0 && clean1) {
218 encode_mask_window(p, neon_percent_mask(hits0), 16, out);
221 out.append(p + 16, 16);
223 encode_mask_window(p + 16, neon_percent_mask(hits1), 16, out);
229 const uint8x16_t hits =
230 neon_percent_hits(vld1q_u8(
reinterpret_cast<const uint8_t*
>(p)), table);
231 if (vmaxvq_u32(vreinterpretq_u32_u8(hits)) == 0) {
234 encode_mask_window(p, neon_percent_mask(hits), 16, out);
244 const uint8_t character_set[],
247 const size_t remaining =
static_cast<size_t>(end - p);
248 const size_t vl = __riscv_vsetvl_e8m1(remaining);
249 const vuint8m1_t word =
250 __riscv_vle8_v_u8m1(
reinterpret_cast<const uint8_t*
>(p), vl);
251 const vuint8m1_t cs_bytes =
252 __riscv_vluxei8(character_set, __riscv_vsrl(word, 3, vl), vl);
253 const vuint8m1_t bit_mask = __riscv_vsll(__riscv_vmv_v_x_u8m1(1, vl),
254 __riscv_vand(word, 7, vl), vl);
255 const long idx = __riscv_vfirst(
256 __riscv_vmsne(__riscv_vand(cs_bytes, bit_mask, vl), 0, vl), vl);
263 out.append(p,
static_cast<size_t>(idx));
272#if ADA_UNICODE_SSSE3 || ADA_NEON || ADA_RVV
276static constexpr size_t kPercentEncodeSimdMin = 48;
278void percent_encode_to_wide(
const char* p,
const char* end,
279 const uint8_t character_set[], std::string& out) {
281 out.reserve(out.size() +
static_cast<size_t>(end - p) * 3);
283 const ssse3_percent_tables tables = load_ssse3_percent_tables(character_set);
284 percent_encode_to_ssse3(p, end, character_set, tables, out);
286 percent_encode_to_neon(p, end, character_set,
287 load_neon_percent_table(character_set), out);
289 percent_encode_to_rvv(p, end, character_set, out);