Ada 4.1.1
Fast spec-compliant URL parser
Loading...
Searching...
No Matches
to_unicode.cpp
Go to the documentation of this file.
2
3#include <algorithm>
4#include <string>
5
6#include "ada/idna/mapping.h"
8#include "ada/idna/punycode.h"
9#include "ada/idna/to_ascii.h"
11#include "ada/idna/validity.h"
12#include "idna.h"
13
14#ifdef ADA_USE_SIMDUTF
15#include "simdutf.h"
16#endif
17
18namespace ada::idna {
19
20// True when an all-ASCII label is valid on its own but violates RFC 5893
21// because the domain contains an RTL label (for example a label starting
22// with a digit). ToUnicode then returns the original input (UTS #46).
23static bool ascii_label_fails_bidi_domain(std::string_view label) {
24 std::u32string utf32(label.size(), 0);
25 for (size_t i = 0; i < label.size(); i++) {
26 utf32[i] = static_cast<unsigned char>(label[i]);
27 }
28 return check_label(utf32, false) && !check_label(utf32, true);
29}
30
31static bool utf8_domain_is_bidi(std::string_view input) {
32 std::u32string decoded;
33 size_t label_start = 0;
34 while (label_start < input.size()) {
35 const size_t loc_dot = input.find('.', label_start);
36 const bool is_last_label = (loc_dot == std::string_view::npos);
37 const size_t label_size =
38 is_last_label ? input.size() - label_start : loc_dot - label_start;
39 const std::string_view label = input.substr(label_start, label_size);
40 if (label.starts_with("xn--") && is_ascii(label)) {
41 decoded.clear();
42 if (label.size() > 4 && punycode_to_utf32(label.substr(4), decoded) &&
43 label_contains_rtl(decoded)) {
44 return true;
45 }
46 } else if (!is_ascii(label)) {
47 const size_t utf32_length =
48 utf32_length_from_utf8(label.data(), label.size());
49 if (utf32_length != 0) {
50 decoded.assign(utf32_length, 0);
51 const size_t written =
52 utf8_to_utf32(label.data(), label.size(), decoded.data());
53 if (written == utf32_length && label_contains_rtl(decoded)) {
54 return true;
55 }
56 }
57 }
58 if (is_last_label) {
59 break;
60 }
61 label_start += label_size + 1;
62 }
63 return false;
64}
65
66[[nodiscard]] bool to_unicode(std::string_view input, std::string& output) {
67 output.clear();
68 if (input.size() > max_domain_input_bytes) {
69 return false;
70 }
71 // An ASCII label that is only illegal because the domain is a Bidi domain
72 // fails ToUnicode for the whole input (UTS #46), not just that label.
73 const bool bidi_domain = utf8_domain_is_bidi(input);
74 if (bidi_domain) {
75 size_t label_start = 0;
76 while (label_start < input.size()) {
77 const size_t loc_dot = input.find('.', label_start);
78 const bool is_last_label = (loc_dot == std::string_view::npos);
79 const size_t label_size =
80 is_last_label ? input.size() - label_start : loc_dot - label_start;
81 const std::string_view label = input.substr(label_start, label_size);
82 if (!label.empty() && is_ascii(label) && !label.starts_with("xn--") &&
84 output.assign(input);
85 return true;
86 }
87 if (is_last_label) {
88 break;
89 }
90 label_start += label_size + 1;
91 }
92 }
93 output.reserve(input.size());
94
95 size_t label_start = 0;
96 std::u32string tmp_buffer;
97 std::u32string post_map;
98 while (label_start < input.size()) {
99 size_t loc_dot = input.find('.', label_start);
100 bool is_last_label = (loc_dot == std::string_view::npos);
101 size_t label_size =
102 is_last_label ? input.size() - label_start : loc_dot - label_start;
103 auto label_view = std::string_view(input.data() + label_start, label_size);
104
105 if (label_view.starts_with("xn--") && ada::idna::is_ascii(label_view)) {
106 label_view.remove_prefix(4);
107 tmp_buffer.clear();
108 if (ada::idna::punycode_to_utf32(label_view, tmp_buffer)) {
109 // Per UTS #46, the decoded label must be re-validated. Reject decodings
110 // that are pure ASCII (xn-- encoding of an ASCII-only label), or whose
111 // mapping/normalization is not stable, or that fail label validity.
112 bool accept_decoded = true;
113 if (ada::idna::is_ascii(tmp_buffer)) {
114 accept_decoded = false;
115 } else {
116 post_map.clear();
117 if (!ada::idna::map(tmp_buffer, post_map) || post_map != tmp_buffer) {
118 accept_decoded = false;
119 } else if (!ada::idna::normalize(post_map) ||
120 post_map != tmp_buffer || post_map.empty() ||
121 !check_label(post_map, bidi_domain)) {
122 accept_decoded = false;
123 }
124 }
125
126 if (accept_decoded) {
127#ifdef ADA_USE_SIMDUTF
128 auto utf8_size = simdutf::utf8_length_from_utf32(tmp_buffer.data(),
129 tmp_buffer.size());
130 size_t old_size = output.size();
131 output.resize(old_size + utf8_size);
132 size_t written = simdutf::convert_utf32_to_utf8(
133 tmp_buffer.data(), tmp_buffer.size(), output.data() + old_size);
134 if (written != utf8_size) {
135 output.resize(old_size);
136 output.append(
137 std::string_view(input.data() + label_start, label_size));
138 }
139#else
140 auto utf8_size = ada::idna::utf8_length_from_utf32(tmp_buffer.data(),
141 tmp_buffer.size());
142 size_t old_size = output.size();
143 output.resize(old_size + utf8_size);
144 size_t written = ada::idna::utf32_to_utf8(
145 tmp_buffer.data(), tmp_buffer.size(), output.data() + old_size);
146 if (written != utf8_size) {
147 output.resize(old_size);
148 output.append(
149 std::string_view(input.data() + label_start, label_size));
150 }
151#endif
152 } else {
153 // ToUnicode never fails. If any step fails, return the original
154 // input sequence for the label.
155 output.append(
156 std::string_view(input.data() + label_start, label_size));
157 }
158 } else {
159 output.append(std::string_view(input.data() + label_start, label_size));
160 }
161 } else {
162 output.append(label_view);
163 }
164
165 if (!is_last_label) {
166 output.push_back('.');
167 }
168
169 label_start += label_size + 1;
170 }
171
172 return true;
173}
174
175std::string to_unicode(std::string_view input) {
176 std::string output;
177 if (!to_unicode(input, output)) {
178 // Over-limit input: return original (ToUnicode never fails in the sense of
179 // producing no answer for valid-length inputs).
180 return std::string(input);
181 }
182 return output;
183}
184} // namespace ada::idna
static bool ascii_label_fails_bidi_domain(std::string_view label)
bool punycode_to_utf32(std::string_view input, std::u32string &out)
size_t utf32_length_from_utf8(const char *buf, size_t len)
Definition ada_idna.cpp:142
size_t utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_output)
Definition ada_idna.cpp:151
static bool check_label(const std::u32string_view label, bool bidi_domain)
static bool utf8_domain_is_bidi(std::string_view input)
bool constexpr is_ascii(std::u32string_view view)
std::string to_unicode(std::string_view input)
bool normalize(std::u32string &input)
size_t utf8_length_from_utf32(const char32_t *buf, size_t len)
Definition ada_idna.cpp:129
std::u32string map(std::u32string_view input)
size_t utf8_to_utf32(const char *buf, size_t len, char32_t *utf32_output)
Definition ada_idna.cpp:32
constexpr size_t max_domain_input_bytes
Definition limits.h:12
static bool label_contains_rtl(std::u32string_view label) noexcept