OpenTTD Source  12.0-beta2
string.cpp
Go to the documentation of this file.
1 /*
2  * This file is part of OpenTTD.
3  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
6  */
7 
10 #include "stdafx.h"
11 #include "debug.h"
12 #include "core/alloc_func.hpp"
13 #include "core/math_func.hpp"
14 #include "string_func.h"
15 #include "string_base.h"
16 
17 #include "table/control_codes.h"
18 
19 #include <stdarg.h>
20 #include <ctype.h> /* required for tolower() */
21 #include <sstream>
22 
23 #ifdef _MSC_VER
24 #include <errno.h> // required by vsnprintf implementation for MSVC
25 #endif
26 
27 #ifdef _WIN32
28 #include "os/windows/win32.h"
29 #endif
30 
31 #ifdef WITH_UNISCRIBE
33 #endif
34 
35 #ifdef WITH_ICU_I18N
36 /* Required by strnatcmp. */
37 #include <unicode/ustring.h>
38 #include "language.h"
39 #include "gfx_func.h"
40 #endif /* WITH_ICU_I18N */
41 
42 #if defined(WITH_COCOA)
43 #include "os/macosx/string_osx.h"
44 #endif
45 
46 /* The function vsnprintf is used internally to perform the required formatting
47  * tasks. As such this one must be allowed, and makes sure it's terminated. */
48 #include "safeguards.h"
49 #undef vsnprintf
50 
61 int CDECL vseprintf(char *str, const char *last, const char *format, va_list ap)
62 {
63  ptrdiff_t diff = last - str;
64  if (diff < 0) return 0;
65  return std::min(static_cast<int>(diff), vsnprintf(str, diff + 1, format, ap));
66 }
67 
84 char *strecat(char *dst, const char *src, const char *last)
85 {
86  assert(dst <= last);
87  while (*dst != '\0') {
88  if (dst == last) return dst;
89  dst++;
90  }
91 
92  return strecpy(dst, src, last);
93 }
94 
95 
112 char *strecpy(char *dst, const char *src, const char *last)
113 {
114  assert(dst <= last);
115  while (dst != last && *src != '\0') {
116  *dst++ = *src++;
117  }
118  *dst = '\0';
119 
120  if (dst == last && *src != '\0') {
121 #if defined(STRGEN) || defined(SETTINGSGEN)
122  error("String too long for destination buffer");
123 #else /* STRGEN || SETTINGSGEN */
124  Debug(misc, 0, "String too long for destination buffer");
125 #endif /* STRGEN || SETTINGSGEN */
126  }
127  return dst;
128 }
129 
137 char *stredup(const char *s, const char *last)
138 {
139  size_t len = last == nullptr ? strlen(s) : ttd_strnlen(s, last - s + 1);
140  char *tmp = CallocT<char>(len + 1);
141  memcpy(tmp, s, len);
142  return tmp;
143 }
144 
150 char *CDECL str_fmt(const char *str, ...)
151 {
152  char buf[4096];
153  va_list va;
154 
155  va_start(va, str);
156  int len = vseprintf(buf, lastof(buf), str, va);
157  va_end(va);
158  char *p = MallocT<char>(len + 1);
159  memcpy(p, buf, len + 1);
160  return p;
161 }
162 
169 void str_fix_scc_encoded(char *str, const char *last)
170 {
171  while (str <= last && *str != '\0') {
172  size_t len = Utf8EncodedCharLen(*str);
173  if ((len == 0 && str + 4 > last) || str + len > last) break;
174 
175  WChar c;
176  Utf8Decode(&c, str);
177  if (c == '\0') break;
178 
179  if (c == 0xE028 || c == 0xE02A) {
180  c = SCC_ENCODED;
181  }
182  str += Utf8Encode(str, c);
183  }
184  *str = '\0';
185 }
186 
187 
188 template <class T>
189 static void StrMakeValidInPlace(T &dst, const char *str, const char *last, StringValidationSettings settings)
190 {
191  /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
192 
193  while (str <= last && *str != '\0') {
194  size_t len = Utf8EncodedCharLen(*str);
195  WChar c;
196  /* If the first byte does not look like the first byte of an encoded
197  * character, i.e. encoded length is 0, then this byte is definitely bad
198  * and it should be skipped.
199  * When the first byte looks like the first byte of an encoded character,
200  * then the remaining bytes in the string are checked whether the whole
201  * encoded character can be there. If that is not the case, this byte is
202  * skipped.
203  * Finally we attempt to decode the encoded character, which does certain
204  * extra validations to see whether the correct number of bytes were used
205  * to encode the character. If that is not the case, the byte is probably
206  * invalid and it is skipped. We could emit a question mark, but then the
207  * logic below cannot just copy bytes, it would need to re-encode the
208  * decoded characters as the length in bytes may have changed.
209  *
210  * The goals here is to get as much valid Utf8 encoded characters from the
211  * source string to the destination string.
212  *
213  * Note: a multi-byte encoded termination ('\0') will trigger the encoded
214  * char length and the decoded length to differ, so it will be ignored as
215  * invalid character data. If it were to reach the termination, then we
216  * would also reach the "last" byte of the string and a normal '\0'
217  * termination will be placed after it.
218  */
219  if (len == 0 || str + len > last || len != Utf8Decode(&c, str)) {
220  /* Maybe the next byte is still a valid character? */
221  str++;
222  continue;
223  }
224 
225  if ((IsPrintable(c) && (c < SCC_SPRITE_START || c > SCC_SPRITE_END)) || ((settings & SVS_ALLOW_CONTROL_CODE) != 0 && c == SCC_ENCODED)) {
226  /* Copy the character back. Even if dst is current the same as str
227  * (i.e. no characters have been changed) this is quicker than
228  * moving the pointers ahead by len */
229  do {
230  *dst++ = *str++;
231  } while (--len != 0);
232  } else if ((settings & SVS_ALLOW_NEWLINE) != 0 && c == '\n') {
233  *dst++ = *str++;
234  } else {
235  if ((settings & SVS_ALLOW_NEWLINE) != 0 && c == '\r' && str[1] == '\n') {
236  str += len;
237  continue;
238  }
239  /* Replace the undesirable character with a question mark */
240  str += len;
241  if ((settings & SVS_REPLACE_WITH_QUESTION_MARK) != 0) *dst++ = '?';
242  }
243  }
244 
245  /* String termination, if needed, is left to the caller of this function. */
246 }
247 
255 void StrMakeValidInPlace(char *str, const char *last, StringValidationSettings settings)
256 {
257  char *dst = str;
258  StrMakeValidInPlace(dst, str, last, settings);
259  *dst = '\0';
260 }
261 
270 {
271  /* We know it is '\0' terminated. */
272  StrMakeValidInPlace(str, str + strlen(str), settings);
273 }
274 
281 std::string StrMakeValid(const std::string &str, StringValidationSettings settings)
282 {
283  auto buf = str.data();
284  auto last = buf + str.size();
285 
286  std::ostringstream dst;
287  std::ostreambuf_iterator<char> dst_iter(dst);
288  StrMakeValidInPlace(dst_iter, buf, last, settings);
289 
290  return dst.str();
291 }
292 
300 bool StrValid(const char *str, const char *last)
301 {
302  /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
303 
304  while (str <= last && *str != '\0') {
305  size_t len = Utf8EncodedCharLen(*str);
306  /* Encoded length is 0 if the character isn't known.
307  * The length check is needed to prevent Utf8Decode to read
308  * over the terminating '\0' if that happens to be placed
309  * within the encoding of an UTF8 character. */
310  if (len == 0 || str + len > last) return false;
311 
312  WChar c;
313  len = Utf8Decode(&c, str);
314  if (!IsPrintable(c) || (c >= SCC_SPRITE_START && c <= SCC_SPRITE_END)) {
315  return false;
316  }
317 
318  str += len;
319  }
320 
321  return *str == '\0';
322 }
323 
330 static void StrLeftTrimInPlace(std::string &str)
331 {
332  size_t pos = str.find_first_not_of(' ');
333  str.erase(0, pos);
334 }
335 
342 static void StrRightTrimInPlace(std::string &str)
343 {
344  size_t pos = str.find_last_not_of(' ');
345  if (pos != std::string::npos) str.erase(pos + 1);
346 }
347 
355 void StrTrimInPlace(std::string &str)
356 {
357  StrLeftTrimInPlace(str);
358  StrRightTrimInPlace(str);
359 }
360 
367 bool StrStartsWith(const std::string_view str, const std::string_view prefix)
368 {
369  size_t prefix_len = prefix.size();
370  if (str.size() < prefix_len) return false;
371  return str.compare(0, prefix_len, prefix, 0, prefix_len) == 0;
372 }
373 
380 bool StrEndsWith(const std::string_view str, const std::string_view suffix)
381 {
382  size_t suffix_len = suffix.size();
383  if (str.size() < suffix_len) return false;
384  return str.compare(str.size() - suffix_len, suffix_len, suffix, 0, suffix_len) == 0;
385 }
386 
387 
389 void str_strip_colours(char *str)
390 {
391  char *dst = str;
392  WChar c;
393  size_t len;
394 
395  for (len = Utf8Decode(&c, str); c != '\0'; len = Utf8Decode(&c, str)) {
396  if (c < SCC_BLUE || c > SCC_BLACK) {
397  /* Copy the character back. Even if dst is current the same as str
398  * (i.e. no characters have been changed) this is quicker than
399  * moving the pointers ahead by len */
400  do {
401  *dst++ = *str++;
402  } while (--len != 0);
403  } else {
404  /* Just skip (strip) the colour codes */
405  str += len;
406  }
407  }
408  *dst = '\0';
409 }
410 
417 size_t Utf8StringLength(const char *s)
418 {
419  size_t len = 0;
420  const char *t = s;
421  while (Utf8Consume(&t) != 0) len++;
422  return len;
423 }
424 
431 size_t Utf8StringLength(const std::string &str)
432 {
433  return Utf8StringLength(str.c_str());
434 }
435 
447 bool strtolower(char *str)
448 {
449  bool changed = false;
450  for (; *str != '\0'; str++) {
451  char new_str = tolower(*str);
452  changed |= new_str != *str;
453  *str = new_str;
454  }
455  return changed;
456 }
457 
458 bool strtolower(std::string &str, std::string::size_type offs)
459 {
460  bool changed = false;
461  for (auto ch = str.begin() + offs; ch != str.end(); ++ch) {
462  auto new_ch = static_cast<char>(tolower(static_cast<unsigned char>(*ch)));
463  changed |= new_ch != *ch;
464  *ch = new_ch;
465  }
466  return changed;
467 }
468 
476 bool IsValidChar(WChar key, CharSetFilter afilter)
477 {
478  switch (afilter) {
479  case CS_ALPHANUMERAL: return IsPrintable(key);
480  case CS_NUMERAL: return (key >= '0' && key <= '9');
481  case CS_NUMERAL_SPACE: return (key >= '0' && key <= '9') || key == ' ';
482  case CS_ALPHA: return IsPrintable(key) && !(key >= '0' && key <= '9');
483  case CS_HEXADECIMAL: return (key >= '0' && key <= '9') || (key >= 'a' && key <= 'f') || (key >= 'A' && key <= 'F');
484  default: NOT_REACHED();
485  }
486 }
487 
488 #ifdef _WIN32
489 #if defined(_MSC_VER) && _MSC_VER < 1900
490 
497 int CDECL vsnprintf(char *str, size_t size, const char *format, va_list ap)
498 {
499  if (size == 0) return 0;
500 
501  errno = 0;
502  int ret = _vsnprintf(str, size, format, ap);
503 
504  if (ret < 0) {
505  if (errno != ERANGE) {
506  /* There's a formatting error, better get that looked
507  * at properly instead of ignoring it. */
508  NOT_REACHED();
509  }
510  } else if ((size_t)ret < size) {
511  /* The buffer is big enough for the number of
512  * characters stored (excluding null), i.e.
513  * the string has been null-terminated. */
514  return ret;
515  }
516 
517  /* The buffer is too small for _vsnprintf to write the
518  * null-terminator at its end and return size. */
519  str[size - 1] = '\0';
520  return (int)size;
521 }
522 #endif /* _MSC_VER */
523 
524 #endif /* _WIN32 */
525 
535 int CDECL seprintf(char *str, const char *last, const char *format, ...)
536 {
537  va_list ap;
538 
539  va_start(ap, format);
540  int ret = vseprintf(str, last, format, ap);
541  va_end(ap);
542  return ret;
543 }
544 
545 
553 char *md5sumToString(char *buf, const char *last, const uint8 md5sum[16])
554 {
555  char *p = buf;
556 
557  for (uint i = 0; i < 16; i++) {
558  p += seprintf(p, last, "%02X", md5sum[i]);
559  }
560 
561  return p;
562 }
563 
564 
565 /* UTF-8 handling routines */
566 
567 
574 size_t Utf8Decode(WChar *c, const char *s)
575 {
576  assert(c != nullptr);
577 
578  if (!HasBit(s[0], 7)) {
579  /* Single byte character: 0xxxxxxx */
580  *c = s[0];
581  return 1;
582  } else if (GB(s[0], 5, 3) == 6) {
583  if (IsUtf8Part(s[1])) {
584  /* Double byte character: 110xxxxx 10xxxxxx */
585  *c = GB(s[0], 0, 5) << 6 | GB(s[1], 0, 6);
586  if (*c >= 0x80) return 2;
587  }
588  } else if (GB(s[0], 4, 4) == 14) {
589  if (IsUtf8Part(s[1]) && IsUtf8Part(s[2])) {
590  /* Triple byte character: 1110xxxx 10xxxxxx 10xxxxxx */
591  *c = GB(s[0], 0, 4) << 12 | GB(s[1], 0, 6) << 6 | GB(s[2], 0, 6);
592  if (*c >= 0x800) return 3;
593  }
594  } else if (GB(s[0], 3, 5) == 30) {
595  if (IsUtf8Part(s[1]) && IsUtf8Part(s[2]) && IsUtf8Part(s[3])) {
596  /* 4 byte character: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
597  *c = GB(s[0], 0, 3) << 18 | GB(s[1], 0, 6) << 12 | GB(s[2], 0, 6) << 6 | GB(s[3], 0, 6);
598  if (*c >= 0x10000 && *c <= 0x10FFFF) return 4;
599  }
600  }
601 
602  /* Debug(misc, 1, "[utf8] invalid UTF-8 sequence"); */
603  *c = '?';
604  return 1;
605 }
606 
607 
615 template <class T>
616 inline size_t Utf8Encode(T buf, WChar c)
617 {
618  if (c < 0x80) {
619  *buf = c;
620  return 1;
621  } else if (c < 0x800) {
622  *buf++ = 0xC0 + GB(c, 6, 5);
623  *buf = 0x80 + GB(c, 0, 6);
624  return 2;
625  } else if (c < 0x10000) {
626  *buf++ = 0xE0 + GB(c, 12, 4);
627  *buf++ = 0x80 + GB(c, 6, 6);
628  *buf = 0x80 + GB(c, 0, 6);
629  return 3;
630  } else if (c < 0x110000) {
631  *buf++ = 0xF0 + GB(c, 18, 3);
632  *buf++ = 0x80 + GB(c, 12, 6);
633  *buf++ = 0x80 + GB(c, 6, 6);
634  *buf = 0x80 + GB(c, 0, 6);
635  return 4;
636  }
637 
638  /* Debug(misc, 1, "[utf8] can't UTF-8 encode value 0x{:X}", c); */
639  *buf = '?';
640  return 1;
641 }
642 
643 size_t Utf8Encode(char *buf, WChar c)
644 {
645  return Utf8Encode<char *>(buf, c);
646 }
647 
648 size_t Utf8Encode(std::ostreambuf_iterator<char> &buf, WChar c)
649 {
650  return Utf8Encode<std::ostreambuf_iterator<char> &>(buf, c);
651 }
652 
660 size_t Utf8TrimString(char *s, size_t maxlen)
661 {
662  size_t length = 0;
663 
664  for (const char *ptr = strchr(s, '\0'); *s != '\0';) {
665  size_t len = Utf8EncodedCharLen(*s);
666  /* Silently ignore invalid UTF8 sequences, our only concern trimming */
667  if (len == 0) len = 1;
668 
669  /* Take care when a hard cutoff was made for the string and
670  * the last UTF8 sequence is invalid */
671  if (length + len >= maxlen || (s + len > ptr)) break;
672  s += len;
673  length += len;
674  }
675 
676  *s = '\0';
677  return length;
678 }
679 
680 #ifdef DEFINE_STRCASESTR
681 char *strcasestr(const char *haystack, const char *needle)
682 {
683  size_t hay_len = strlen(haystack);
684  size_t needle_len = strlen(needle);
685  while (hay_len >= needle_len) {
686  if (strncasecmp(haystack, needle, needle_len) == 0) return const_cast<char *>(haystack);
687 
688  haystack++;
689  hay_len--;
690  }
691 
692  return nullptr;
693 }
694 #endif /* DEFINE_STRCASESTR */
695 
704 static const char *SkipGarbage(const char *str)
705 {
706  while (*str != '\0' && (*str < '0' || IsInsideMM(*str, ';', '@' + 1) || IsInsideMM(*str, '[', '`' + 1) || IsInsideMM(*str, '{', '~' + 1))) str++;
707  return str;
708 }
709 
718 int strnatcmp(const char *s1, const char *s2, bool ignore_garbage_at_front)
719 {
720  if (ignore_garbage_at_front) {
721  s1 = SkipGarbage(s1);
722  s2 = SkipGarbage(s2);
723  }
724 
725 #ifdef WITH_ICU_I18N
726  if (_current_collator) {
727  UErrorCode status = U_ZERO_ERROR;
728  int result = _current_collator->compareUTF8(s1, s2, status);
729  if (U_SUCCESS(status)) return result;
730  }
731 #endif /* WITH_ICU_I18N */
732 
733 #if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
734  int res = OTTDStringCompare(s1, s2);
735  if (res != 0) return res - 2; // Convert to normal C return values.
736 #endif
737 
738 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
739  int res = MacOSStringCompare(s1, s2);
740  if (res != 0) return res - 2; // Convert to normal C return values.
741 #endif
742 
743  /* Do a normal comparison if ICU is missing or if we cannot create a collator. */
744  return strcasecmp(s1, s2);
745 }
746 
747 #ifdef WITH_UNISCRIBE
748 
750 {
751  return new UniscribeStringIterator();
752 }
753 
754 #elif defined(WITH_ICU_I18N)
755 
756 #include <unicode/utext.h>
757 #include <unicode/brkiter.h>
758 
761 {
762  icu::BreakIterator *char_itr;
763  icu::BreakIterator *word_itr;
764 
765  std::vector<UChar> utf16_str;
766  std::vector<size_t> utf16_to_utf8;
767 
768 public:
769  IcuStringIterator() : char_itr(nullptr), word_itr(nullptr)
770  {
771  UErrorCode status = U_ZERO_ERROR;
772  this->char_itr = icu::BreakIterator::createCharacterInstance(icu::Locale(_current_language != nullptr ? _current_language->isocode : "en"), status);
773  this->word_itr = icu::BreakIterator::createWordInstance(icu::Locale(_current_language != nullptr ? _current_language->isocode : "en"), status);
774 
775  this->utf16_str.push_back('\0');
776  this->utf16_to_utf8.push_back(0);
777  }
778 
779  ~IcuStringIterator() override
780  {
781  delete this->char_itr;
782  delete this->word_itr;
783  }
784 
785  void SetString(const char *s) override
786  {
787  const char *string_base = s;
788 
789  /* Unfortunately current ICU versions only provide rudimentary support
790  * for word break iterators (especially for CJK languages) in combination
791  * with UTF-8 input. As a work around we have to convert the input to
792  * UTF-16 and create a mapping back to UTF-8 character indices. */
793  this->utf16_str.clear();
794  this->utf16_to_utf8.clear();
795 
796  while (*s != '\0') {
797  size_t idx = s - string_base;
798 
799  WChar c = Utf8Consume(&s);
800  if (c < 0x10000) {
801  this->utf16_str.push_back((UChar)c);
802  } else {
803  /* Make a surrogate pair. */
804  this->utf16_str.push_back((UChar)(0xD800 + ((c - 0x10000) >> 10)));
805  this->utf16_str.push_back((UChar)(0xDC00 + ((c - 0x10000) & 0x3FF)));
806  this->utf16_to_utf8.push_back(idx);
807  }
808  this->utf16_to_utf8.push_back(idx);
809  }
810  this->utf16_str.push_back('\0');
811  this->utf16_to_utf8.push_back(s - string_base);
812 
813  UText text = UTEXT_INITIALIZER;
814  UErrorCode status = U_ZERO_ERROR;
815  utext_openUChars(&text, this->utf16_str.data(), this->utf16_str.size() - 1, &status);
816  this->char_itr->setText(&text, status);
817  this->word_itr->setText(&text, status);
818  this->char_itr->first();
819  this->word_itr->first();
820  }
821 
822  size_t SetCurPosition(size_t pos) override
823  {
824  /* Convert incoming position to an UTF-16 string index. */
825  uint utf16_pos = 0;
826  for (uint i = 0; i < this->utf16_to_utf8.size(); i++) {
827  if (this->utf16_to_utf8[i] == pos) {
828  utf16_pos = i;
829  break;
830  }
831  }
832 
833  /* isBoundary has the documented side-effect of setting the current
834  * position to the first valid boundary equal to or greater than
835  * the passed value. */
836  this->char_itr->isBoundary(utf16_pos);
837  return this->utf16_to_utf8[this->char_itr->current()];
838  }
839 
840  size_t Next(IterType what) override
841  {
842  int32_t pos;
843  switch (what) {
844  case ITER_CHARACTER:
845  pos = this->char_itr->next();
846  break;
847 
848  case ITER_WORD:
849  pos = this->word_itr->following(this->char_itr->current());
850  /* The ICU word iterator considers both the start and the end of a word a valid
851  * break point, but we only want word starts. Move to the next location in
852  * case the new position points to whitespace. */
853  while (pos != icu::BreakIterator::DONE &&
854  IsWhitespace(Utf16DecodeChar((const uint16 *)&this->utf16_str[pos]))) {
855  int32_t new_pos = this->word_itr->next();
856  /* Don't set it to DONE if it was valid before. Otherwise we'll return END
857  * even though the iterator wasn't at the end of the string before. */
858  if (new_pos == icu::BreakIterator::DONE) break;
859  pos = new_pos;
860  }
861 
862  this->char_itr->isBoundary(pos);
863  break;
864 
865  default:
866  NOT_REACHED();
867  }
868 
869  return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
870  }
871 
872  size_t Prev(IterType what) override
873  {
874  int32_t pos;
875  switch (what) {
876  case ITER_CHARACTER:
877  pos = this->char_itr->previous();
878  break;
879 
880  case ITER_WORD:
881  pos = this->word_itr->preceding(this->char_itr->current());
882  /* The ICU word iterator considers both the start and the end of a word a valid
883  * break point, but we only want word starts. Move to the previous location in
884  * case the new position points to whitespace. */
885  while (pos != icu::BreakIterator::DONE &&
886  IsWhitespace(Utf16DecodeChar((const uint16 *)&this->utf16_str[pos]))) {
887  int32_t new_pos = this->word_itr->previous();
888  /* Don't set it to DONE if it was valid before. Otherwise we'll return END
889  * even though the iterator wasn't at the start of the string before. */
890  if (new_pos == icu::BreakIterator::DONE) break;
891  pos = new_pos;
892  }
893 
894  this->char_itr->isBoundary(pos);
895  break;
896 
897  default:
898  NOT_REACHED();
899  }
900 
901  return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
902  }
903 };
904 
906 {
907  return new IcuStringIterator();
908 }
909 
910 #else
911 
913 class DefaultStringIterator : public StringIterator
914 {
915  const char *string;
916  size_t len;
917  size_t cur_pos;
918 
919 public:
920  DefaultStringIterator() : string(nullptr), len(0), cur_pos(0)
921  {
922  }
923 
924  virtual void SetString(const char *s)
925  {
926  this->string = s;
927  this->len = strlen(s);
928  this->cur_pos = 0;
929  }
930 
931  virtual size_t SetCurPosition(size_t pos)
932  {
933  assert(this->string != nullptr && pos <= this->len);
934  /* Sanitize in case we get a position inside an UTF-8 sequence. */
935  while (pos > 0 && IsUtf8Part(this->string[pos])) pos--;
936  return this->cur_pos = pos;
937  }
938 
939  virtual size_t Next(IterType what)
940  {
941  assert(this->string != nullptr);
942 
943  /* Already at the end? */
944  if (this->cur_pos >= this->len) return END;
945 
946  switch (what) {
947  case ITER_CHARACTER: {
948  WChar c;
949  this->cur_pos += Utf8Decode(&c, this->string + this->cur_pos);
950  return this->cur_pos;
951  }
952 
953  case ITER_WORD: {
954  WChar c;
955  /* Consume current word. */
956  size_t offs = Utf8Decode(&c, this->string + this->cur_pos);
957  while (this->cur_pos < this->len && !IsWhitespace(c)) {
958  this->cur_pos += offs;
959  offs = Utf8Decode(&c, this->string + this->cur_pos);
960  }
961  /* Consume whitespace to the next word. */
962  while (this->cur_pos < this->len && IsWhitespace(c)) {
963  this->cur_pos += offs;
964  offs = Utf8Decode(&c, this->string + this->cur_pos);
965  }
966 
967  return this->cur_pos;
968  }
969 
970  default:
971  NOT_REACHED();
972  }
973 
974  return END;
975  }
976 
977  virtual size_t Prev(IterType what)
978  {
979  assert(this->string != nullptr);
980 
981  /* Already at the beginning? */
982  if (this->cur_pos == 0) return END;
983 
984  switch (what) {
985  case ITER_CHARACTER:
986  return this->cur_pos = Utf8PrevChar(this->string + this->cur_pos) - this->string;
987 
988  case ITER_WORD: {
989  const char *s = this->string + this->cur_pos;
990  WChar c;
991  /* Consume preceding whitespace. */
992  do {
993  s = Utf8PrevChar(s);
994  Utf8Decode(&c, s);
995  } while (s > this->string && IsWhitespace(c));
996  /* Consume preceding word. */
997  while (s > this->string && !IsWhitespace(c)) {
998  s = Utf8PrevChar(s);
999  Utf8Decode(&c, s);
1000  }
1001  /* Move caret back to the beginning of the word. */
1002  if (IsWhitespace(c)) Utf8Consume(&s);
1003 
1004  return this->cur_pos = s - this->string;
1005  }
1006 
1007  default:
1008  NOT_REACHED();
1009  }
1010 
1011  return END;
1012  }
1013 };
1014 
1015 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
1016 /* static */ StringIterator *StringIterator::Create()
1017 {
1018  StringIterator *i = OSXStringIterator::Create();
1019  if (i != nullptr) return i;
1020 
1021  return new DefaultStringIterator();
1022 }
1023 #else
1024 /* static */ StringIterator *StringIterator::Create()
1025 {
1026  return new DefaultStringIterator();
1027 }
1028 #endif /* defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN) */
1029 
1030 #endif
StringIterator::Prev
virtual size_t Prev(IterType what=ITER_CHARACTER)=0
Move the cursor back by one iteration unit.
IcuStringIterator::utf16_to_utf8
std::vector< size_t > utf16_to_utf8
Mapping from UTF-16 code point position to index in the UTF-8 source string.
Definition: string.cpp:766
WChar
char32_t WChar
Type for wide characters, i.e.
Definition: string_type.h:35
SVS_ALLOW_NEWLINE
@ SVS_ALLOW_NEWLINE
Allow newlines.
Definition: string_type.h:51
GB
static uint GB(const T x, const uint8 s, const uint8 n)
Fetch n bits from x, started at bit s.
Definition: bitmath_func.hpp:32
StringIterator::IterType
IterType
Type of the iterator.
Definition: string_base.h:17
strtolower
bool strtolower(char *str)
Convert a given ASCII string to lowercase.
Definition: string.cpp:447
win32.h
StrRightTrimInPlace
static void StrRightTrimInPlace(std::string &str)
Trim the spaces from the end of given string in place, i.e.
Definition: string.cpp:342
StringIterator::END
static const size_t END
Sentinel to indicate end-of-iteration.
Definition: string_base.h:23
math_func.hpp
str_fix_scc_encoded
void str_fix_scc_encoded(char *str, const char *last)
Scan the string for old values of SCC_ENCODED and fix it to it's new, static value.
Definition: string.cpp:169
HasBit
static bool HasBit(const T x, const uint8 y)
Checks if a bit in a value is set.
Definition: bitmath_func.hpp:103
IcuStringIterator::word_itr
icu::BreakIterator * word_itr
ICU iterator for words.
Definition: string.cpp:763
_current_collator
std::unique_ptr< icu::Collator > _current_collator
Collator for the language currently in use.
Definition: strings.cpp:51
Utf8Encode
size_t Utf8Encode(T buf, WChar c)
Encode a unicode character and place it in the buffer.
Definition: string.cpp:616
StringIterator::Next
virtual size_t Next(IterType what=ITER_CHARACTER)=0
Advance the cursor by one iteration unit.
CS_ALPHA
@ CS_ALPHA
Only alphabetic values.
Definition: string_type.h:30
UniscribeStringIterator
String iterator using Uniscribe as a backend.
Definition: string_uniscribe.h:67
Utf16DecodeChar
static WChar Utf16DecodeChar(const uint16 *c)
Decode an UTF-16 character.
Definition: string_func.h:210
StringIterator::SetString
virtual void SetString(const char *s)=0
Set a new iteration string.
IcuStringIterator::char_itr
icu::BreakIterator * char_itr
ICU iterator for characters.
Definition: string.cpp:762
StringIterator::ITER_CHARACTER
@ ITER_CHARACTER
Iterate over characters (or more exactly grapheme clusters).
Definition: string_base.h:18
control_codes.h
IsInsideMM
static bool IsInsideMM(const T x, const size_t min, const size_t max)
Checks if a value is in an interval.
Definition: math_func.hpp:204
IcuStringIterator::Next
size_t Next(IterType what) override
Advance the cursor by one iteration unit.
Definition: string.cpp:840
string_osx.h
gfx_func.h
Utf8StringLength
size_t Utf8StringLength(const char *s)
Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes th...
Definition: string.cpp:417
StrValid
bool StrValid(const char *str, const char *last)
Checks whether the given string is valid, i.e.
Definition: string.cpp:300
StringIterator::Create
static StringIterator * Create()
Create a new iterator instance.
Definition: string.cpp:905
SVS_ALLOW_CONTROL_CODE
@ SVS_ALLOW_CONTROL_CODE
Allow the special control codes.
Definition: string_type.h:52
StringIterator
Class for iterating over different kind of parts of a string.
Definition: string_base.h:14
IcuStringIterator::SetCurPosition
size_t SetCurPosition(size_t pos) override
Change the current string cursor.
Definition: string.cpp:822
StringIterator::SetCurPosition
virtual size_t SetCurPosition(size_t pos)=0
Change the current string cursor.
StrMakeValidInPlace
void StrMakeValidInPlace(char *str, const char *last, StringValidationSettings settings)
Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).
Definition: string.cpp:255
StrTrimInPlace
void StrTrimInPlace(std::string &str)
Trim the spaces from given string in place, i.e.
Definition: string.cpp:355
str_strip_colours
void str_strip_colours(char *str)
Scans the string for colour codes and strips them.
Definition: string.cpp:389
StrStartsWith
bool StrStartsWith(const std::string_view str, const std::string_view prefix)
Check whether the given string starts with the given prefix.
Definition: string.cpp:367
_current_language
const LanguageMetadata * _current_language
The currently loaded language.
Definition: strings.cpp:46
safeguards.h
IcuStringIterator::utf16_str
std::vector< UChar > utf16_str
UTF-16 copy of the string.
Definition: string.cpp:765
IsValidChar
bool IsValidChar(WChar key, CharSetFilter afilter)
Only allow certain keys.
Definition: string.cpp:476
ttd_strnlen
static size_t ttd_strnlen(const char *str, size_t maxlen)
Get the length of a string, within a limited buffer.
Definition: string_func.h:76
settings
fluid_settings_t * settings
FluidSynth settings handle.
Definition: fluidsynth.cpp:21
vseprintf
int CDECL vseprintf(char *str, const char *last, const char *format, va_list ap)
Safer implementation of vsnprintf; same as vsnprintf except:
Definition: string.cpp:61
StringIterator::ITER_WORD
@ ITER_WORD
Iterate over words.
Definition: string_base.h:19
language.h
stdafx.h
CS_ALPHANUMERAL
@ CS_ALPHANUMERAL
Both numeric and alphabetic and spaces and stuff.
Definition: string_type.h:27
LanguagePackHeader::isocode
char isocode[16]
the ISO code for the language (not country code)
Definition: language.h:31
StringValidationSettings
StringValidationSettings
Settings for the string validation.
Definition: string_type.h:48
StrEndsWith
bool StrEndsWith(const std::string_view str, const std::string_view suffix)
Check whether the given string ends with the given suffix.
Definition: string.cpp:380
Utf8Decode
size_t Utf8Decode(WChar *c, const char *s)
Decode and consume the next UTF-8 encoded character.
Definition: string.cpp:574
string_func.h
StrMakeValid
std::string StrMakeValid(const std::string &str, StringValidationSettings settings)
Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).
Definition: string.cpp:281
str_fmt
char *CDECL str_fmt(const char *str,...)
Format, "printf", into a newly allocated string.
Definition: string.cpp:150
alloc_func.hpp
IcuStringIterator::SetString
void SetString(const char *s) override
Set a new iteration string.
Definition: string.cpp:785
StrLeftTrimInPlace
static void StrLeftTrimInPlace(std::string &str)
Trim the spaces from the begin of given string in place, i.e.
Definition: string.cpp:330
IcuStringIterator
String iterator using ICU as a backend.
Definition: string.cpp:760
seprintf
int CDECL seprintf(char *str, const char *last, const char *format,...)
Safer implementation of snprintf; same as snprintf except:
Definition: string.cpp:535
stredup
char * stredup(const char *s, const char *last)
Create a duplicate of the given string.
Definition: string.cpp:137
error
void CDECL error(const char *s,...)
Error handling for fatal non-user errors.
Definition: openttd.cpp:132
Debug
#define Debug(name, level, format_string,...)
Ouptut a line of debugging information.
Definition: debug.h:37
Utf8TrimString
size_t Utf8TrimString(char *s, size_t maxlen)
Properly terminate an UTF8 string to some maximum length.
Definition: string.cpp:660
CS_NUMERAL_SPACE
@ CS_NUMERAL_SPACE
Only numbers and spaces.
Definition: string_type.h:29
SkipGarbage
static const char * SkipGarbage(const char *str)
Skip some of the 'garbage' in the string that we don't want to use to sort on.
Definition: string.cpp:704
IsWhitespace
static bool IsWhitespace(WChar c)
Check whether UNICODE character is whitespace or not, i.e.
Definition: string_func.h:257
strnatcmp
int strnatcmp(const char *s1, const char *s2, bool ignore_garbage_at_front)
Compares two strings using case insensitive natural sort.
Definition: string.cpp:718
md5sumToString
char * md5sumToString(char *buf, const char *last, const uint8 md5sum[16])
Convert the md5sum to a hexadecimal string representation.
Definition: string.cpp:553
strecpy
char * strecpy(char *dst, const char *src, const char *last)
Copies characters from one buffer to another.
Definition: string.cpp:112
CS_HEXADECIMAL
@ CS_HEXADECIMAL
Only hexadecimal characters.
Definition: string_type.h:31
strecat
char * strecat(char *dst, const char *src, const char *last)
Appends characters from one string to another.
Definition: string.cpp:84
MacOSStringCompare
int MacOSStringCompare(const char *s1, const char *s2)
Compares two strings using case insensitive natural sort.
Definition: string_osx.cpp:323
lastof
#define lastof(x)
Get the last element of an fixed size array.
Definition: stdafx.h:394
SVS_REPLACE_WITH_QUESTION_MARK
@ SVS_REPLACE_WITH_QUESTION_MARK
Replace the unknown/bad bits with question marks.
Definition: string_type.h:50
CS_NUMERAL
@ CS_NUMERAL
Only numeric ones.
Definition: string_type.h:28
CharSetFilter
CharSetFilter
Valid filter types for IsValidChar.
Definition: string_type.h:26
Utf8PrevChar
static char * Utf8PrevChar(char *s)
Retrieve the previous UNICODE character in an UTF-8 encoded string.
Definition: string_func.h:157
debug.h
string_uniscribe.h
Utf8EncodedCharLen
static int8 Utf8EncodedCharLen(char c)
Return the length of an UTF-8 encoded value based on a single char.
Definition: string_func.h:132
IcuStringIterator::Prev
size_t Prev(IterType what) override
Move the cursor back by one iteration unit.
Definition: string.cpp:872