A wrapper of the native StringScanner class that works correctly with multibyte character encodings. The native class deals only in bytes, not characters, for methods like [pos] and [matched_size]. This class deals only in characters, instead.
# File lib/sass/util/multibyte_string_scanner.rb, line 11 def self.new(str) return StringScanner.new(str) if str.ascii_only? super end
# File lib/sass/util/multibyte_string_scanner.rb, line 16 def initialize(str) super @mb_pos = 0 @mb_matched_size = nil @mb_last_pos = nil end
# File lib/sass/util/multibyte_string_scanner.rb, line 26 def check(pattern); _match super; end
# File lib/sass/util/multibyte_string_scanner.rb, line 27 def check_until(pattern); _matched super; end
# File lib/sass/util/multibyte_string_scanner.rb, line 41 def get_byte raise "MultibyteStringScanner doesn't support #get_byte." end
# File lib/sass/util/multibyte_string_scanner.rb, line 45 def getbyte raise "MultibyteStringScanner doesn't support #getbyte." end
# File lib/sass/util/multibyte_string_scanner.rb, line 28 def getch; _forward _match super; end
# File lib/sass/util/multibyte_string_scanner.rb, line 29 def match?(pattern); _size check(pattern); end
# File lib/sass/util/multibyte_string_scanner.rb, line 30 def matched_size; @mb_matched_size; end
# File lib/sass/util/multibyte_string_scanner.rb, line 31 def peek(len); string[@mb_pos, len]; end
# File lib/sass/util/multibyte_string_scanner.rb, line 33 def pos; @mb_pos; end
# File lib/sass/util/multibyte_string_scanner.rb, line 49 def pos=(n) @mb_last_pos = nil # We set position kind of a lot during parsing, so we want it to be as # efficient as possible. This is complicated by the fact that UTF-8 is a # variable-length encoding, so it's difficult to find the byte length that # corresponds to a given character length. # # Our heuristic here is to try to count the fewest possible characters. So # if the new position is close to the current one, just count the # characters between the two; if the new position is closer to the # beginning of the string, just count the characters from there. if @mb_pos - n < @mb_pos / 2 # New position is close to old position byte_delta = @mb_pos > n ? -string[n...@mb_pos].bytesize : string[@mb_pos...n].bytesize super(byte_pos + byte_delta) else # New position is close to BOS super(string[0...n].bytesize) end @mb_pos = n end
# File lib/sass/util/multibyte_string_scanner.rb, line 72 def reset @mb_pos = 0 @mb_matched_size = nil @mb_last_pos = nil super end
# File lib/sass/util/multibyte_string_scanner.rb, line 35 def rest_size; rest.size; end
# File lib/sass/util/multibyte_string_scanner.rb, line 36 def scan(pattern); _forward _match super; end
# File lib/sass/util/multibyte_string_scanner.rb, line 79 def scan_full(pattern, advance_pointer_p, return_string_p) res = _match super(pattern, advance_pointer_p, true) _forward res if advance_pointer_p return res if return_string_p end
# File lib/sass/util/multibyte_string_scanner.rb, line 37 def scan_until(pattern); _forward _matched super; end
# File lib/sass/util/multibyte_string_scanner.rb, line 85 def search_full(pattern, advance_pointer_p, return_string_p) res = super(pattern, advance_pointer_p, true) _forward res if advance_pointer_p _matched((res if return_string_p)) end
# File lib/sass/util/multibyte_string_scanner.rb, line 38 def skip(pattern); _size scan(pattern); end
# File lib/sass/util/multibyte_string_scanner.rb, line 39 def skip_until(pattern); _matched _size scan_until(pattern); end
# File lib/sass/util/multibyte_string_scanner.rb, line 91 def string=(str) @mb_pos = 0 @mb_matched_size = nil @mb_last_pos = nil super end
# File lib/sass/util/multibyte_string_scanner.rb, line 98 def terminate @mb_pos = string.size @mb_matched_size = nil @mb_last_pos = nil super end
# File lib/sass/util/multibyte_string_scanner.rb, line 106 def unscan super @mb_pos = @mb_last_pos @mb_last_pos = @mb_matched_size = nil end