Definitions of all tokens involved in the lexical analysis
{reading => token_class} {reading => [token_class, *opt]}
# File rdoc/parsers/parse_rb.rb, line 270 def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts) token_n = token_n.id2name unless token_n.kind_of?(String) if RubyToken.const_defined?(token_n) IRB.fail AlreadyDefinedToken, token_n end token_c = Class.new super_token RubyToken.const_set token_n, token_c # token_c.inspect if reading if TkReading2Token[reading] IRB.fail TkReading2TokenDuplicateError, token_n, reading end if opts.empty? TkReading2Token[reading] = [token_c] else TkReading2Token[reading] = [token_c].concat(opts) end end TkSymbol2Token[token_n.intern] = token_c if token_c <= TkOp token_c.class_eval %{ def self.op_name; "#{reading}"; end } end end
# File rdoc/parsers/parse_rb.rb, line 112 def Token(token, value = nil) tk = nil case token when String, Symbol source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token if (tk = source[token]).nil? IRB.fail TkReading2TokenNoKey, token end tk = Token(tk[0], value) else tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty? token.new(@prev_line_no, @prev_char_no) else token.new(@prev_line_no, @prev_char_no, value) end end tk end