#$@%#$%!

Ngiler SH3LL 360
: / opt / alt / ruby18 / lib64 / ruby / 1.8 / rdoc / parsers /

Information Server
MySQL : OFF Perl : OFF CURL : ON WGET : OFF PKEXEC : OFF

Directive	Local Value
IP Address	162.0.209.101
System	Linux premium169.web-hosting.com 4.18.0-553.50.1.lve.el8.x86_64 #1 SMP Thu Apr 17 19:10:24 UTC 2025 x86_64
User	luxeelil
PHP Version	8.0.30
Software	LiteSpeed
Doc root	Writable

Edit File : parse_rb.rb \| Size : 58.81 KB
#!/usr/local/bin/ruby # Parse a Ruby source file, building a set of objects # representing the modules, classes, methods, # requires, and includes we find (these classes # are defined in code_objects.rb). # This file contains stuff stolen outright from: # # rtags.rb - # ruby-lex.rb - ruby lexcal analizer # ruby-token.rb - ruby tokens # by Keiju ISHITSUKA (Nippon Rational Inc.) # require "e2mmap" require "irb/slex" require "rdoc/code_objects" require "rdoc/tokenstream" require "rdoc/markup/simple_markup/preprocess" require "rdoc/parsers/parserfactory" $TOKEN_DEBUG = $DEBUG # Definitions of all tokens involved in the lexical analysis module RubyToken EXPR_BEG = :EXPR_BEG EXPR_MID = :EXPR_MID EXPR_END = :EXPR_END EXPR_ARG = :EXPR_ARG EXPR_FNAME = :EXPR_FNAME EXPR_DOT = :EXPR_DOT EXPR_CLASS = :EXPR_CLASS class Token NO_TEXT = "??".freeze attr :text def initialize(line_no, char_no) @line_no = line_no @char_no = char_no @text = NO_TEXT end # Because we're used in contexts that expect to return a token, # we set the text string and then return ourselves def set_text(text) @text = text self end attr_reader :line_no, :char_no, :text end class TkNode < Token attr :node end class TkId < Token def initialize(line_no, char_no, name) super(line_no, char_no) @name = name end attr :name end class TkKW < TkId end class TkVal < Token def initialize(line_no, char_no, value = nil) super(line_no, char_no) set_text(value) end end class TkOp < Token def name self.class.op_name end end class TkOPASGN < TkOp def initialize(line_no, char_no, op) super(line_no, char_no) op = TkReading2Token[op] unless op.kind_of?(Symbol) @op = op end attr :op end class TkUnknownChar < Token def initialize(line_no, char_no, id) super(line_no, char_no) @name = char_no.chr end attr :name end class TkError < Token end def set_token_position(line, char) @prev_line_no = line @prev_char_no = char end def Token(token, value = nil) tk = nil case token when String, Symbol source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token if (tk = source[token]).nil? IRB.fail TkReading2TokenNoKey, token end tk = Token(tk[0], value) else tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty? token.new(@prev_line_no, @prev_char_no) else token.new(@prev_line_no, @prev_char_no, value) end end tk end TokenDefinitions = [ [:TkCLASS, TkKW, "class", EXPR_CLASS], [:TkMODULE, TkKW, "module", EXPR_BEG], [:TkDEF, TkKW, "def", EXPR_FNAME], [:TkUNDEF, TkKW, "undef", EXPR_FNAME], [:TkBEGIN, TkKW, "begin", EXPR_BEG], [:TkRESCUE, TkKW, "rescue", EXPR_MID], [:TkENSURE, TkKW, "ensure", EXPR_BEG], [:TkEND, TkKW, "end", EXPR_END], [:TkIF, TkKW, "if", EXPR_BEG, :TkIF_MOD], [:TkUNLESS, TkKW, "unless", EXPR_BEG, :TkUNLESS_MOD], [:TkTHEN, TkKW, "then", EXPR_BEG], [:TkELSIF, TkKW, "elsif", EXPR_BEG], [:TkELSE, TkKW, "else", EXPR_BEG], [:TkCASE, TkKW, "case", EXPR_BEG], [:TkWHEN, TkKW, "when", EXPR_BEG], [:TkWHILE, TkKW, "while", EXPR_BEG, :TkWHILE_MOD], [:TkUNTIL, TkKW, "until", EXPR_BEG, :TkUNTIL_MOD], [:TkFOR, TkKW, "for", EXPR_BEG], [:TkBREAK, TkKW, "break", EXPR_END], [:TkNEXT, TkKW, "next", EXPR_END], [:TkREDO, TkKW, "redo", EXPR_END], [:TkRETRY, TkKW, "retry", EXPR_END], [:TkIN, TkKW, "in", EXPR_BEG], [:TkDO, TkKW, "do", EXPR_BEG], [:TkRETURN, TkKW, "return", EXPR_MID], [:TkYIELD, TkKW, "yield", EXPR_END], [:TkSUPER, TkKW, "super", EXPR_END], [:TkSELF, TkKW, "self", EXPR_END], [:TkNIL, TkKW, "nil", EXPR_END], [:TkTRUE, TkKW, "true", EXPR_END], [:TkFALSE, TkKW, "false", EXPR_END], [:TkAND, TkKW, "and", EXPR_BEG], [:TkOR, TkKW, "or", EXPR_BEG], [:TkNOT, TkKW, "not", EXPR_BEG], [:TkIF_MOD, TkKW], [:TkUNLESS_MOD, TkKW], [:TkWHILE_MOD, TkKW], [:TkUNTIL_MOD, TkKW], [:TkALIAS, TkKW, "alias", EXPR_FNAME], [:TkDEFINED, TkKW, "defined?", EXPR_END], [:TklBEGIN, TkKW, "BEGIN", EXPR_END], [:TklEND, TkKW, "END", EXPR_END], [:Tk__LINE__, TkKW, "__LINE__", EXPR_END], [:Tk__FILE__, TkKW, "__FILE__", EXPR_END], [:TkIDENTIFIER, TkId], [:TkFID, TkId], [:TkGVAR, TkId], [:TkIVAR, TkId], [:TkCONSTANT, TkId], [:TkINTEGER, TkVal], [:TkFLOAT, TkVal], [:TkSTRING, TkVal], [:TkXSTRING, TkVal], [:TkREGEXP, TkVal], [:TkCOMMENT, TkVal], [:TkDSTRING, TkNode], [:TkDXSTRING, TkNode], [:TkDREGEXP, TkNode], [:TkNTH_REF, TkId], [:TkBACK_REF, TkId], [:TkUPLUS, TkOp, "+@"], [:TkUMINUS, TkOp, "-@"], [:TkPOW, TkOp, "*"], [:TkCMP, TkOp, "<=>"], [:TkEQ, TkOp, "=="], [:TkEQQ, TkOp, "==="], [:TkNEQ, TkOp, "!="], [:TkGEQ, TkOp, ">="], [:TkLEQ, TkOp, "<="], [:TkANDOP, TkOp, "&&"], [:TkOROP, TkOp, "\|\|"], [:TkMATCH, TkOp, "=~"], [:TkNMATCH, TkOp, "!~"], [:TkDOT2, TkOp, ".."], [:TkDOT3, TkOp, "..."], [:TkAREF, TkOp, "[]"], [:TkASET, TkOp, "[]="], [:TkLSHFT, TkOp, "<<"], [:TkRSHFT, TkOp, ">>"], [:TkCOLON2, TkOp], [:TkCOLON3, TkOp], # [:OPASGN, TkOp], # +=, -= etc. # [:TkASSOC, TkOp, "=>"], [:TkQUESTION, TkOp, "?"], #? [:TkCOLON, TkOp, ":"], #: [:TkfLPAREN], # func( # [:TkfLBRACK], # func[ # [:TkfLBRACE], # func{ # [:TkSTAR], # arg [:TkAMPER], # &arg # [:TkSYMBOL, TkId], # :SYMBOL [:TkSYMBEG, TkId], [:TkGT, TkOp, ">"], [:TkLT, TkOp, "<"], [:TkPLUS, TkOp, "+"], [:TkMINUS, TkOp, "-"], [:TkMULT, TkOp, ""], [:TkDIV, TkOp, "/"], [:TkMOD, TkOp, "%"], [:TkBITOR, TkOp, "\|"], [:TkBITXOR, TkOp, "^"], [:TkBITAND, TkOp, "&"], [:TkBITNOT, TkOp, "~"], [:TkNOTOP, TkOp, "!"], [:TkBACKQUOTE, TkOp, "`"], [:TkASSIGN, Token, "="], [:TkDOT, Token, "."], [:TkLPAREN, Token, "("], #(exp) [:TkLBRACK, Token, "["], #[arry] [:TkLBRACE, Token, "{"], #{hash} [:TkRPAREN, Token, ")"], [:TkRBRACK, Token, "]"], [:TkRBRACE, Token, "}"], [:TkCOMMA, Token, ","], [:TkSEMICOLON, Token, ";"], [:TkRD_COMMENT], [:TkSPACE], [:TkNL], [:TkEND_OF_SCRIPT], [:TkBACKSLASH, TkUnknownChar, "\\"], [:TkAT, TkUnknownChar, "@"], [:TkDOLLAR, TkUnknownChar, "\$"], #" ] # {reading => token_class} # {reading => [token_class, opt]} TkReading2Token = {} TkSymbol2Token = {} def RubyToken.def_token(token_n, super_token = Token, reading = nil, opts) token_n = token_n.id2name unless token_n.kind_of?(String) if RubyToken.const_defined?(token_n) IRB.fail AlreadyDefinedToken, token_n end token_c = Class.new super_token RubyToken.const_set token_n, token_c # token_c.inspect if reading if TkReading2Token[reading] IRB.fail TkReading2TokenDuplicateError, token_n, reading end if opts.empty? TkReading2Token[reading] = [token_c] else TkReading2Token[reading] = [token_c].concat(opts) end end TkSymbol2Token[token_n.intern] = token_c if token_c <= TkOp token_c.class_eval %{ def self.op_name; "#{reading}"; end } end end for defs in TokenDefinitions def_token(defs) end NEWLINE_TOKEN = TkNL.new(0,0) NEWLINE_TOKEN.set_text("\n") end # Lexical analyzer for Ruby source class RubyLex ###################################################################### # # Read an input stream character by character. We allow for unlimited # ungetting of characters just read. # # We simplify the implementation greatly by reading the entire input # into a buffer initially, and then simply traversing it using # pointers. # # We also have to allow for the <i>here document diversion</i>. This # little gem comes about when the lexer encounters a here # document. At this point we effectively need to split the input # stream into two parts: one to read the body of the here document, # the other to read the rest of the input line where the here # document was initially encountered. For example, we might have # # do_something(<<-A, <<-B) # stuff # for # A # stuff # for # B # # When the lexer encounters the <<A, it reads until the end of the # line, and keeps it around for later. It then reads the body of the # here document. Once complete, it needs to read the rest of the # original line, but then skip the here document body. # class BufferedReader attr_reader :line_num def initialize(content) if /\t/ =~ content tab_width = Options.instance.tab_width content = content.split(/\n/).map do \|line\| 1 while line.gsub!(/\t+/) { ' ' * (tab_width$&.length - $`.length % tab_width)} && $~ #` line end .join("\n") end @content = content @content << "\n" unless @content[-1,1] == "\n" @size = @content.size @offset = 0 @hwm = 0 @line_num = 1 @read_back_offset = 0 @last_newline = 0 @newline_pending = false end def column @offset - @last_newline end def getc return nil if @offset >= @size ch = @content[@offset, 1] @offset += 1 @hwm = @offset if @hwm < @offset if @newline_pending @line_num += 1 @last_newline = @offset - 1 @newline_pending = false end if ch == "\n" @newline_pending = true end ch end def getc_already_read getc end def ungetc(ch) raise "unget past beginning of file" if @offset <= 0 @offset -= 1 if @content[@offset] == ?\n @newline_pending = false end end def get_read res = @content[@read_back_offset...@offset] @read_back_offset = @offset res end def peek(at) pos = @offset + at if pos >= @size nil else @content[pos, 1] end end def peek_equal(str) @content[@offset, str.length] == str end def divert_read_from(reserve) @content[@offset, 0] = reserve @size = @content.size end end # end of nested class BufferedReader extend Exception2MessageMapper def_exception(:AlreadyDefinedToken, "Already defined token(%s)") def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')") def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')") def_exception(:TkReading2TokenDuplicateError, "key duplicate(token_n='%s', key='%s')") def_exception(:SyntaxError, "%s") include RubyToken include IRB attr_reader :continue attr_reader :lex_state def RubyLex.debug? false end def initialize(content) lex_init @reader = BufferedReader.new(content) @exp_line_no = @line_no = 1 @base_char_no = 0 @indent = 0 @ltype = nil @quoted = nil @lex_state = EXPR_BEG @space_seen = false @continue = false @line = "" @skip_space = false @read_auto_clean_up = false @exception_on_syntax_error = true end attr :skip_space, true attr :read_auto_clean_up, true attr :exception_on_syntax_error, true attr :indent # io functions def line_no @reader.line_num end def char_no @reader.column end def get_read @reader.get_read end def getc @reader.getc end def getc_of_rests @reader.getc_already_read end def gets c = getc or return l = "" begin l.concat c unless c == "\r" break if c == "\n" end while c = getc l end def ungetc(c = nil) @reader.ungetc(c) end def peek_equal?(str) @reader.peek_equal(str) end def peek(i = 0) @reader.peek(i) end def lex until (((tk = token).kind_of?(TkNL) \|\| tk.kind_of?(TkEND_OF_SCRIPT)) && !@continue or tk.nil?) end line = get_read if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) \|\| tk.nil? nil else line end end def token set_token_position(line_no, char_no) begin begin tk = @OP.match(self) @space_seen = tk.kind_of?(TkSPACE) rescue SyntaxError abort if @exception_on_syntax_error tk = TkError.new(line_no, char_no) end end while @skip_space and tk.kind_of?(TkSPACE) if @read_auto_clean_up get_read end # throw :eof unless tk p tk if $DEBUG tk end ENINDENT_CLAUSE = [ "case", "class", "def", "do", "for", "if", "module", "unless", "until", "while", "begin" #, "when" ] DEINDENT_CLAUSE = ["end" #, "when" ] PERCENT_LTYPE = { "q" => "\'", "Q" => "\"", "x" => "\`", "r" => "/", "w" => "]" } PERCENT_PAREN = { "{" => "}", "[" => "]", "<" => ">", "(" => ")" } Ltype2Token = { "\'" => TkSTRING, "\"" => TkSTRING, "\`" => TkXSTRING, "/" => TkREGEXP, "]" => TkDSTRING } Ltype2Token.default = TkSTRING DLtype2Token = { "\"" => TkDSTRING, "\`" => TkDXSTRING, "/" => TkDREGEXP, } def lex_init() @OP = SLex.new @OP.def_rules("\0", "\004", "\032") do \|chars, io\| Token(TkEND_OF_SCRIPT).set_text(chars) end @OP.def_rules(" ", "\t", "\f", "\r", "\13") do \|chars, io\| @space_seen = TRUE while (ch = getc) =~ /[ \t\f\r\13]/ chars << ch end ungetc Token(TkSPACE).set_text(chars) end @OP.def_rule("#") do \|op, io\| identify_comment end @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do \|op, io\| str = op @ltype = "=" begin line = "" begin ch = getc line << ch end until ch == "\n" str << line end until line =~ /^=end/ ungetc @ltype = nil if str =~ /\A=begin\s+rdoc/i str.sub!(/\A=begin.\n/, '') str.sub!(/^=end./m, '') Token(TkCOMMENT).set_text(str) else Token(TkRD_COMMENT)#.set_text(str) end end @OP.def_rule("\n") do print "\\n\n" if RubyLex.debug? case @lex_state when EXPR_BEG, EXPR_FNAME, EXPR_DOT @continue = TRUE else @continue = FALSE @lex_state = EXPR_BEG end Token(TkNL).set_text("\n") end @OP.def_rules("", "*", "!", "!=", "!~", "=", "==", "===", "=~", "<=>", "<", "<=", ">", ">=", ">>") do \|op, io\| @lex_state = EXPR_BEG Token(op).set_text(op) end @OP.def_rules("<<") do \|op, io\| tk = nil if @lex_state != EXPR_END && @lex_state != EXPR_CLASS && (@lex_state != EXPR_ARG \|\| @space_seen) c = peek(0) if /[-\w_\"\'\`]/ =~ c tk = identify_here_document end end if !tk @lex_state = EXPR_BEG tk = Token(op).set_text(op) end tk end @OP.def_rules("'", '"') do \|op, io\| identify_string(op) end @OP.def_rules("`") do \|op, io\| if @lex_state == EXPR_FNAME Token(op).set_text(op) else identify_string(op) end end @OP.def_rules('?') do \|op, io\| if @lex_state == EXPR_END @lex_state = EXPR_BEG Token(TkQUESTION).set_text(op) else ch = getc if @lex_state == EXPR_ARG && ch !~ /\s/ ungetc @lex_state = EXPR_BEG; Token(TkQUESTION).set_text(op) else str = op str << ch if (ch == '\\') #' str << read_escape end @lex_state = EXPR_END Token(TkINTEGER).set_text(str) end end end @OP.def_rules("&", "&&", "\|", "\|\|") do \|op, io\| @lex_state = EXPR_BEG Token(op).set_text(op) end @OP.def_rules("+=", "-=", "=", "*=", "&=", "\|=", "^=", "<<=", ">>=", "\|\|=", "&&=") do \|op, io\| @lex_state = EXPR_BEG op =~ /^(.)=$/ Token(TkOPASGN, $1).set_text(op) end @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do \|op, io\| Token(TkUPLUS).set_text(op) end @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do \|op, io\| Token(TkUMINUS).set_text(op) end @OP.def_rules("+", "-") do \|op, io\| catch(:RET) do if @lex_state == EXPR_ARG if @space_seen and peek(0) =~ /[0-9]/ throw :RET, identify_number(op) else @lex_state = EXPR_BEG end elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/ throw :RET, identify_number(op) else @lex_state = EXPR_BEG end Token(op).set_text(op) end end @OP.def_rule(".") do @lex_state = EXPR_BEG if peek(0) =~ /[0-9]/ ungetc identify_number("") else # for obj.if @lex_state = EXPR_DOT Token(TkDOT).set_text(".") end end @OP.def_rules("..", "...") do \|op, io\| @lex_state = EXPR_BEG Token(op).set_text(op) end lex_int2 end def lex_int2 @OP.def_rules("]", "}", ")") do \|op, io\| @lex_state = EXPR_END @indent -= 1 Token(op).set_text(op) end @OP.def_rule(":") do if @lex_state == EXPR_END \|\| peek(0) =~ /\s/ @lex_state = EXPR_BEG tk = Token(TkCOLON) else @lex_state = EXPR_FNAME; tk = Token(TkSYMBEG) end tk.set_text(":") end @OP.def_rule("::") do # p @lex_state.id2name, @space_seen if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen @lex_state = EXPR_BEG tk = Token(TkCOLON3) else @lex_state = EXPR_DOT tk = Token(TkCOLON2) end tk.set_text("::") end @OP.def_rule("/") do \|op, io\| if @lex_state == EXPR_BEG \|\| @lex_state == EXPR_MID identify_string(op) elsif peek(0) == '=' getc @lex_state = EXPR_BEG Token(TkOPASGN, :/).set_text("/=") #") elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/ identify_string(op) else @lex_state = EXPR_BEG Token("/").set_text(op) end end @OP.def_rules("^") do @lex_state = EXPR_BEG Token("^").set_text("^") end # @OP.def_rules("^=") do # @lex_state = EXPR_BEG # Token(TkOPASGN, :^) # end @OP.def_rules(",", ";") do \|op, io\| @lex_state = EXPR_BEG Token(op).set_text(op) end @OP.def_rule("~") do @lex_state = EXPR_BEG Token("~").set_text("~") end @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do @lex_state = EXPR_BEG Token("~").set_text("~@") end @OP.def_rule("(") do @indent += 1 if @lex_state == EXPR_BEG \|\| @lex_state == EXPR_MID @lex_state = EXPR_BEG tk = Token(TkfLPAREN) else @lex_state = EXPR_BEG tk = Token(TkLPAREN) end tk.set_text("(") end @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do Token("[]").set_text("[]") end @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do Token("[]=").set_text("[]=") end @OP.def_rule("[") do @indent += 1 if @lex_state == EXPR_FNAME t = Token(TkfLBRACK) else if @lex_state == EXPR_BEG \|\| @lex_state == EXPR_MID t = Token(TkLBRACK) elsif @lex_state == EXPR_ARG && @space_seen t = Token(TkLBRACK) else t = Token(TkfLBRACK) end @lex_state = EXPR_BEG end t.set_text("[") end @OP.def_rule("{") do @indent += 1 if @lex_state != EXPR_END && @lex_state != EXPR_ARG t = Token(TkLBRACE) else t = Token(TkfLBRACE) end @lex_state = EXPR_BEG t.set_text("{") end @OP.def_rule('\\') do #' if getc == "\n" @space_seen = true @continue = true Token(TkSPACE).set_text("\\\n") else ungetc Token("\\").set_text("\\") #" end end @OP.def_rule('%') do \|op, io\| if @lex_state == EXPR_BEG \|\| @lex_state == EXPR_MID identify_quotation('%') elsif peek(0) == '=' getc Token(TkOPASGN, "%").set_text("%=") elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/ identify_quotation('%') else @lex_state = EXPR_BEG Token("%").set_text("%") end end @OP.def_rule('$') do #' identify_gvar end @OP.def_rule('@') do if peek(0) =~ /[@\w_]/ ungetc identify_identifier else Token("@").set_text("@") end end # @OP.def_rule("def", proc{\|op, io\| /\s/ =~ io.peek(0)}) do # \|op, io\| # @indent += 1 # @lex_state = EXPR_FNAME # # @lex_state = EXPR_END # # until @rests[0] == "\n" or @rests[0] == ";" # # rests.shift # # end # end @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do throw :eof end @OP.def_rule("") do \|op, io\| printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug? if peek(0) =~ /[0-9]/ t = identify_number("") elsif peek(0) =~ /[\w_]/ t = identify_identifier end printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug? t end p @OP if RubyLex.debug? end def identify_gvar @lex_state = EXPR_END str = "$" tk = case ch = getc when /[~_$?!@\/\\;,=:<>".]/ #" str << ch Token(TkGVAR, str) when "-" str << "-" << getc Token(TkGVAR, str) when "&", "`", "'", "+" str << ch Token(TkBACK_REF, str) when /[1-9]/ str << ch while (ch = getc) =~ /[0-9]/ str << ch end ungetc Token(TkNTH_REF) when /\w/ ungetc ungetc return identify_identifier else ungetc Token("$") end tk.set_text(str) end def identify_identifier token = "" token.concat getc if peek(0) =~ /[$@]/ token.concat getc if peek(0) == "@" while (ch = getc) =~ /\w\|_/ print ":", ch, ":" if RubyLex.debug? token.concat ch end ungetc if ch == "!" or ch == "?" token.concat getc end # fix token # $stderr.puts "identifier - #{token}, state = #@lex_state" case token when /^\$/ return Token(TkGVAR, token).set_text(token) when /^\@/ @lex_state = EXPR_END return Token(TkIVAR, token).set_text(token) end if @lex_state != EXPR_DOT print token, "\n" if RubyLex.debug? token_c, trans = TkReading2Token[token] if token_c # reserved word? if (@lex_state != EXPR_BEG && @lex_state != EXPR_FNAME && trans[1]) # modifiers token_c = TkSymbol2Token[trans[1]] @lex_state = trans[0] else if @lex_state != EXPR_FNAME if ENINDENT_CLAUSE.include?(token) @indent += 1 elsif DEINDENT_CLAUSE.include?(token) @indent -= 1 end @lex_state = trans[0] else @lex_state = EXPR_END end end return Token(token_c, token).set_text(token) end end if @lex_state == EXPR_FNAME @lex_state = EXPR_END if peek(0) == '=' token.concat getc end elsif @lex_state == EXPR_BEG \|\| @lex_state == EXPR_DOT @lex_state = EXPR_ARG else @lex_state = EXPR_END end if token[0, 1] =~ /[A-Z]/ return Token(TkCONSTANT, token).set_text(token) elsif token[token.size - 1, 1] =~ /[!?]/ return Token(TkFID, token).set_text(token) else return Token(TkIDENTIFIER, token).set_text(token) end end def identify_here_document ch = getc if ch == "-" ch = getc indent = true end if /['"`]/ =~ ch # ' lt = ch quoted = "" while (c = getc) && c != lt quoted.concat c end else lt = '"' quoted = ch.dup while (c = getc) && c =~ /\w/ quoted.concat c end ungetc end ltback, @ltype = @ltype, lt reserve = "" while ch = getc reserve << ch if ch == "\\" #" ch = getc reserve << ch elsif ch == "\n" break end end str = "" while (l = gets) l.chomp! l.strip! if indent break if l == quoted str << l.chomp << "\n" end @reader.divert_read_from(reserve) @ltype = ltback @lex_state = EXPR_END Token(Ltype2Token[lt], str).set_text(str.dump) end def identify_quotation(initial_char) ch = getc if lt = PERCENT_LTYPE[ch] initial_char += ch ch = getc elsif ch =~ /\W/ lt = "\"" else RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')" end # if ch !~ /\W/ # ungetc # next # end #@ltype = lt @quoted = ch unless @quoted = PERCENT_PAREN[ch] identify_string(lt, @quoted, ch, initial_char) end def identify_number(start) str = start.dup if start == "+" or start == "-" or start == "" start = getc str << start end @lex_state = EXPR_END if start == "0" if peek(0) == "x" ch = getc str << ch match = /[0-9a-f_]/ else match = /[0-7_]/ end while ch = getc if ch !~ match ungetc break else str << ch end end return Token(TkINTEGER).set_text(str) end type = TkINTEGER allow_point = TRUE allow_e = TRUE while ch = getc case ch when /[0-9_]/ str << ch when allow_point && "." type = TkFLOAT if peek(0) !~ /[0-9]/ ungetc break end str << ch allow_point = false when allow_e && "e", allow_e && "E" str << ch type = TkFLOAT if peek(0) =~ /[+-]/ str << getc end allow_e = false allow_point = false else ungetc break end end Token(type).set_text(str) end def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil) @ltype = ltype @quoted = quoted subtype = nil str = "" str << initial_char if initial_char str << (opener\|\|quoted) nest = 0 begin while ch = getc str << ch if @quoted == ch if nest == 0 break else nest -= 1 end elsif opener == ch nest += 1 elsif @ltype != "'" && @ltype != "]" and ch == "#" ch = getc if ch == "{" subtype = true str << ch << skip_inner_expression else ungetc(ch) end elsif ch == '\\' #' str << read_escape end end if @ltype == "/" if peek(0) =~ /i\|o\|n\|e\|s/ str << getc end end if subtype Token(DLtype2Token[ltype], str) else Token(Ltype2Token[ltype], str) end.set_text(str) ensure @ltype = nil @quoted = nil @lex_state = EXPR_END end end def skip_inner_expression res = "" nest = 0 while (ch = getc) res << ch if ch == '}' break if nest.zero? nest -= 1 elsif ch == '{' nest += 1 end end res end def identify_comment @ltype = "#" comment = "#" while ch = getc if ch == "\\" ch = getc if ch == "\n" ch = " " else comment << "\\" end else if ch == "\n" @ltype = nil ungetc break end end comment << ch end return Token(TkCOMMENT).set_text(comment) end def read_escape res = "" case ch = getc when /[0-7]/ ungetc ch 3.times do case ch = getc when /[0-7]/ when nil break else ungetc break end res << ch end when "x" res << ch 2.times do case ch = getc when /[0-9a-fA-F]/ when nil break else ungetc break end res << ch end when "M" res << ch if (ch = getc) != '-' ungetc else res << ch if (ch = getc) == "\\" #" res << ch res << read_escape else res << ch end end when "C", "c" #, "^" res << ch if ch == "C" and (ch = getc) != "-" ungetc else res << ch if (ch = getc) == "\\" #" res << ch res << read_escape else res << ch end end else res << ch end res end end # Extract code elements from a source file, returning a TopLevel # object containing the constituent file elements. # # This file is based on rtags module RDoc GENERAL_MODIFIERS = [ 'nodoc' ].freeze CLASS_MODIFIERS = GENERAL_MODIFIERS ATTR_MODIFIERS = GENERAL_MODIFIERS CONSTANT_MODIFIERS = GENERAL_MODIFIERS METHOD_MODIFIERS = GENERAL_MODIFIERS + [ 'arg', 'args', 'yield', 'yields', 'notnew', 'not-new', 'not_new', 'doc' ] class RubyParser include RubyToken include TokenStream extend ParserFactory parse_files_matching(/\.rbw?$/) def initialize(top_level, file_name, content, options, stats) @options = options @stats = stats @size = 0 @token_listeners = nil @input_file_name = file_name @scanner = RubyLex.new(content) @scanner.exception_on_syntax_error = false @top_level = top_level @progress = $stderr unless options.quiet end def scan @tokens = [] @unget_read = [] @read = [] catch(:eof) do catch(:enddoc) do begin parse_toplevel_statements(@top_level) rescue Exception => e $stderr.puts "\n\n" $stderr.puts "RDoc failure in #@input_file_name at or around " + "line #{@scanner.line_no} column #{@scanner.char_no}" $stderr.puts $stderr.puts "Before reporting this, could you check that the file" $stderr.puts "you're documenting compiles cleanly--RDoc is not a" $stderr.puts "full Ruby parser, and gets confused easily if fed" $stderr.puts "invalid programs." $stderr.puts $stderr.puts "The internal error was:\n\n" e.set_backtrace(e.backtrace[0,4]) raise end end end @top_level end private def make_message(msg) prefix = "\n" + @input_file_name + ":" if @scanner prefix << "#{@scanner.line_no}:#{@scanner.char_no}: " end return prefix + msg end def warn(msg) return if @options.quiet msg = make_message msg $stderr.puts msg end def error(msg) msg = make_message msg $stderr.puts msg exit(1) end def progress(char) unless @options.quiet @progress.print(char) @progress.flush end end def add_token_listener(obj) @token_listeners \|\|= [] @token_listeners << obj end def remove_token_listener(obj) @token_listeners.delete(obj) end def get_tk tk = nil if @tokens.empty? tk = @scanner.token @read.push @scanner.get_read puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG else @read.push @unget_read.shift tk = @tokens.shift puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG end if tk.kind_of?(TkSYMBEG) set_token_position(tk.line_no, tk.char_no) tk1 = get_tk if tk1.kind_of?(TkId) \|\| tk1.kind_of?(TkOp) tk = Token(TkSYMBOL).set_text(":" + tk1.name) # remove the identifier we just read (we're about to # replace it with a symbol) @token_listeners.each do \|obj\| obj.pop_token end if @token_listeners else warn("':' not followed by identifier or operator") tk = tk1 end end # inform any listeners of our shiny new token @token_listeners.each do \|obj\| obj.add_token(tk) end if @token_listeners tk end def peek_tk unget_tk(tk = get_tk) tk end def unget_tk(tk) @tokens.unshift tk @unget_read.unshift @read.pop # Remove this token from any listeners @token_listeners.each do \|obj\| obj.pop_token end if @token_listeners end def skip_tkspace(skip_nl = true) tokens = [] while ((tk = get_tk).kind_of?(TkSPACE) \|\| (skip_nl && tk.kind_of?(TkNL))) tokens.push tk end unget_tk(tk) tokens end def get_tkread read = @read.join("") @read = [] read end def peek_read @read.join('') end NORMAL = "::" SINGLE = "<<" # Look for the first comment in a file that isn't # a shebang line. def collect_first_comment skip_tkspace res = '' first_line = true tk = get_tk while tk.kind_of?(TkCOMMENT) if first_line && /\A#!/ =~ tk.text skip_tkspace tk = get_tk elsif first_line && /\A#\s-\-/ =~ tk.text first_line = false skip_tkspace tk = get_tk else first_line = false res << tk.text << "\n" tk = get_tk if tk.kind_of? TkNL skip_tkspace(false) tk = get_tk end end end unget_tk(tk) res end def parse_toplevel_statements(container) comment = collect_first_comment look_for_directives_in(container, comment) container.comment = comment unless comment.empty? parse_statements(container, NORMAL, nil, comment) end def parse_statements(container, single=NORMAL, current_method=nil, comment='') nest = 1 save_visibility = container.visibility # if container.kind_of?(TopLevel) # else # comment = '' # end non_comment_seen = true while tk = get_tk keep_comment = false non_comment_seen = true unless tk.kind_of?(TkCOMMENT) case tk when TkNL skip_tkspace(true) # Skip blanks and newlines tk = get_tk if tk.kind_of?(TkCOMMENT) if non_comment_seen comment = '' non_comment_seen = false end while tk.kind_of?(TkCOMMENT) comment << tk.text << "\n" tk = get_tk # this is the newline skip_tkspace(false) # leading spaces tk = get_tk end unless comment.empty? look_for_directives_in(container, comment) if container.done_documenting container.ongoing_visibility = save_visibility # return end end keep_comment = true else non_comment_seen = true end unget_tk(tk) keep_comment = true when TkCLASS if container.document_children parse_class(container, single, tk, comment) else nest += 1 end when TkMODULE if container.document_children parse_module(container, single, tk, comment) else nest += 1 end when TkDEF if container.document_self parse_method(container, single, tk, comment) else nest += 1 end when TkCONSTANT if container.document_self parse_constant(container, single, tk, comment) end when TkALIAS if container.document_self parse_alias(container, single, tk, comment) end when TkYIELD if current_method.nil? warn("Warning: yield outside of method") if container.document_self else parse_yield(container, single, tk, current_method) end # Until and While can have a 'do', which shouldn't increas # the nesting. We can't solve the general case, but we can # handle most occurrences by ignoring a do at the end of a line when TkUNTIL, TkWHILE nest += 1 puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " + "line #{tk.line_no}" if $DEBUG skip_optional_do_after_expression # 'for' is trickier when TkFOR nest += 1 puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " + "line #{tk.line_no}" if $DEBUG skip_for_variable skip_optional_do_after_expression when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN nest += 1 puts "Found #{tk.class} in #{container.name}, nest = #{nest}, " + "line #{tk.line_no}" if $DEBUG when TkIDENTIFIER if nest == 1 and current_method.nil? case tk.name when "private", "protected", "public", "private_class_method", "public_class_method" parse_visibility(container, single, tk) keep_comment = true when "attr" parse_attr(container, single, tk, comment) when /^attr_(reader\|writer\|accessor)$/, @options.extra_accessors parse_attr_accessor(container, single, tk, comment) when "alias_method" if container.document_self parse_alias(container, single, tk, comment) end end end case tk.name when "require" parse_require(container, comment) when "include" parse_include(container, comment) end when TkEND nest -= 1 puts "Found 'end' in #{container.name}, nest = #{nest}, line #{tk.line_no}" if $DEBUG puts "Method = #{current_method.name}" if $DEBUG and current_method if nest == 0 read_documentation_modifiers(container, CLASS_MODIFIERS) container.ongoing_visibility = save_visibility return end end comment = '' unless keep_comment begin get_tkread skip_tkspace(false) end while peek_tk == TkNL end end def parse_class(container, single, tk, comment, &block) progress("c") @stats.num_classes += 1 container, name_t = get_class_or_module(container) case name_t when TkCONSTANT name = name_t.name superclass = "Object" if peek_tk.kind_of?(TkLT) get_tk skip_tkspace(true) superclass = get_class_specification superclass = "<unknown>" if superclass.empty? end if single == SINGLE cls_type = SingleClass else cls_type = NormalClass end cls = container.add_class(cls_type, name, superclass) read_documentation_modifiers(cls, CLASS_MODIFIERS) cls.record_location(@top_level) parse_statements(cls) cls.comment = comment when TkLSHFT case name = get_class_specification when "self", container.name parse_statements(container, SINGLE, &block) else other = TopLevel.find_class_named(name) unless other # other = @top_level.add_class(NormalClass, name, nil) # other.record_location(@top_level) # other.comment = comment other = NormalClass.new("Dummy", nil) end read_documentation_modifiers(other, CLASS_MODIFIERS) parse_statements(other, SINGLE, &block) end else warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}") end end def parse_module(container, single, tk, comment) progress("m") @stats.num_modules += 1 container, name_t = get_class_or_module(container) # skip_tkspace name = name_t.name mod = container.add_module(NormalModule, name) mod.record_location(@top_level) read_documentation_modifiers(mod, CLASS_MODIFIERS) parse_statements(mod) mod.comment = comment end # Look for the name of a class of module (optionally with a leading :: or # with :: separated named) and return the ultimate name and container def get_class_or_module(container) skip_tkspace name_t = get_tk # class ::A -> A is in the top level if name_t.kind_of?(TkCOLON2) name_t = get_tk container = @top_level end skip_tkspace(false) while peek_tk.kind_of?(TkCOLON2) prev_container = container container = container.find_module_named(name_t.name) if !container # warn("Couldn't find module #{name_t.name}") container = prev_container.add_module(NormalModule, name_t.name) end get_tk name_t = get_tk end skip_tkspace(false) return [container, name_t] end def parse_constant(container, single, tk, comment) name = tk.name skip_tkspace(false) eq_tk = get_tk unless eq_tk.kind_of?(TkASSIGN) unget_tk(eq_tk) return end nest = 0 get_tkread tk = get_tk if tk.kind_of? TkGT unget_tk(tk) unget_tk(eq_tk) return end loop do puts("Param: #{tk}, #{@scanner.continue} " + "#{@scanner.lex_state} #{nest}") if $DEBUG case tk when TkSEMICOLON break when TkLPAREN, TkfLPAREN nest += 1 when TkRPAREN nest -= 1 when TkCOMMENT if nest <= 0 && @scanner.lex_state == EXPR_END unget_tk(tk) break end when TkNL if (@scanner.lex_state == EXPR_END and nest <= 0) \|\| !@scanner.continue unget_tk(tk) break end end tk = get_tk end res = get_tkread.tr("\n", " ").strip res = "" if res == ";" con = Constant.new(name, res, comment) read_documentation_modifiers(con, CONSTANT_MODIFIERS) if con.document_self container.add_constant(con) end end def parse_method(container, single, tk, comment) progress(".") @stats.num_methods += 1 line_no = tk.line_no column = tk.char_no start_collecting_tokens add_token(tk) add_token_listener(self) @scanner.instance_eval{@lex_state = EXPR_FNAME} skip_tkspace(false) name_t = get_tk back_tk = skip_tkspace meth = nil added_container = false dot = get_tk if dot.kind_of?(TkDOT) or dot.kind_of?(TkCOLON2) @scanner.instance_eval{@lex_state = EXPR_FNAME} skip_tkspace name_t2 = get_tk case name_t when TkSELF name = name_t2.name when TkCONSTANT name = name_t2.name prev_container = container container = container.find_module_named(name_t.name) if !container added_container = true obj = name_t.name.split("::").inject(Object) do \|state, item\| state.const_get(item) end rescue nil type = obj.class == Class ? NormalClass : NormalModule if not [Class, Module].include?(obj.class) warn("Couldn't find #{name_t.name}. Assuming it's a module") end if type == NormalClass then container = prev_container.add_class(type, name_t.name, obj.superclass.name) else container = prev_container.add_module(type, name_t.name) end end else # warn("Unexpected token '#{name_t2.inspect}'") # break skip_method(container) return end meth = AnyMethod.new(get_tkread, name) meth.singleton = true else unget_tk dot back_tk.reverse_each do \|tk\| unget_tk tk end name = name_t.name meth = AnyMethod.new(get_tkread, name) meth.singleton = (single == SINGLE) end remove_token_listener(self) meth.start_collecting_tokens indent = TkSPACE.new(1,1) indent.set_text(" " * column) meth.add_tokens([TkCOMMENT.new(line_no, 1, "# File #{@top_level.file_absolute_name}, line #{line_no}"), NEWLINE_TOKEN, indent]) meth.add_tokens(@token_stream) add_token_listener(meth) @scanner.instance_eval{@continue = false} parse_method_parameters(meth) if meth.document_self container.add_method(meth) elsif added_container container.document_self = false end # Having now read the method parameters and documentation modifiers, we # now know whether we have to rename #initialize to ::new if name == "initialize" && !meth.singleton if meth.dont_rename_initialize meth.visibility = :protected else meth.singleton = true meth.name = "new" meth.visibility = :public end end parse_statements(container, single, meth) remove_token_listener(meth) # Look for a 'call-seq' in the comment, and override the # normal parameter stuff if comment.sub!(/:?call-seq:(.?)^\s\#?\s$/m, '') seq = $1 seq.gsub!(/^\s\#\s/, '') meth.call_seq = seq end meth.comment = comment end def skip_method(container) meth = AnyMethod.new("", "anon") parse_method_parameters(meth) parse_statements(container, false, meth) end # Capture the method's parameters. Along the way, # look for a comment containing # # # yields: .... # # and add this as the block_params for the method def parse_method_parameters(method) res = parse_method_or_yield_parameters(method) res = "(" + res + ")" unless res[0] == ?( method.params = res unless method.params if method.block_params.nil? skip_tkspace(false) read_documentation_modifiers(method, METHOD_MODIFIERS) end end def parse_method_or_yield_parameters(method=nil, modifiers=METHOD_MODIFIERS) skip_tkspace(false) tk = get_tk # Little hack going on here. In the statement # f = 2(1+yield) # We see the RPAREN as the next token, so we need # to exit early. This still won't catch all cases # (such as "a = yield + 1" end_token = case tk when TkLPAREN, TkfLPAREN TkRPAREN when TkRPAREN return "" else TkNL end nest = 0 loop do puts("Param: #{tk.inspect}, #{@scanner.continue} " + "#{@scanner.lex_state} #{nest}") if $DEBUG case tk when TkSEMICOLON break when TkLBRACE nest += 1 when TkRBRACE # we might have a.each {\|i\| yield i } unget_tk(tk) if nest.zero? nest -= 1 break if nest <= 0 when TkLPAREN, TkfLPAREN nest += 1 when end_token if end_token == TkRPAREN nest -= 1 break if @scanner.lex_state == EXPR_END and nest <= 0 else break unless @scanner.continue end when method && method.block_params.nil? && TkCOMMENT unget_tk(tk) read_documentation_modifiers(method, modifiers) end tk = get_tk end res = get_tkread.tr("\n", " ").strip res = "" if res == ";" res end # skip the var [in] part of a 'for' statement def skip_for_variable skip_tkspace(false) tk = get_tk skip_tkspace(false) tk = get_tk unget_tk(tk) unless tk.kind_of?(TkIN) end # while, until, and for have an optional def skip_optional_do_after_expression skip_tkspace(false) tk = get_tk case tk when TkLPAREN, TkfLPAREN end_token = TkRPAREN else end_token = TkNL end nest = 0 @scanner.instance_eval{@continue = false} loop do puts("\nWhile: #{tk}, #{@scanner.continue} " + "#{@scanner.lex_state} #{nest}") if $DEBUG case tk when TkSEMICOLON break when TkLPAREN, TkfLPAREN nest += 1 when TkDO break if nest.zero? when end_token if end_token == TkRPAREN nest -= 1 break if @scanner.lex_state == EXPR_END and nest.zero? else break unless @scanner.continue end end tk = get_tk end skip_tkspace(false) if peek_tk.kind_of? TkDO get_tk end end # Return a superclass, which can be either a constant # of an expression def get_class_specification tk = get_tk return "self" if tk.kind_of?(TkSELF) res = "" while tk.kind_of?(TkCOLON2) \|\| tk.kind_of?(TkCOLON3) \|\| tk.kind_of?(TkCONSTANT) res += tk.text tk = get_tk end unget_tk(tk) skip_tkspace(false) get_tkread # empty out read buffer tk = get_tk case tk when TkNL, TkCOMMENT, TkSEMICOLON unget_tk(tk) return res end res += parse_call_parameters(tk) res end def parse_call_parameters(tk) end_token = case tk when TkLPAREN, TkfLPAREN TkRPAREN when TkRPAREN return "" else TkNL end nest = 0 loop do puts("Call param: #{tk}, #{@scanner.continue} " + "#{@scanner.lex_state} #{nest}") if $DEBUG case tk when TkSEMICOLON break when TkLPAREN, TkfLPAREN nest += 1 when end_token if end_token == TkRPAREN nest -= 1 break if @scanner.lex_state == EXPR_END and nest <= 0 else break unless @scanner.continue end when TkCOMMENT unget_tk(tk) break end tk = get_tk end res = get_tkread.tr("\n", " ").strip res = "" if res == ";" res end # Parse a constant, which might be qualified by # one or more class or module names def get_constant res = "" skip_tkspace(false) tk = get_tk while tk.kind_of?(TkCOLON2) \|\| tk.kind_of?(TkCOLON3) \|\| tk.kind_of?(TkCONSTANT) res += tk.text tk = get_tk end # if res.empty? # warn("Unexpected token #{tk} in constant") # end unget_tk(tk) res end # Get a constant that may be surrounded by parens def get_constant_with_optional_parens skip_tkspace(false) nest = 0 while (tk = peek_tk).kind_of?(TkLPAREN) \|\| tk.kind_of?(TkfLPAREN) get_tk skip_tkspace(true) nest += 1 end name = get_constant while nest > 0 skip_tkspace(true) tk = get_tk nest -= 1 if tk.kind_of?(TkRPAREN) end name end # Directives are modifier comments that can appear after class, module, # or method names. For example # # def fred # :yields: a, b # # or # # class SM # :nodoc: # # we return the directive name and any parameters as a two element array def read_directive(allowed) tk = get_tk puts "directive: #{tk.inspect}" if $DEBUG result = nil if tk.kind_of?(TkCOMMENT) if tk.text =~ /\s:?(\w+):\s(.)/ directive = $1.downcase if allowed.include?(directive) result = [directive, $2] end end else unget_tk(tk) end result end def read_documentation_modifiers(context, allow) dir = read_directive(allow) case dir[0] when "notnew", "not_new", "not-new" context.dont_rename_initialize = true when "nodoc" context.document_self = false if dir[1].downcase == "all" context.document_children = false end when "doc" context.document_self = true context.force_documentation = true when "yield", "yields" unless context.params.nil? context.params.sub!(/(,\|)\s&\w+/,'') # remove parameter &proc end context.block_params = dir[1] when "arg", "args" context.params = dir[1] end if dir end # Look for directives in a normal comment block: # # #-- - don't display comment from this point forward # # # This routine modifies it's parameter def look_for_directives_in(context, comment) preprocess = SM::PreProcess.new(@input_file_name, @options.rdoc_include) preprocess.handle(comment) do \|directive, param\| case directive when "stopdoc" context.stop_doc "" when "startdoc" context.start_doc context.force_documentation = true "" when "enddoc" #context.done_documenting = true #"" throw :enddoc when "main" options = Options.instance options.main_page = param "" when "title" options = Options.instance options.title = param "" when "section" context.set_current_section(param, comment) comment.replace("") # 1.8 doesn't support #clear break else warn "Unrecognized directive '#{directive}'" break end end remove_private_comments(comment) end def remove_private_comments(comment) comment.gsub!(/^#--.?^#\+\+/m, '') comment.sub!(/^#--./m, '') end def get_symbol_or_name tk = get_tk case tk when TkSYMBOL tk.text.sub(/^:/, '') when TkId, TkOp tk.name when TkSTRING tk.text else raise "Name or symbol expected (got #{tk})" end end def parse_alias(context, single, tk, comment) skip_tkspace if (peek_tk.kind_of? TkLPAREN) get_tk skip_tkspace end new_name = get_symbol_or_name @scanner.instance_eval{@lex_state = EXPR_FNAME} skip_tkspace if (peek_tk.kind_of? TkCOMMA) get_tk skip_tkspace end old_name = get_symbol_or_name al = Alias.new(get_tkread, old_name, new_name, comment) read_documentation_modifiers(al, ATTR_MODIFIERS) if al.document_self context.add_alias(al) end end def parse_yield_parameters parse_method_or_yield_parameters end def parse_yield(context, single, tk, method) if method.block_params.nil? get_tkread @scanner.instance_eval{@continue = false} method.block_params = parse_yield_parameters end end def parse_require(context, comment) skip_tkspace_comment tk = get_tk if tk.kind_of? TkLPAREN skip_tkspace_comment tk = get_tk end name = nil case tk when TkSTRING name = tk.text # when TkCONSTANT, TkIDENTIFIER, TkIVAR, TkGVAR # name = tk.name when TkDSTRING warn "Skipping require of dynamic string: #{tk.text}" # else # warn "'require' used as variable" end if name context.add_require(Require.new(name, comment)) else unget_tk(tk) end end def parse_include(context, comment) loop do skip_tkspace_comment name = get_constant_with_optional_parens unless name.empty? context.add_include(Include.new(name, comment)) end return unless peek_tk.kind_of?(TkCOMMA) get_tk end end def get_bool skip_tkspace tk = get_tk case tk when TkTRUE true when TkFALSE, TkNIL false else unget_tk tk true end end def parse_attr(context, single, tk, comment) args = parse_symbol_arg(1) if args.size > 0 name = args[0] rw = "R" skip_tkspace(false) tk = get_tk if tk.kind_of? TkCOMMA rw = "RW" if get_bool else unget_tk tk end att = Attr.new(get_tkread, name, rw, comment) read_documentation_modifiers(att, ATTR_MODIFIERS) if att.document_self context.add_attribute(att) end else warn("'attr' ignored - looks like a variable") end end def parse_visibility(container, single, tk) singleton = (single == SINGLE) vis = case tk.name when "private" then :private when "protected" then :protected when "public" then :public when "private_class_method" singleton = true :private when "public_class_method" singleton = true :public else raise "Invalid visibility: #{tk.name}" end skip_tkspace_comment(false) case peek_tk # Ryan Davis suggested the extension to ignore modifiers, because he # often writes # # protected unless $TESTING # when TkNL, TkUNLESS_MOD, TkIF_MOD # error("Missing argument") if singleton container.ongoing_visibility = vis else args = parse_symbol_arg container.set_visibility_for(args, vis, singleton) end end def parse_attr_accessor(context, single, tk, comment) args = parse_symbol_arg read = get_tkread rw = "?" # If nodoc is given, don't document any of them tmp = CodeObject.new read_documentation_modifiers(tmp, ATTR_MODIFIERS) return unless tmp.document_self case tk.name when "attr_reader" then rw = "R" when "attr_writer" then rw = "W" when "attr_accessor" then rw = "RW" else rw = @options.extra_accessor_flags[tk.name] end for name in args att = Attr.new(get_tkread, name, rw, comment) context.add_attribute(att) end end def skip_tkspace_comment(skip_nl = true) loop do skip_tkspace(skip_nl) return unless peek_tk.kind_of? TkCOMMENT get_tk end end def parse_symbol_arg(no = nil) args = [] skip_tkspace_comment case tk = get_tk when TkLPAREN loop do skip_tkspace_comment if tk1 = parse_symbol_in_arg args.push tk1 break if no and args.size >= no end skip_tkspace_comment case tk2 = get_tk when TkRPAREN break when TkCOMMA else warn("unexpected token: '#{tk2.inspect}'") if $DEBUG break end end else unget_tk tk if tk = parse_symbol_in_arg args.push tk return args if no and args.size >= no end loop do # skip_tkspace_comment(false) skip_tkspace(false) tk1 = get_tk unless tk1.kind_of?(TkCOMMA) unget_tk tk1 break end skip_tkspace_comment if tk = parse_symbol_in_arg args.push tk break if no and args.size >= no end end end args end def parse_symbol_in_arg case tk = get_tk when TkSYMBOL tk.text.sub(/^:/, '') when TkSTRING eval @read[-1] else warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG nil end end end end Back

Edit File : parse_rb.rb | Size : 58.81 KB

#!/usr/local/bin/ruby

# Parse a Ruby source file, building a set of objects
# representing the modules, classes, methods,
# requires, and includes we find (these classes
# are defined in code_objects.rb).

# This file contains stuff stolen outright from:
#
#   rtags.rb - 
#   ruby-lex.rb - ruby lexcal analizer
#   ruby-token.rb - ruby tokens 
#   	by Keiju ISHITSUKA (Nippon Rational Inc.)
#

require "e2mmap"
require "irb/slex"

require "rdoc/code_objects"
require "rdoc/tokenstream"

require "rdoc/markup/simple_markup/preprocess"

require "rdoc/parsers/parserfactory"

$TOKEN_DEBUG = $DEBUG

# Definitions of all tokens involved in the lexical analysis

module RubyToken
  EXPR_BEG   = :EXPR_BEG
  EXPR_MID   = :EXPR_MID
  EXPR_END   = :EXPR_END
  EXPR_ARG   = :EXPR_ARG
  EXPR_FNAME = :EXPR_FNAME
  EXPR_DOT   = :EXPR_DOT
  EXPR_CLASS = :EXPR_CLASS
  
  class Token
    NO_TEXT = "??".freeze
    attr :text

def initialize(line_no, char_no)
      @line_no = line_no
      @char_no = char_no
      @text    = NO_TEXT
    end

# Because we're used in contexts that expect to return a token,
    # we set the text string and then return ourselves
    def set_text(text)
      @text = text
      self
    end

attr_reader :line_no, :char_no, :text
  end

class TkNode < Token
    attr :node
  end

class TkId < Token
    def initialize(line_no, char_no, name)
      super(line_no, char_no)
      @name = name
    end
    attr :name
  end

class TkKW < TkId
  end

class TkVal < Token
    def initialize(line_no, char_no, value = nil)
      super(line_no, char_no)
      set_text(value)
    end
  end

class TkOp < Token
    def name
      self.class.op_name
    end
  end

class TkOPASGN < TkOp
    def initialize(line_no, char_no, op)
      super(line_no, char_no)
      op = TkReading2Token[op] unless op.kind_of?(Symbol)
      @op = op
    end
    attr :op
  end

class TkUnknownChar < Token
    def initialize(line_no, char_no, id)
      super(line_no, char_no)
      @name = char_no.chr
    end
    attr :name
  end

class TkError < Token
  end

def set_token_position(line, char)
    @prev_line_no = line
    @prev_char_no = char
  end

def Token(token, value = nil)
    tk = nil
    case token
    when String, Symbol
      source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
      if (tk = source[token]).nil?
	IRB.fail TkReading2TokenNoKey, token
      end
      tk = Token(tk[0], value) 
    else 
      tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
             token.new(@prev_line_no, @prev_char_no)
           else
             token.new(@prev_line_no, @prev_char_no, value)
           end
    end
    tk
  end

TokenDefinitions = [
    [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
    [:TkMODULE,     TkKW,  "module", EXPR_BEG],
    [:TkDEF,	    TkKW,  "def",    EXPR_FNAME],
    [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
    [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
    [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
    [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
    [:TkEND,	    TkKW,  "end",    EXPR_END],
    [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
    [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
    [:TkTHEN,	    TkKW,  "then",   EXPR_BEG],
    [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
    [:TkELSE,	    TkKW,  "else",   EXPR_BEG],
    [:TkCASE,	    TkKW,  "case",   EXPR_BEG],
    [:TkWHEN,	    TkKW,  "when",   EXPR_BEG],
    [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
    [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
    [:TkFOR,	    TkKW,  "for",    EXPR_BEG],
    [:TkBREAK,      TkKW,  "break",  EXPR_END],
    [:TkNEXT,	    TkKW,  "next",   EXPR_END],
    [:TkREDO,	    TkKW,  "redo",   EXPR_END],
    [:TkRETRY,      TkKW,  "retry",  EXPR_END],
    [:TkIN,	    TkKW,  "in",     EXPR_BEG],
    [:TkDO,	    TkKW,  "do",     EXPR_BEG],
    [:TkRETURN,     TkKW,  "return", EXPR_MID],
    [:TkYIELD,      TkKW,  "yield",  EXPR_END],
    [:TkSUPER,      TkKW,  "super",  EXPR_END],
    [:TkSELF,	    TkKW,  "self",   EXPR_END],
    [:TkNIL, 	    TkKW,  "nil",    EXPR_END],
    [:TkTRUE,	    TkKW,  "true",   EXPR_END],
    [:TkFALSE,      TkKW,  "false",  EXPR_END],
    [:TkAND,	    TkKW,  "and",    EXPR_BEG],
    [:TkOR, 	    TkKW,  "or",     EXPR_BEG],
    [:TkNOT,	    TkKW,  "not",    EXPR_BEG],
    [:TkIF_MOD,     TkKW],
    [:TkUNLESS_MOD, TkKW],
    [:TkWHILE_MOD,  TkKW],
    [:TkUNTIL_MOD,  TkKW],
    [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
    [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
    [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
    [:TklEND,	    TkKW,  "END",      EXPR_END],
    [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
    [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],

[:TkIDENTIFIER, TkId],
    [:TkFID,	    TkId],
    [:TkGVAR,	    TkId],
    [:TkIVAR,	    TkId],
    [:TkCONSTANT,   TkId],

[:TkINTEGER,    TkVal],
    [:TkFLOAT,      TkVal],
    [:TkSTRING,     TkVal],
    [:TkXSTRING,    TkVal],
    [:TkREGEXP,     TkVal],
    [:TkCOMMENT,    TkVal],

[:TkDSTRING,    TkNode],
    [:TkDXSTRING,   TkNode],
    [:TkDREGEXP,    TkNode],
    [:TkNTH_REF,    TkId],
    [:TkBACK_REF,   TkId],

[:TkUPLUS,      TkOp,   "+@"],
    [:TkUMINUS,     TkOp,   "-@"],
    [:TkPOW,	    TkOp,   "**"],
    [:TkCMP,	    TkOp,   "<=>"],
    [:TkEQ,	    TkOp,   "=="],
    [:TkEQQ,	    TkOp,   "==="],
    [:TkNEQ,	    TkOp,   "!="],
    [:TkGEQ,	    TkOp,   ">="],
    [:TkLEQ,	    TkOp,   "<="],
    [:TkANDOP,      TkOp,   "&&"],
    [:TkOROP,	    TkOp,   "||"],
    [:TkMATCH,      TkOp,   "=~"],
    [:TkNMATCH,     TkOp,   "!~"],
    [:TkDOT2,	    TkOp,   ".."],
    [:TkDOT3,	    TkOp,   "..."],
    [:TkAREF,	    TkOp,   "[]"],
    [:TkASET,	    TkOp,   "[]="],
    [:TkLSHFT,      TkOp,   "<<"],
    [:TkRSHFT,      TkOp,   ">>"],
    [:TkCOLON2,     TkOp],
    [:TkCOLON3,     TkOp],
#   [:OPASGN,	    TkOp],               # +=, -=  etc. #
    [:TkASSOC,      TkOp,   "=>"],
    [:TkQUESTION,   TkOp,   "?"],	 #?
    [:TkCOLON,      TkOp,   ":"],        #:
    
    [:TkfLPAREN],         # func( #
    [:TkfLBRACK],         # func[ #
    [:TkfLBRACE],         # func{ #
    [:TkSTAR],            # *arg
    [:TkAMPER],           # &arg #
    [:TkSYMBOL,     TkId],          # :SYMBOL
    [:TkSYMBEG,     TkId], 
    [:TkGT,	    TkOp,   ">"],
    [:TkLT,	    TkOp,   "<"],
    [:TkPLUS,	    TkOp,   "+"],
    [:TkMINUS,      TkOp,   "-"],
    [:TkMULT,	    TkOp,   "*"],
    [:TkDIV,	    TkOp,   "/"],
    [:TkMOD,	    TkOp,   "%"],
    [:TkBITOR,      TkOp,   "|"],
    [:TkBITXOR,     TkOp,   "^"],
    [:TkBITAND,     TkOp,   "&"],
    [:TkBITNOT,     TkOp,   "~"],
    [:TkNOTOP,      TkOp,   "!"],

[:TkBACKQUOTE,  TkOp,   "`"],

[:TkASSIGN,     Token,  "="],
    [:TkDOT,	    Token,  "."],
    [:TkLPAREN,     Token,  "("],  #(exp)
    [:TkLBRACK,     Token,  "["],  #[arry]
    [:TkLBRACE,     Token,  "{"],  #{hash}
    [:TkRPAREN,     Token,  ")"],
    [:TkRBRACK,     Token,  "]"],
    [:TkRBRACE,     Token,  "}"],
    [:TkCOMMA,      Token,  ","],
    [:TkSEMICOLON,  Token,  ";"],

[:TkRD_COMMENT],
    [:TkSPACE],
    [:TkNL],
    [:TkEND_OF_SCRIPT],

[:TkBACKSLASH,  TkUnknownChar,  "\\"],
    [:TkAT,	    TkUnknownChar,  "@"],
    [:TkDOLLAR,     TkUnknownChar,  "\$"], #"
  ]

# {reading => token_class}
  # {reading => [token_class, *opt]}
  TkReading2Token = {}
  TkSymbol2Token = {}

def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
    token_n = token_n.id2name unless token_n.kind_of?(String)
    if RubyToken.const_defined?(token_n)
      IRB.fail AlreadyDefinedToken, token_n
    end

token_c =  Class.new super_token
    RubyToken.const_set token_n, token_c
#    token_c.inspect
 
    if reading
      if TkReading2Token[reading]
	IRB.fail TkReading2TokenDuplicateError, token_n, reading
      end
      if opts.empty?
	TkReading2Token[reading] = [token_c]
      else
	TkReading2Token[reading] = [token_c].concat(opts)
      end
    end
    TkSymbol2Token[token_n.intern] = token_c

if token_c <= TkOp
      token_c.class_eval %{
        def self.op_name; "#{reading}"; end
      }
    end
  end

for defs in TokenDefinitions
    def_token(*defs)
  end

NEWLINE_TOKEN = TkNL.new(0,0)
  NEWLINE_TOKEN.set_text("\n")

end

# Lexical analyzer for Ruby source

class RubyLex

######################################################################
  #
  # Read an input stream character by character. We allow for unlimited
  # ungetting of characters just read.
  #
  # We simplify the implementation greatly by reading the entire input
  # into a buffer initially, and then simply traversing it using
  # pointers.
  #
  # We also have to allow for the <i>here document diversion</i>. This
  # little gem comes about when the lexer encounters a here
  # document. At this point we effectively need to split the input
  # stream into two parts: one to read the body of the here document,
  # the other to read the rest of the input line where the here
  # document was initially encountered. For example, we might have
  #
  #   do_something(<<-A, <<-B)
  #     stuff
  #     for
  #   A
  #     stuff
  #     for
  #   B
  #
  # When the lexer encounters the <<A, it reads until the end of the
  # line, and keeps it around for later. It then reads the body of the
  # here document.  Once complete, it needs to read the rest of the
  # original line, but then skip the here document body.
  #
  
  class BufferedReader
    
    attr_reader :line_num
    
    def initialize(content)
      if /\t/ =~ content
        tab_width = Options.instance.tab_width
        content = content.split(/\n/).map do |line|
          1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)}  && $~ #`
          line
        end .join("\n")
      end
      @content   = content
      @content << "\n" unless @content[-1,1] == "\n"
      @size      = @content.size
      @offset    = 0
      @hwm       = 0
      @line_num  = 1
      @read_back_offset = 0
      @last_newline = 0
      @newline_pending = false
    end
    
    def column
      @offset - @last_newline
    end
    
    def getc
      return nil if @offset >= @size
      ch = @content[@offset, 1]
      
      @offset += 1
      @hwm = @offset if @hwm < @offset
      
      if @newline_pending
        @line_num += 1
        @last_newline = @offset - 1
        @newline_pending = false
      end
      
      if ch == "\n"
        @newline_pending = true
      end
      ch
    end
    
    def getc_already_read
      getc
    end
    
    def ungetc(ch)
      raise "unget past beginning of file" if @offset <= 0
      @offset -= 1
      if @content[@offset] == ?\n
        @newline_pending = false
      end
    end
    
    def get_read
      res = @content[@read_back_offset...@offset]
      @read_back_offset = @offset
      res
    end
    
    def peek(at)
      pos = @offset + at
      if pos >= @size
        nil
      else
        @content[pos, 1]
      end
    end
    
    def peek_equal(str)
      @content[@offset, str.length] == str
    end
    
    def divert_read_from(reserve)
      @content[@offset, 0] = reserve
      @size      = @content.size
    end
  end

# end of nested class BufferedReader

extend Exception2MessageMapper
  def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
  def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
  def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
  def_exception(:TkReading2TokenDuplicateError, 
		"key duplicate(token_n='%s', key='%s')")
  def_exception(:SyntaxError, "%s")
  
  include RubyToken
  include IRB

attr_reader :continue
  attr_reader :lex_state

def RubyLex.debug?
    false
  end

def initialize(content)
    lex_init

@reader = BufferedReader.new(content)

@exp_line_no = @line_no = 1
    @base_char_no = 0
    @indent = 0

@ltype = nil
    @quoted = nil
    @lex_state = EXPR_BEG
    @space_seen = false
    
    @continue = false
    @line = ""

@skip_space = false
    @read_auto_clean_up = false
    @exception_on_syntax_error = true
  end

attr :skip_space, true
  attr :read_auto_clean_up, true
  attr :exception_on_syntax_error, true

attr :indent

# io functions
  def line_no
    @reader.line_num
  end

def char_no
    @reader.column
  end

def get_read
    @reader.get_read
  end

def getc
    @reader.getc
  end

def getc_of_rests
    @reader.getc_already_read
  end

def gets
    c = getc or return
    l = ""
    begin
      l.concat c unless c == "\r"
      break if c == "\n"
    end while c = getc
    l
  end

def ungetc(c = nil)
    @reader.ungetc(c)
  end

def peek_equal?(str)
    @reader.peek_equal(str)
  end

def peek(i = 0)
    @reader.peek(i)
  end

def lex
    until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
	     !@continue or
	     tk.nil?)
    end
    line = get_read

if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
      nil
    else
      line
    end
  end

def token
    set_token_position(line_no, char_no)
    begin
      begin
	tk = @OP.match(self)
	@space_seen = tk.kind_of?(TkSPACE)
      rescue SyntaxError
	abort if @exception_on_syntax_error
	tk = TkError.new(line_no, char_no)
      end
    end while @skip_space and tk.kind_of?(TkSPACE)
    if @read_auto_clean_up
      get_read
    end
#   throw :eof unless tk
    p tk if $DEBUG
    tk
  end
  
  ENINDENT_CLAUSE = [
    "case", "class", "def", "do", "for", "if",
    "module", "unless", "until", "while", "begin" #, "when"
  ]
  DEINDENT_CLAUSE = ["end" #, "when"
  ]

PERCENT_LTYPE = {
    "q" => "\'",
    "Q" => "\"",
    "x" => "\`",
    "r" => "/",
    "w" => "]"
  }
  
  PERCENT_PAREN = {
    "{" => "}",
    "[" => "]",
    "<" => ">",
    "(" => ")"
  }

Ltype2Token = {
    "\'" => TkSTRING,
    "\"" => TkSTRING,
    "\`" => TkXSTRING,
    "/" => TkREGEXP,
    "]" => TkDSTRING
  }
  Ltype2Token.default = TkSTRING

DLtype2Token = {
    "\"" => TkDSTRING,
    "\`" => TkDXSTRING,
    "/" => TkDREGEXP,
  }

def lex_init()
    @OP = SLex.new
    @OP.def_rules("\0", "\004", "\032") do |chars, io|
      Token(TkEND_OF_SCRIPT).set_text(chars)
    end

@OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
      @space_seen = TRUE
      while (ch = getc) =~ /[ \t\f\r\13]/
        chars << ch
      end
      ungetc
      Token(TkSPACE).set_text(chars)
    end

@OP.def_rule("#") do
      |op, io|
      identify_comment
    end

@OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
      |op, io|
      str = op
      @ltype = "="

begin
        line = ""
        begin
          ch = getc
          line << ch
        end until ch == "\n"
        str << line
      end until line =~ /^=end/

ungetc

@ltype = nil

if str =~ /\A=begin\s+rdoc/i
        str.sub!(/\A=begin.*\n/, '')
        str.sub!(/^=end.*/m, '')
        Token(TkCOMMENT).set_text(str)
      else
        Token(TkRD_COMMENT)#.set_text(str)
      end
    end

@OP.def_rule("\n") do
      print "\\n\n" if RubyLex.debug?
      case @lex_state
      when EXPR_BEG, EXPR_FNAME, EXPR_DOT
	@continue = TRUE
      else
	@continue = FALSE
	@lex_state = EXPR_BEG
      end
      Token(TkNL).set_text("\n")
    end

@OP.def_rules("*", "**",	
		  "!", "!=", "!~",
		  "=", "==", "===", 
		  "=~", "<=>",	
		  "<", "<=",
		  ">", ">=", ">>") do
      |op, io|
      @lex_state = EXPR_BEG
      Token(op).set_text(op)
    end

@OP.def_rules("<<") do
      |op, io|
      tk = nil
      if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
	  (@lex_state != EXPR_ARG || @space_seen)
	c = peek(0)
	if /[-\w_\"\'\`]/ =~ c
	  tk = identify_here_document
	end
      end
      if !tk
        @lex_state = EXPR_BEG
        tk = Token(op).set_text(op)
      end
      tk
    end

@OP.def_rules("'", '"') do
      |op, io|
      identify_string(op)
    end

@OP.def_rules("`") do
      |op, io|
      if @lex_state == EXPR_FNAME
	Token(op).set_text(op)
      else
	identify_string(op)
      end
    end

@OP.def_rules('?') do
      |op, io|
      if @lex_state == EXPR_END
	@lex_state = EXPR_BEG
	Token(TkQUESTION).set_text(op)
      else
	ch = getc
	if @lex_state == EXPR_ARG && ch !~ /\s/
	  ungetc
	  @lex_state = EXPR_BEG;
	  Token(TkQUESTION).set_text(op)
	else
          str = op
          str << ch
	  if (ch == '\\') #'
	    str << read_escape
	  end
	  @lex_state = EXPR_END
	  Token(TkINTEGER).set_text(str)
	end
      end
    end

@OP.def_rules("&", "&&", "|", "||") do
      |op, io|
      @lex_state = EXPR_BEG
      Token(op).set_text(op)
    end
    
    @OP.def_rules("+=", "-=", "*=", "**=", 
		  "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
      |op, io|
      @lex_state = EXPR_BEG
      op =~ /^(.*)=$/
      Token(TkOPASGN, $1).set_text(op)
    end

@OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
      Token(TkUPLUS).set_text(op)
    end

@OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
      Token(TkUMINUS).set_text(op)
    end

@OP.def_rules("+", "-") do
      |op, io|
      catch(:RET) do
	if @lex_state == EXPR_ARG
	  if @space_seen and peek(0) =~ /[0-9]/
	    throw :RET, identify_number(op)
	  else
	    @lex_state = EXPR_BEG
	  end
	elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
	  throw :RET, identify_number(op)
	else
	  @lex_state = EXPR_BEG
	end
	Token(op).set_text(op)
      end
    end

@OP.def_rule(".") do
      @lex_state = EXPR_BEG
      if peek(0) =~ /[0-9]/
	ungetc
	identify_number("")
      else
	# for obj.if
	@lex_state = EXPR_DOT
	Token(TkDOT).set_text(".")
      end
    end

@OP.def_rules("..", "...") do
      |op, io|
      @lex_state = EXPR_BEG
      Token(op).set_text(op)
    end

lex_int2
  end
  
  def lex_int2
    @OP.def_rules("]", "}", ")") do
      |op, io|
      @lex_state = EXPR_END
      @indent -= 1
      Token(op).set_text(op)
    end

@OP.def_rule(":") do
      if @lex_state == EXPR_END || peek(0) =~ /\s/
	@lex_state = EXPR_BEG
	tk = Token(TkCOLON)
      else
	@lex_state = EXPR_FNAME;
	tk = Token(TkSYMBEG)
      end
      tk.set_text(":")
    end

@OP.def_rule("::") do
#      p @lex_state.id2name, @space_seen
      if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
	@lex_state = EXPR_BEG
	tk = Token(TkCOLON3)
      else
	@lex_state = EXPR_DOT
	tk = Token(TkCOLON2)
      end
      tk.set_text("::")
    end

@OP.def_rule("/") do
      |op, io|
      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
	identify_string(op)
      elsif peek(0) == '='
	getc
	@lex_state = EXPR_BEG
	Token(TkOPASGN, :/).set_text("/=") #")
      elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
	identify_string(op)
      else 
	@lex_state = EXPR_BEG
        Token("/").set_text(op)
      end
    end

@OP.def_rules("^") do
      @lex_state = EXPR_BEG
      Token("^").set_text("^")
    end

#       @OP.def_rules("^=") do
    # 	@lex_state = EXPR_BEG
    # 	Token(TkOPASGN, :^)
    #       end
    
    @OP.def_rules(",", ";") do
      |op, io|
      @lex_state = EXPR_BEG
      Token(op).set_text(op)
    end

@OP.def_rule("~") do
      @lex_state = EXPR_BEG
      Token("~").set_text("~")
    end

@OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
      @lex_state = EXPR_BEG
      Token("~").set_text("~@")
    end
    
    @OP.def_rule("(") do
      @indent += 1
      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
	@lex_state = EXPR_BEG
	tk = Token(TkfLPAREN)
      else
	@lex_state = EXPR_BEG
	tk = Token(TkLPAREN)
      end
      tk.set_text("(")
    end

@OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
      Token("[]").set_text("[]")
    end

@OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
      Token("[]=").set_text("[]=")
    end

@OP.def_rule("[") do
      @indent += 1
      if @lex_state == EXPR_FNAME
	t = Token(TkfLBRACK)
      else
	if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
	  t = Token(TkLBRACK)
	elsif @lex_state == EXPR_ARG && @space_seen
	  t = Token(TkLBRACK)
	else
	  t = Token(TkfLBRACK)
	end
	@lex_state = EXPR_BEG
      end
      t.set_text("[")
    end

@OP.def_rule("{") do
      @indent += 1
      if @lex_state != EXPR_END && @lex_state != EXPR_ARG
	t = Token(TkLBRACE)
      else
	t = Token(TkfLBRACE)
      end
      @lex_state = EXPR_BEG
      t.set_text("{")
    end

@OP.def_rule('\\') do   #'
      if getc == "\n" 
	@space_seen = true
	@continue = true
	Token(TkSPACE).set_text("\\\n")
      else 
	ungetc
	Token("\\").set_text("\\")  #"
      end 
    end

@OP.def_rule('%') do
      |op, io|
      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
	identify_quotation('%')
      elsif peek(0) == '='
	getc
	Token(TkOPASGN, "%").set_text("%=")
      elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
	identify_quotation('%')
      else
	@lex_state = EXPR_BEG
	Token("%").set_text("%")
      end
    end

@OP.def_rule('$') do  #'
      identify_gvar
    end

@OP.def_rule('@') do
      if peek(0) =~ /[@\w_]/
	ungetc
	identify_identifier
      else
	Token("@").set_text("@")
      end
    end

#       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do 
    # 	|op, io|
    # 	@indent += 1
    # 	@lex_state = EXPR_FNAME
    # #	@lex_state = EXPR_END
    # #	until @rests[0] == "\n" or @rests[0] == ";"
    # #	  rests.shift
    # #	end
    #       end

@OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
      throw :eof
    end

@OP.def_rule("") do
      |op, io|
      printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
      if peek(0) =~ /[0-9]/
	t = identify_number("")
      elsif peek(0) =~ /[\w_]/
	t = identify_identifier
      end
      printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
      t
    end
    
    p @OP if RubyLex.debug?
  end
  
  def identify_gvar
    @lex_state = EXPR_END
    str = "$"

tk = case ch = getc
         when /[~_*$?!@\/\\;,=:<>".]/   #"
           str << ch
           Token(TkGVAR, str)
           
         when "-"
           str << "-" << getc
           Token(TkGVAR, str)
           
         when "&", "`", "'", "+"
           str << ch
           Token(TkBACK_REF, str)
           
         when /[1-9]/
           str << ch
           while (ch = getc) =~ /[0-9]/
             str << ch
           end
           ungetc
           Token(TkNTH_REF)
         when /\w/
           ungetc
           ungetc
           return identify_identifier
         else 
           ungetc
           Token("$")     
         end
    tk.set_text(str)
  end
  
  def identify_identifier
    token = ""
    token.concat getc if peek(0) =~ /[$@]/
    token.concat getc if peek(0) == "@"

while (ch = getc) =~ /\w|_/
      print ":", ch, ":" if RubyLex.debug?
      token.concat ch
    end
    ungetc
    
    if ch == "!" or ch == "?"
      token.concat getc
    end
    # fix token

# $stderr.puts "identifier - #{token}, state = #@lex_state"

case token
    when /^\$/
      return Token(TkGVAR, token).set_text(token)
    when /^\@/
      @lex_state = EXPR_END
      return Token(TkIVAR, token).set_text(token)
    end
    
    if @lex_state != EXPR_DOT
      print token, "\n" if RubyLex.debug?

token_c, *trans = TkReading2Token[token]
      if token_c
	# reserved word?

if (@lex_state != EXPR_BEG &&
	    @lex_state != EXPR_FNAME &&
	    trans[1])
	  # modifiers
	  token_c = TkSymbol2Token[trans[1]]
	  @lex_state = trans[0]
	else
	  if @lex_state != EXPR_FNAME
	    if ENINDENT_CLAUSE.include?(token)
	      @indent += 1
	    elsif DEINDENT_CLAUSE.include?(token)
	      @indent -= 1
	    end
	    @lex_state = trans[0]
	  else
	    @lex_state = EXPR_END
	  end
	end
	return Token(token_c, token).set_text(token)
      end
    end

if @lex_state == EXPR_FNAME
      @lex_state = EXPR_END
      if peek(0) == '='
	token.concat getc
      end
    elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
      @lex_state = EXPR_ARG
    else
      @lex_state = EXPR_END
    end

if token[0, 1] =~ /[A-Z]/
      return Token(TkCONSTANT, token).set_text(token)
    elsif token[token.size - 1, 1] =~ /[!?]/
      return Token(TkFID, token).set_text(token)
    else
      return Token(TkIDENTIFIER, token).set_text(token)
    end
  end

def identify_here_document
    ch = getc
    if ch == "-"
      ch = getc
      indent = true
    end
    if /['"`]/ =~ ch            # '
      lt = ch
      quoted = ""
      while (c = getc) && c != lt
	quoted.concat c
      end
    else
      lt = '"'
      quoted = ch.dup
      while (c = getc) && c =~ /\w/
	quoted.concat c
      end
      ungetc
    end

ltback, @ltype = @ltype, lt
    reserve = ""

while ch = getc
      reserve << ch
      if ch == "\\"    #"
        ch = getc
	reserve << ch
      elsif ch == "\n"
	break
      end
    end

str = ""
    while (l = gets)
      l.chomp!
      l.strip! if indent
      break if l == quoted
      str << l.chomp << "\n"
    end

@reader.divert_read_from(reserve)

@ltype = ltback
    @lex_state = EXPR_END
    Token(Ltype2Token[lt], str).set_text(str.dump)
  end
  
  def identify_quotation(initial_char)
    ch = getc
    if lt = PERCENT_LTYPE[ch]
      initial_char += ch
      ch = getc
    elsif ch =~ /\W/
      lt = "\""
    else
      RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
    end
#     if ch !~ /\W/
#       ungetc
#       next
#     end
    #@ltype = lt
    @quoted = ch unless @quoted = PERCENT_PAREN[ch]
    identify_string(lt, @quoted, ch, initial_char)
  end

def identify_number(start)
    str = start.dup

if start == "+" or start == "-" or start == ""
      start = getc
      str << start
    end

@lex_state = EXPR_END

if start == "0"
      if peek(0) == "x"
        ch = getc
        str << ch
        match = /[0-9a-f_]/
      else
        match = /[0-7_]/
      end
      while ch = getc
        if ch !~ match
          ungetc
          break
        else
          str << ch
        end
      end
      return Token(TkINTEGER).set_text(str)
    end

type = TkINTEGER
    allow_point = TRUE
    allow_e = TRUE
    while ch = getc
      case ch
      when /[0-9_]/
        str << ch

when allow_point && "."
	type = TkFLOAT
	if peek(0) !~ /[0-9]/
	  ungetc
	  break
	end
        str << ch
	allow_point = false

when allow_e && "e", allow_e && "E"
        str << ch
	type = TkFLOAT
	if peek(0) =~ /[+-]/
	  str << getc
	end
	allow_e = false
	allow_point = false
      else
	ungetc
	break
      end
    end
    Token(type).set_text(str)
  end
  
  def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
    @ltype = ltype
    @quoted = quoted
    subtype = nil

str = ""
    str << initial_char if initial_char
    str << (opener||quoted)

nest = 0
    begin
      while ch = getc 
	str << ch
	if @quoted == ch 
          if nest == 0
            break
          else
            nest -= 1
          end
        elsif opener == ch
          nest += 1
	elsif @ltype != "'" && @ltype != "]" and ch == "#"
          ch = getc
          if ch == "{"
            subtype = true
            str << ch << skip_inner_expression
          else
            ungetc(ch)
          end
	elsif ch == '\\' #'
	  str << read_escape
	end
      end
      if @ltype == "/"
	if peek(0) =~ /i|o|n|e|s/
	  str << getc
	end
      end
      if subtype
	Token(DLtype2Token[ltype], str)
      else
	Token(Ltype2Token[ltype], str)
      end.set_text(str)
    ensure
      @ltype = nil
      @quoted = nil
      @lex_state = EXPR_END
    end
  end

def skip_inner_expression
    res = ""
    nest = 0
    while (ch = getc)
      res << ch
      if ch == '}'
        break if nest.zero?
        nest -= 1
      elsif ch == '{'
        nest += 1
      end
    end
    res
  end

def identify_comment
    @ltype = "#"
    comment = "#"
    while ch = getc
      if ch == "\\"
        ch = getc
        if ch == "\n"
          ch = " "
        else
          comment << "\\" 
        end
      else
        if ch == "\n"
          @ltype = nil
          ungetc
          break
        end
      end
      comment << ch
    end
    return Token(TkCOMMENT).set_text(comment)
  end
  
  def read_escape
    res = ""
    case ch = getc
    when /[0-7]/
      ungetc ch
      3.times do
	case ch = getc
	when /[0-7]/
	when nil
	  break
	else
	  ungetc
	  break
	end
        res << ch
      end
      
    when "x"
      res << ch
      2.times do
	case ch = getc
	when /[0-9a-fA-F]/
	when nil
	  break
	else
	  ungetc
	  break
	end
        res << ch
      end

when "M"
      res << ch
      if (ch = getc) != '-'
	ungetc
      else
        res << ch
	if (ch = getc) == "\\" #"
          res << ch
	  res << read_escape
        else
          res << ch
	end
      end

when "C", "c" #, "^"
      res << ch
      if ch == "C" and (ch = getc) != "-"
	ungetc
      else
        res << ch
        if (ch = getc) == "\\" #"
          res << ch
          res << read_escape
        else
          res << ch
        end
      end
    else
      res << ch
    end
    res
  end
end

# Extract code elements from a source file, returning a TopLevel
# object containing the constituent file elements.
#
# This file is based on rtags

module RDoc

GENERAL_MODIFIERS = [ 'nodoc' ].freeze

CLASS_MODIFIERS = GENERAL_MODIFIERS

ATTR_MODIFIERS  = GENERAL_MODIFIERS

CONSTANT_MODIFIERS = GENERAL_MODIFIERS

METHOD_MODIFIERS = GENERAL_MODIFIERS + 
    [ 'arg', 'args', 'yield', 'yields', 'notnew', 'not-new', 'not_new', 'doc' ]

class RubyParser
    include RubyToken
    include TokenStream

extend ParserFactory

parse_files_matching(/\.rbw?$/)

def initialize(top_level, file_name, content, options, stats)
      @options = options
      @stats   = stats
      @size = 0
      @token_listeners = nil
      @input_file_name = file_name
      @scanner = RubyLex.new(content)
      @scanner.exception_on_syntax_error = false
      @top_level = top_level
      @progress = $stderr unless options.quiet
    end

def scan
      @tokens = []
      @unget_read = []
      @read = []
      catch(:eof) do
        catch(:enddoc) do
          begin
            parse_toplevel_statements(@top_level)
          rescue Exception => e
            $stderr.puts "\n\n"
            $stderr.puts "RDoc failure in #@input_file_name at or around " +
                         "line #{@scanner.line_no} column #{@scanner.char_no}"
            $stderr.puts 
            $stderr.puts "Before reporting this, could you check that the file"
            $stderr.puts "you're documenting compiles cleanly--RDoc is not a"
            $stderr.puts "full Ruby parser, and gets confused easily if fed"
            $stderr.puts "invalid programs."
            $stderr.puts
            $stderr.puts "The internal error was:\n\n"
            
            e.set_backtrace(e.backtrace[0,4])
            raise
          end
        end
      end
      @top_level
    end

private

def make_message(msg)
      prefix = "\n" + @input_file_name + ":"
      if @scanner
        prefix << "#{@scanner.line_no}:#{@scanner.char_no}: "
      end
      return prefix + msg
    end

def warn(msg)
      return if @options.quiet
      msg = make_message msg
      $stderr.puts msg
    end

def error(msg)
      msg = make_message msg
      $stderr.puts msg
      exit(1)
    end

def progress(char)
      unless @options.quiet
        @progress.print(char)
	@progress.flush
      end
    end

def add_token_listener(obj)
      @token_listeners ||= []
      @token_listeners << obj
    end

def remove_token_listener(obj)
      @token_listeners.delete(obj)
    end

def get_tk
      tk = nil
      if @tokens.empty?
	tk = @scanner.token
	@read.push @scanner.get_read
	puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
      else
	@read.push @unget_read.shift
	tk = @tokens.shift
	puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
      end

if tk.kind_of?(TkSYMBEG)
        set_token_position(tk.line_no, tk.char_no)
        tk1 = get_tk
        if tk1.kind_of?(TkId) || tk1.kind_of?(TkOp)
          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
          # remove the identifier we just read (we're about to
          # replace it with a symbol)
          @token_listeners.each do |obj|
            obj.pop_token
          end if @token_listeners
        else
          warn("':' not followed by identifier or operator")
          tk = tk1
        end
      end

# inform any listeners of our shiny new token
      @token_listeners.each do |obj|
        obj.add_token(tk)
      end if @token_listeners

tk
    end

def peek_tk
      unget_tk(tk = get_tk)
      tk
    end

def unget_tk(tk)
      @tokens.unshift tk
      @unget_read.unshift @read.pop

# Remove this token from any listeners
      @token_listeners.each do |obj|
        obj.pop_token
      end if @token_listeners
    end

def skip_tkspace(skip_nl = true)
      tokens = []
      while ((tk = get_tk).kind_of?(TkSPACE) ||
	     (skip_nl && tk.kind_of?(TkNL)))
	tokens.push tk
      end
      unget_tk(tk)
      tokens
    end

def get_tkread
      read = @read.join("")
      @read = []
      read
    end

def peek_read
      @read.join('')
    end

NORMAL = "::"
    SINGLE = "<<"

# Look for the first comment in a file that isn't
    # a shebang line.

def collect_first_comment
      skip_tkspace
      res = ''
      first_line = true

tk = get_tk
      while tk.kind_of?(TkCOMMENT)
        if first_line && /\A#!/ =~ tk.text
          skip_tkspace
          tk = get_tk
        elsif first_line && /\A#\s*-\*-/ =~ tk.text
          first_line = false
          skip_tkspace
          tk = get_tk
        else
          first_line = false
          res << tk.text << "\n"
          tk = get_tk
          if tk.kind_of? TkNL
            skip_tkspace(false)
            tk = get_tk
          end
        end
      end
      unget_tk(tk)
      res
    end

def parse_toplevel_statements(container)
      comment = collect_first_comment
      look_for_directives_in(container, comment)
      container.comment = comment unless comment.empty?
      parse_statements(container, NORMAL, nil, comment)
    end
    
    def parse_statements(container, single=NORMAL, current_method=nil, comment='')
      nest = 1
      save_visibility = container.visibility
      
#      if container.kind_of?(TopLevel)
#      else
#        comment = ''
#      end

non_comment_seen = true
      
      while tk = get_tk
        
        keep_comment = false
        
        non_comment_seen = true unless tk.kind_of?(TkCOMMENT)
        
	case tk

when TkNL
          skip_tkspace(true)   # Skip blanks and newlines
          tk = get_tk
          if tk.kind_of?(TkCOMMENT)
            if non_comment_seen
              comment = ''
              non_comment_seen = false
            end
            while tk.kind_of?(TkCOMMENT)
              comment << tk.text << "\n"
              tk = get_tk          # this is the newline 
              skip_tkspace(false)  # leading spaces
              tk = get_tk
            end
            unless comment.empty?
              look_for_directives_in(container, comment) 
              if container.done_documenting
                container.ongoing_visibility = save_visibility
#                return
              end
            end
            keep_comment = true
          else
            non_comment_seen = true
          end
          unget_tk(tk)
          keep_comment = true

when TkCLASS
	  if container.document_children
            parse_class(container, single, tk, comment)
	  else
	    nest += 1
          end

when TkMODULE
	  if container.document_children
            parse_module(container, single, tk, comment)
	  else
	    nest += 1
          end

when TkDEF
	  if container.document_self
	    parse_method(container, single, tk, comment)
	  else
	    nest += 1
          end

when TkCONSTANT
          if container.document_self
            parse_constant(container, single, tk, comment)
          end

when TkALIAS
 	  if container.document_self
	    parse_alias(container, single, tk, comment)
	  end

when TkYIELD
          if current_method.nil?
            warn("Warning: yield outside of method") if container.document_self
          else
            parse_yield(container, single, tk, current_method)
          end

# Until and While can have a 'do', which shouldn't increas
          # the nesting. We can't solve the general case, but we can
          # handle most occurrences by ignoring a do at the end of a line

when  TkUNTIL, TkWHILE
          nest += 1
          puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
            "line #{tk.line_no}" if $DEBUG
          skip_optional_do_after_expression

# 'for' is trickier
        when TkFOR
          nest += 1
          puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
            "line #{tk.line_no}" if $DEBUG
          skip_for_variable
          skip_optional_do_after_expression

when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN
	  nest += 1
          puts "Found #{tk.class} in #{container.name}, nest = #{nest}, " +
            "line #{tk.line_no}" if $DEBUG

when TkIDENTIFIER
          if nest == 1 and current_method.nil?
            case tk.name
            when "private", "protected", "public",
                 "private_class_method", "public_class_method"
              parse_visibility(container, single, tk)
              keep_comment = true
            when "attr"
              parse_attr(container, single, tk, comment)
            when /^attr_(reader|writer|accessor)$/, @options.extra_accessors
              parse_attr_accessor(container, single, tk, comment)
            when "alias_method"
              if container.document_self
	        parse_alias(container, single, tk, comment)
	      end
            end
	  end
	  
	  case tk.name
	  when "require"
	    parse_require(container, comment)
	  when "include"
	    parse_include(container, comment)
	  end

when TkEND
          nest -= 1
          puts "Found 'end' in #{container.name}, nest = #{nest}, line #{tk.line_no}" if $DEBUG
          puts "Method = #{current_method.name}" if $DEBUG and current_method
	  if nest == 0
            read_documentation_modifiers(container, CLASS_MODIFIERS)
            container.ongoing_visibility = save_visibility
            return
          end

end

comment = '' unless keep_comment
	begin
	  get_tkread
	  skip_tkspace(false)
	end while peek_tk == TkNL

end
    end
    
    def parse_class(container, single, tk, comment, &block)
      progress("c")

@stats.num_classes += 1

container, name_t = get_class_or_module(container)

case name_t
      when TkCONSTANT
	name = name_t.name
        superclass = "Object"

if peek_tk.kind_of?(TkLT)
          get_tk
          skip_tkspace(true)
          superclass = get_class_specification
          superclass = "<unknown>" if superclass.empty?
        end

if single == SINGLE
	  cls_type = SingleClass
	else
	  cls_type = NormalClass
	end

cls = container.add_class(cls_type, name, superclass)
        read_documentation_modifiers(cls, CLASS_MODIFIERS)
        cls.record_location(@top_level)
	parse_statements(cls)
        cls.comment = comment

when TkLSHFT
	case name = get_class_specification
	when "self", container.name
	  parse_statements(container, SINGLE, &block)
	else
          other = TopLevel.find_class_named(name)
          unless other
#            other = @top_level.add_class(NormalClass, name, nil)
#            other.record_location(@top_level)
#            other.comment = comment
            other = NormalClass.new("Dummy", nil)
          end
          read_documentation_modifiers(other, CLASS_MODIFIERS)
          parse_statements(other, SINGLE, &block)
	end

else
	warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
      end
    end

def parse_module(container, single, tk, comment)
      progress("m")
      @stats.num_modules += 1
      container, name_t  = get_class_or_module(container)
#      skip_tkspace
      name = name_t.name
      mod = container.add_module(NormalModule, name)
      mod.record_location(@top_level)
      read_documentation_modifiers(mod, CLASS_MODIFIERS)
      parse_statements(mod)
      mod.comment = comment
    end

# Look for the name of a class of module (optionally with a leading :: or
    # with :: separated named) and return the ultimate name and container

def get_class_or_module(container)
      skip_tkspace
      name_t = get_tk

# class ::A -> A is in the top level
      if name_t.kind_of?(TkCOLON2)
        name_t = get_tk
        container = @top_level
      end

skip_tkspace(false)

while peek_tk.kind_of?(TkCOLON2)
        prev_container = container
        container = container.find_module_named(name_t.name)
        if !container
#          warn("Couldn't find module #{name_t.name}")
          container = prev_container.add_module(NormalModule, name_t.name)
        end
        get_tk
        name_t = get_tk
      end
      skip_tkspace(false)
      return [container, name_t]
    end

def parse_constant(container, single, tk, comment)
      name = tk.name
      skip_tkspace(false)
      eq_tk = get_tk

unless eq_tk.kind_of?(TkASSIGN)
        unget_tk(eq_tk)
        return
      end

nest = 0
      get_tkread

tk = get_tk
      if tk.kind_of? TkGT
        unget_tk(tk)
        unget_tk(eq_tk)
        return
      end

loop do
        puts("Param: #{tk}, #{@scanner.continue} " +
          "#{@scanner.lex_state} #{nest}")  if $DEBUG

case tk
        when TkSEMICOLON
          break
        when TkLPAREN, TkfLPAREN
          nest += 1
        when TkRPAREN
          nest -= 1
        when TkCOMMENT
          if nest <= 0 && @scanner.lex_state == EXPR_END
            unget_tk(tk)
            break
          end
        when TkNL
          if (@scanner.lex_state == EXPR_END and nest <= 0) || !@scanner.continue
            unget_tk(tk)
            break
          end
        end
        tk = get_tk
      end

res = get_tkread.tr("\n", " ").strip
      res = "" if res == ";"
      con = Constant.new(name, res, comment)
      read_documentation_modifiers(con, CONSTANT_MODIFIERS)
      if con.document_self
	container.add_constant(con)
      end
    end

def parse_method(container, single, tk, comment)
      progress(".")
      @stats.num_methods += 1
      line_no = tk.line_no
      column  = tk.char_no
      
      start_collecting_tokens
      add_token(tk)
      add_token_listener(self)
      
      @scanner.instance_eval{@lex_state = EXPR_FNAME}
      skip_tkspace(false)
      name_t = get_tk
      back_tk = skip_tkspace
      meth = nil
      added_container = false

dot = get_tk
      if dot.kind_of?(TkDOT) or dot.kind_of?(TkCOLON2)
	@scanner.instance_eval{@lex_state = EXPR_FNAME}
	skip_tkspace
	name_t2 = get_tk
	case name_t
	when TkSELF
	  name = name_t2.name
	when TkCONSTANT
          name = name_t2.name
          prev_container = container
          container = container.find_module_named(name_t.name)
          if !container
            added_container = true
            obj = name_t.name.split("::").inject(Object) do |state, item|
              state.const_get(item)
            end rescue nil

type = obj.class == Class ? NormalClass : NormalModule
            if not [Class, Module].include?(obj.class)
              warn("Couldn't find #{name_t.name}. Assuming it's a module")
            end

if type == NormalClass then
              container = prev_container.add_class(type, name_t.name, obj.superclass.name)
            else
              container = prev_container.add_module(type, name_t.name)
            end
          end
	else
	  # warn("Unexpected token '#{name_t2.inspect}'")
	  # break
          skip_method(container)
          return
	end
	meth =  AnyMethod.new(get_tkread, name)
        meth.singleton = true
      else
	unget_tk dot
	back_tk.reverse_each do
	  |tk|
	  unget_tk tk
	end
	name = name_t.name

meth =  AnyMethod.new(get_tkread, name)
        meth.singleton = (single == SINGLE)
      end

remove_token_listener(self)

meth.start_collecting_tokens
      indent = TkSPACE.new(1,1)
      indent.set_text(" " * column)

meth.add_tokens([TkCOMMENT.new(line_no,
                                     1,
                                     "# File #{@top_level.file_absolute_name}, line #{line_no}"),
                        NEWLINE_TOKEN,
                        indent])

meth.add_tokens(@token_stream)

add_token_listener(meth)

@scanner.instance_eval{@continue = false}
      parse_method_parameters(meth)

if meth.document_self
        container.add_method(meth)
      elsif added_container
        container.document_self = false
      end

# Having now read the method parameters and documentation modifiers, we
      # now know whether we have to rename #initialize to ::new

if name == "initialize" && !meth.singleton
        if meth.dont_rename_initialize
          meth.visibility = :protected
        else
          meth.singleton = true
          meth.name = "new"
          meth.visibility = :public
        end
      end
      
      parse_statements(container, single, meth)
      
      remove_token_listener(meth)

# Look for a 'call-seq' in the comment, and override the
      # normal parameter stuff

if comment.sub!(/:?call-seq:(.*?)^\s*\#?\s*$/m, '')
        seq = $1
        seq.gsub!(/^\s*\#\s*/, '')
        meth.call_seq = seq
      end
      
      meth.comment = comment

end
    
    def skip_method(container)
      meth =  AnyMethod.new("", "anon")
      parse_method_parameters(meth)
      parse_statements(container, false, meth)
    end
    
    # Capture the method's parameters. Along the way,
    # look for a comment containing 
    #
    #    # yields: ....
    #
    # and add this as the block_params for the method

def parse_method_parameters(method)
      res = parse_method_or_yield_parameters(method)
      res = "(" + res + ")" unless res[0] == ?(
      method.params = res unless method.params
      if method.block_params.nil?
          skip_tkspace(false)
	  read_documentation_modifiers(method, METHOD_MODIFIERS)
      end
    end

def parse_method_or_yield_parameters(method=nil, modifiers=METHOD_MODIFIERS)
      skip_tkspace(false)
      tk = get_tk

# Little hack going on here. In the statement
      #  f = 2*(1+yield)
      # We see the RPAREN as the next token, so we need
      # to exit early. This still won't catch all cases
      # (such as "a = yield + 1"
      end_token = case tk
                  when TkLPAREN, TkfLPAREN
                    TkRPAREN
                  when TkRPAREN
                    return ""
                  else
                    TkNL
                  end
      nest = 0

loop do
        puts("Param: #{tk.inspect}, #{@scanner.continue} " +
          "#{@scanner.lex_state} #{nest}")  if $DEBUG
        case tk
        when TkSEMICOLON
          break
        when TkLBRACE
          nest += 1
        when TkRBRACE
          # we might have a.each {|i| yield i }
          unget_tk(tk) if nest.zero?
          nest -= 1
          break if nest <= 0
        when TkLPAREN, TkfLPAREN
          nest += 1
        when end_token
          if end_token == TkRPAREN
            nest -= 1
            break if @scanner.lex_state == EXPR_END and nest <= 0
          else
            break unless @scanner.continue
          end
        when method && method.block_params.nil? && TkCOMMENT
	  unget_tk(tk)
	  read_documentation_modifiers(method, modifiers)
        end
        tk = get_tk
      end
      res = get_tkread.tr("\n", " ").strip
      res = "" if res == ";"
      res
    end

# skip the var [in] part of a 'for' statement
    def skip_for_variable
      skip_tkspace(false)
      tk = get_tk
      skip_tkspace(false)
      tk = get_tk
      unget_tk(tk) unless tk.kind_of?(TkIN)
    end

# while, until, and for have an optional 
    def skip_optional_do_after_expression
      skip_tkspace(false)
      tk = get_tk
      case tk
      when TkLPAREN, TkfLPAREN
        end_token = TkRPAREN
      else
        end_token = TkNL
      end

nest = 0
      @scanner.instance_eval{@continue = false}

loop do
        puts("\nWhile: #{tk}, #{@scanner.continue} " +
          "#{@scanner.lex_state} #{nest}") if $DEBUG
        case tk
        when TkSEMICOLON
          break
        when TkLPAREN, TkfLPAREN
          nest += 1
        when TkDO
          break if nest.zero?
        when end_token
          if end_token == TkRPAREN
            nest -= 1
            break if @scanner.lex_state == EXPR_END and nest.zero?
          else
            break unless @scanner.continue
          end
        end
        tk = get_tk
      end
      skip_tkspace(false)
      if peek_tk.kind_of? TkDO
        get_tk
      end
    end
    
    # Return a superclass, which can be either a constant
    # of an expression

def get_class_specification
      tk = get_tk
      return "self" if tk.kind_of?(TkSELF)
        
      res = ""
      while tk.kind_of?(TkCOLON2) ||
          tk.kind_of?(TkCOLON3)   ||
          tk.kind_of?(TkCONSTANT)   
        
        res += tk.text
        tk = get_tk
      end

unget_tk(tk)
      skip_tkspace(false)

get_tkread # empty out read buffer

tk = get_tk

case tk
      when TkNL, TkCOMMENT, TkSEMICOLON
        unget_tk(tk)
        return res
      end

res += parse_call_parameters(tk)
      res
    end

def parse_call_parameters(tk)

end_token = case tk
                  when TkLPAREN, TkfLPAREN
                    TkRPAREN
                  when TkRPAREN
                    return ""
                  else
                    TkNL
                  end
      nest = 0

loop do
        puts("Call param: #{tk}, #{@scanner.continue} " +
          "#{@scanner.lex_state} #{nest}") if $DEBUG
        case tk
        when TkSEMICOLON
          break
        when TkLPAREN, TkfLPAREN
          nest += 1
        when end_token
          if end_token == TkRPAREN
            nest -= 1
            break if @scanner.lex_state == EXPR_END and nest <= 0
          else
            break unless @scanner.continue
          end
        when TkCOMMENT
	  unget_tk(tk)
	  break
        end
        tk = get_tk
      end
      res = get_tkread.tr("\n", " ").strip
      res = "" if res == ";"
      res
    end

# Parse a constant, which might be qualified by
    # one or more class or module names

def get_constant
      res = ""
      skip_tkspace(false)
      tk = get_tk

while tk.kind_of?(TkCOLON2) ||
          tk.kind_of?(TkCOLON3)   ||
          tk.kind_of?(TkCONSTANT)          
        
        res += tk.text
        tk = get_tk
      end

#      if res.empty?
#        warn("Unexpected token #{tk} in constant")
#      end 
      unget_tk(tk)
      res
    end

# Get a constant that may be surrounded by parens
    
    def get_constant_with_optional_parens
      skip_tkspace(false)
      nest = 0
      while (tk = peek_tk).kind_of?(TkLPAREN)  || tk.kind_of?(TkfLPAREN)
        get_tk
        skip_tkspace(true)
        nest += 1
      end

name = get_constant

while nest > 0
        skip_tkspace(true)
        tk = get_tk
        nest -= 1 if tk.kind_of?(TkRPAREN)
      end
      name
    end

# Directives are modifier comments that can appear after class, module,
    # or method names. For example
    #
    #   def fred    # :yields:  a, b
    #
    # or
    #
    #   class SM  # :nodoc:
    #
    # we return the directive name and any parameters as a two element array
    
    def read_directive(allowed)
      tk = get_tk
      puts "directive: #{tk.inspect}" if $DEBUG
      result = nil
      if tk.kind_of?(TkCOMMENT) 
        if tk.text =~ /\s*:?(\w+):\s*(.*)/
          directive = $1.downcase
          if allowed.include?(directive)
            result = [directive, $2]
          end
        end
      else
        unget_tk(tk)
      end
      result
    end

def read_documentation_modifiers(context, allow)
      dir = read_directive(allow)

case dir[0]

when "notnew", "not_new", "not-new"
        context.dont_rename_initialize = true

when "nodoc"
        context.document_self = false
	if dir[1].downcase == "all"
	  context.document_children = false
	end

when "doc"
        context.document_self = true
        context.force_documentation = true

when "yield", "yields"
        unless context.params.nil?
          context.params.sub!(/(,|)\s*&\w+/,'') # remove parameter &proc
        end
	context.block_params = dir[1]

when "arg", "args"
        context.params = dir[1]
      end if dir
    end

# Look for directives in a normal comment block:
    #
    #   #--       - don't display comment from this point forward
    #  
    #
    # This routine modifies it's parameter

def look_for_directives_in(context, comment)

preprocess = SM::PreProcess.new(@input_file_name,
                                      @options.rdoc_include)

preprocess.handle(comment) do |directive, param|
        case directive
        when "stopdoc"
          context.stop_doc
          ""
        when "startdoc"
          context.start_doc
          context.force_documentation = true
          ""

when "enddoc"
          #context.done_documenting = true
          #""
          throw :enddoc

when "main"
          options = Options.instance
          options.main_page = param
	  ""

when "title"
          options = Options.instance
          options.title = param
          ""

when "section"
          context.set_current_section(param, comment)
          comment.replace("") # 1.8 doesn't support #clear
          break 
        else
          warn "Unrecognized directive '#{directive}'"
          break
        end
      end

remove_private_comments(comment)
    end

def remove_private_comments(comment)
      comment.gsub!(/^#--.*?^#\+\+/m, '')
      comment.sub!(/^#--.*/m, '')
    end

def get_symbol_or_name
      tk = get_tk
      case tk
      when  TkSYMBOL
        tk.text.sub(/^:/, '')
      when TkId, TkOp
        tk.name
      when TkSTRING
        tk.text
      else
        raise "Name or symbol expected (got #{tk})"
      end
    end
    
    def parse_alias(context, single, tk, comment)
      skip_tkspace
      if (peek_tk.kind_of? TkLPAREN)
        get_tk
        skip_tkspace
      end
      new_name = get_symbol_or_name
      @scanner.instance_eval{@lex_state = EXPR_FNAME}
      skip_tkspace
      if (peek_tk.kind_of? TkCOMMA)
        get_tk
        skip_tkspace
      end
      old_name = get_symbol_or_name

al = Alias.new(get_tkread, old_name, new_name, comment)
      read_documentation_modifiers(al, ATTR_MODIFIERS)
      if al.document_self
	context.add_alias(al)
      end
    end

def parse_yield_parameters
      parse_method_or_yield_parameters
    end

def parse_yield(context, single, tk, method)
    if method.block_params.nil?
      get_tkread
      @scanner.instance_eval{@continue = false}
      method.block_params = parse_yield_parameters
    end
  end

def parse_require(context, comment)
    skip_tkspace_comment
    tk = get_tk
    if tk.kind_of? TkLPAREN
      skip_tkspace_comment
      tk = get_tk
    end

name = nil
    case tk
    when TkSTRING
      name = tk.text
#    when TkCONSTANT, TkIDENTIFIER, TkIVAR, TkGVAR
#      name = tk.name
    when TkDSTRING
      warn "Skipping require of dynamic string: #{tk.text}"
 #   else
 #     warn "'require' used as variable"
    end
    if name
      context.add_require(Require.new(name, comment))
    else
      unget_tk(tk)
    end
  end

def parse_include(context, comment)
    loop do
      skip_tkspace_comment
      name = get_constant_with_optional_parens
      unless name.empty?
        context.add_include(Include.new(name, comment))
      end
      return unless peek_tk.kind_of?(TkCOMMA)
      get_tk
    end
  end

def get_bool
      skip_tkspace
      tk = get_tk
      case tk
      when TkTRUE
        true
      when TkFALSE, TkNIL
        false
      else
        unget_tk tk
        true
      end
    end

def parse_attr(context, single, tk, comment)
      args = parse_symbol_arg(1)
      if args.size > 0
	name = args[0]
        rw = "R"
        skip_tkspace(false)
        tk = get_tk
        if tk.kind_of? TkCOMMA
          rw = "RW" if get_bool
        else
          unget_tk tk
        end
	att = Attr.new(get_tkread, name, rw, comment)
	read_documentation_modifiers(att, ATTR_MODIFIERS)
	if att.document_self
	  context.add_attribute(att)
	end
      else
	warn("'attr' ignored - looks like a variable")
      end

end

def parse_visibility(container, single, tk)
      singleton = (single == SINGLE)
      vis = case tk.name
            when "private"   then :private
            when "protected" then :protected
            when "public"    then :public
            when "private_class_method"
              singleton = true
              :private
            when "public_class_method"
              singleton = true
              :public
            else raise "Invalid visibility: #{tk.name}"
            end
            
      skip_tkspace_comment(false)
      case peek_tk
        # Ryan Davis suggested the extension to ignore modifiers, because he
        # often writes
        #
        #   protected unless $TESTING
        #
      when TkNL, TkUNLESS_MOD, TkIF_MOD
#        error("Missing argument") if singleton        
        container.ongoing_visibility = vis
      else
        args = parse_symbol_arg
        container.set_visibility_for(args, vis, singleton)
      end
    end

def parse_attr_accessor(context, single, tk, comment)
      args = parse_symbol_arg
      read = get_tkread
      rw = "?"

# If nodoc is given, don't document any of them

tmp = CodeObject.new
      read_documentation_modifiers(tmp, ATTR_MODIFIERS)
      return unless tmp.document_self

case tk.name
      when "attr_reader"   then rw = "R"
      when "attr_writer"   then rw = "W"
      when "attr_accessor" then rw = "RW"
      else
        rw = @options.extra_accessor_flags[tk.name]
      end
      
      for name in args
	att = Attr.new(get_tkread, name, rw, comment)
        context.add_attribute(att)
      end    
    end

def skip_tkspace_comment(skip_nl = true)
      loop do
        skip_tkspace(skip_nl)
        return unless peek_tk.kind_of? TkCOMMENT
        get_tk
      end
    end

def parse_symbol_arg(no = nil)

args = []
      skip_tkspace_comment
      case tk = get_tk
      when TkLPAREN
	loop do
	  skip_tkspace_comment
	  if tk1 = parse_symbol_in_arg
	    args.push tk1
	    break if no and args.size >= no
	  end
	  
	  skip_tkspace_comment
	  case tk2 = get_tk
	  when TkRPAREN
	    break
	  when TkCOMMA
	  else
           warn("unexpected token: '#{tk2.inspect}'") if $DEBUG
	    break
	  end
	end
      else
	unget_tk tk
	if tk = parse_symbol_in_arg
	  args.push tk
	  return args if no and args.size >= no
	end

loop do
#	  skip_tkspace_comment(false)
	  skip_tkspace(false)

tk1 = get_tk
	  unless tk1.kind_of?(TkCOMMA) 
	    unget_tk tk1
	    break
	  end
	  
	  skip_tkspace_comment
	  if tk = parse_symbol_in_arg
	    args.push tk
	    break if no and args.size >= no
	  end
	end
      end
      args
    end

def parse_symbol_in_arg
      case tk = get_tk
      when TkSYMBOL
        tk.text.sub(/^:/, '')
      when TkSTRING
	eval @read[-1]
      else
	warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG
	nil
      end
    end
  end

end

Back