add documentation on scanner and parser.

compiler/parser/parser.go

@@ -1,3 +1,13 @@
+/*
+	Parser parses the Tengo source files.
+
+	Parser is a modified version of Go's parser implementation.
+
+	Copyright 2009 The Go Authors. All rights reserved.
+	Use of this source code is governed by a BSD-style
+	license that can be found in the LICENSE file.
+*/
+
 package parser
 
 import (
@@ -135,7 +145,7 @@ L:
 				x = p.parseSelector(x)
 			default:
 				pos := p.pos
-				p.errorExpected(pos, "selector string")
+				p.errorExpected(pos, "selector")
 				p.advance(stmtStart)
 				return &ast.BadExpr{From: pos, To: p.pos}
 			}
@@ -647,7 +657,7 @@ func (p *Parser) parseIfStmt() ast.Stmt {
 			elseStmt = p.parseBlockStmt()
 			p.expectSemi()
 		default:
-			p.errorExpected(p.pos, "if statement or block")
+			p.errorExpected(p.pos, "if or {")
 			elseStmt = &ast.BadStmt{From: p.pos, To: p.pos}
 		}
 	} else {

compiler/scanner/scanner.go

@@ -1,3 +1,13 @@
+/*
+	Scanner reads the Tengo source text and tokenize them.
+
+	Scanner is a modified version of Go's scanner implementation.
+
+	Copyright 2009 The Go Authors. All rights reserved.
+	Use of this source code is governed by a BSD-style
+	license that can be found in the LICENSE file.
+*/
+
 package scanner
 
 import (
@@ -295,10 +305,8 @@ exit:
 	lit := s.src[offs:s.offset]
 
 	// On Windows, a (//-comment) line may end in "\r\n".
-	// Remove the final '\r' before analyzing the text for
+	// Remove the final '\r' before analyzing the text for line directives (matching the compiler).
-	// line directives (matching the compiler). Remove any
+	// Remove any other '\r' afterwards (matching the pre-existing behavior of the scanner).
-	// other '\r' afterwards (matching the pre-existing be-
-	// havior of the scanner).
 	if numCR > 0 && len(lit) >= 2 && lit[1] == '/' && lit[len(lit)-1] == '\r' {
 		lit = lit[:len(lit)-1]
 		numCR--