TypeParser - support comments at EOL with //

Author: shmax

src/Ast/Attribute.php

@@ -13,4 +13,6 @@ final class Attribute
 
 	public const ORIGINAL_NODE = 'originalNode';
 
+	public const COMMENTS = 'comments';
+
 }

src/Ast/Comment.php

@@ -0,0 +1,28 @@
+<?php declare(strict_types = 1);
+
+namespace PHPStan\PhpDocParser\Ast;
+
+use function trim;
+
+class Comment
+{
+
+	public string $text;
+
+	public int $startLine;
+
+	public int $startIndex;
+
+	public function __construct(string $text, int $startLine = -1, int $startIndex = -1)
+	{
+		$this->text = $text;
+		$this->startLine = $startLine;
+		$this->startIndex = $startIndex;
+	}
+
+	public function getReformattedText(): ?string
+	{
+		return trim($this->text);
+	}
+
+}

src/Lexer/Lexer.php

@@ -51,6 +51,8 @@ class Lexer
 	public const TOKEN_NEGATED = 35;
 	public const TOKEN_ARROW = 36;
 
+	public const TOKEN_COMMENT = 37;
+
 	public const TOKEN_LABELS = [
 		self::TOKEN_REFERENCE => '\'&\'',
 		self::TOKEN_UNION => '\'|\'',
@@ -66,6 +68,7 @@ class Lexer
 		self::TOKEN_OPEN_CURLY_BRACKET => '\'{\'',
 		self::TOKEN_CLOSE_CURLY_BRACKET => '\'}\'',
 		self::TOKEN_COMMA => '\',\'',
+		self::TOKEN_COMMENT => '\'//\'',
 		self::TOKEN_COLON => '\':\'',
 		self::TOKEN_VARIADIC => '\'...\'',
 		self::TOKEN_DOUBLE_COLON => '\'::\'',
@@ -160,6 +163,7 @@ private function generateRegexp(): string
 			self::TOKEN_CLOSE_CURLY_BRACKET => '\\}',
 
 			self::TOKEN_COMMA => ',',
+			self::TOKEN_COMMENT => '\/\/[^\\r\\n]*(?=\n|\r|\*/)',
 			self::TOKEN_VARIADIC => '\\.\\.\\.',
 			self::TOKEN_DOUBLE_COLON => '::',
 			self::TOKEN_DOUBLE_ARROW => '=>',

src/Parser/PhpDocParser.php

@@ -116,9 +116,19 @@ public function parse(TokenIterator $tokens): Ast\PhpDoc\PhpDocNode
 
 			$tokens->forwardToTheEnd();
 
+			$comments = $tokens->flushComments();
+			if ($comments !== []) {
+				throw new LogicException('Comments should already be flushed');
+			}
+
 			return $this->enrichWithAttributes($tokens, new Ast\PhpDoc\PhpDocNode([$this->enrichWithAttributes($tokens, $tag, $startLine, $startIndex)]), 1, 0);
 		}
 
+		$comments = $tokens->flushComments();
+		if ($comments !== []) {
+			throw new LogicException('Comments should already be flushed');
+		}
+
 		return $this->enrichWithAttributes($tokens, new Ast\PhpDoc\PhpDocNode($children), 1, 0);
 	}
 

src/Parser/TokenIterator.php

@@ -3,6 +3,7 @@
 namespace PHPStan\PhpDocParser\Parser;
 
 use LogicException;
+use PHPStan\PhpDocParser\Ast\Comment;
 use PHPStan\PhpDocParser\Lexer\Lexer;
 use function array_pop;
 use function assert;
@@ -19,6 +20,9 @@ class TokenIterator
 
 	private int $index;
 
+	/** @var array<Comment> */
+	private array $comments = [];
+
 	/** @var int[] */
 	private array $savePoints = [];
 
@@ -152,8 +156,7 @@ public function consumeTokenType(int $tokenType): void
 			}
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 	}
 
 
@@ -166,8 +169,7 @@ public function consumeTokenValue(int $tokenType, string $tokenValue): void
 			$this->throwError($tokenType, $tokenValue);
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 	}
 
 
@@ -178,12 +180,20 @@ public function tryConsumeTokenValue(string $tokenValue): bool
 			return false;
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 
 		return true;
 	}
 
+	/**
+	 * @return Comment[]
+	 */
+	public function flushComments(): array
+	{
+		$res = $this->comments;
+		$this->comments = [];
+		return $res;
+	}
 
 	/** @phpstan-impure */
 	public function tryConsumeTokenType(int $tokenType): bool
@@ -198,14 +208,15 @@ public function tryConsumeTokenType(int $tokenType): bool
 			}
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 
 		return true;
 	}
 
 
-	/** @phpstan-impure */
+	/**
+	 * @deprecated Use skipNewLineTokensAndConsumeComments instead (when parsing a type)
+	 */
 	public function skipNewLineTokens(): void
 	{
 		if (!$this->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
@@ -218,6 +229,29 @@ public function skipNewLineTokens(): void
 	}
 
 
+	public function skipNewLineTokensAndConsumeComments(): void
+	{
+		if ($this->currentTokenType() === Lexer::TOKEN_COMMENT) {
+			$this->comments[] = new Comment($this->currentTokenValue(), $this->currentTokenLine(), $this->currentTokenIndex());
+			$this->next();
+		}
+
+		if (!$this->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
+			return;
+		}
+
+		do {
+			$foundNewLine = $this->tryConsumeTokenType(Lexer::TOKEN_PHPDOC_EOL);
+			if ($this->currentTokenType() !== Lexer::TOKEN_COMMENT) {
+				continue;
+			}
+
+			$this->comments[] = new Comment($this->currentTokenValue(), $this->currentTokenLine(), $this->currentTokenIndex());
+			$this->next();
+		} while ($foundNewLine === true);
+	}
+
+
 	private function detectNewline(): void
 	{
 		$value = $this->currentTokenValue();