Dekko
MarkdownTokenizer.h
1 #ifndef MARKDOWNTOKENIZER_H
2 #define MARKDOWNTOKENIZER_H
3 
4 #include <QString>
5 #include <QList>
6 #include <QMap>
7 #include <QRegExp>
8 #include "MarkdownToken.h"
9 
10 class MarkdownTokenizer
11 {
12 public:
13  MarkdownTokenizer();
14  ~MarkdownTokenizer();
15 
16  enum TokenState {
17  Unknown = -1,
18  Paragraph,
19  ParagraphBreak,
20  ListLineBreak,
21  AtxHeading1,
22  AtxHeading2,
23  AtxHeading3,
24  AtxHeading4,
25  AtxHeading5,
26  AtxHeading6,
27  Blockquote,
28  CodeBlock,
29  GFMCodeFence,
30  CodeFenceEnd,
31  Comment,
32  HorizontalRule,
33  NumList,
34  BulletList,
35  SetextHead1Line1,
36  SetextHead1Line2,
37  SetextHead2Line1,
38  SetextHead2Line2,
39  TableHeader,
40  TableDiv,
41  TableRow
42  };
43 
44 
45  void tokenize(const QString &text, TokenState current, TokenState previous, TokenState next);
46 
47  typedef QMap<int, MarkdownToken> TokenPositionMap;
48 
49  TokenPositionMap tokens() const;
50 
51  TokenState state() const;
52 
53  bool shouldBackTrack() const;
54 
55  void clear();
56 
57 private:
58  void tokenizeMatches(
59  MarkdownToken::TokenType tokenType,
60  QString& text,
61  QRegExp& regex,
62  const int markupStartCount = 0,
63  const int markupEndCount = 0,
64  const bool replaceMarkupChars = false,
65  const bool replaceAllChars = false
66  );
67  void addToken(const MarkdownToken &token);
68  void setState(TokenState state);
69  void setStartEarlier(const bool startEarlier);
70 
71  bool tokenizeSetextHeadingLine1(const QString& text);
72  bool tokenizeSetextHeadingLine2(const QString& text);
73  bool tokenizeAtxHeading(const QString& text);
74 
75  bool tokenizeNumberedList(const QString& text);
76  bool tokenizeBulletPointList(const QString& text);
77  bool tokenizeHorizontalRule(const QString& text);
78  bool tokenizeBlockquote(const QString& text);
79  bool tokenizeCodeBlock(const QString& text);
80  bool tokenizeMultilineComment(const QString& text);
81 
82  bool tokenizeInline(const QString& text);
83  void tokenizeVerbatim(QString& text);
84  void tokenizeHtmlComments(QString& text);
85  void tokenizeTableHeaderRow(QString& text);
86  bool tokenizeTableDivider(const QString& text);
87  void tokenizeTableRow(QString& text);
88 
89  TokenState m_current;
90  TokenState m_previous;
91  TokenState m_next;
92  TokenState m_state;
93  TokenPositionMap m_tokens;
94  bool m_startEarlier;
95 
96  QRegExp paragraphBreakRegex;
97  QRegExp heading1SetextRegex;
98  QRegExp heading2SetextRegex;
99  QRegExp blockquoteRegex;
100  QRegExp githubCodeFenceStartRegex;
101  QRegExp githubCodeFenceEndRegex;
102  QRegExp pandocCodeFenceStartRegex;
103  QRegExp pandocCodeFenceEndRegex;
104  QRegExp numberedListRegex;
105  QRegExp numberedNestedListRegex;
106  QRegExp hruleRegex;
107  QRegExp emphasisRegex;
108  QRegExp strongRegex;
109  QRegExp strikethroughRegex;
110  QRegExp verbatimRegex;
111  QRegExp htmlTagRegex;
112  QRegExp htmlEntityRegex;
113  QRegExp automaticLinkRegex;
114  QRegExp inlineLinkRegex;
115  QRegExp referenceLinkRegex;
116  QRegExp referenceDefinitionRegex;
117  QRegExp imageRegex;
118  QRegExp htmlInlineCommentRegex;
119  QRegExp mentionRegex;
120  QRegExp pipeTableDividerRegex;
121 
122 };
123 
124 #endif // MARKDOWNTOKENIZER_H
MarkdownTokenizer
Definition: MarkdownTokenizer.h:10
MarkdownToken
Definition: MarkdownToken.h:4