#include <OperationDAGLanguageParser.hh>

Collaboration diagram for TokenizerData:

Classes
class	Token

class	TokenTreeNode

Public Types
enum	OperationID { IDENTIFIER, STRING_LITERAL, INTEGER_LITERAL, FLOAT_LITERAL, PRIMARY_EXPRESSION, UNARY_EXPRESSION, UNARY_ADDROF, UNARY_STAR, UNARY_PLUS, UNARY_MINUS, UNARY_TILDE, UNARY_BANG, POSTFIX_EXPRESSION, CAST_EXPRESSION, MULTIPLICATIVE_EXPRESSION, ADDITIVE_EXPRESSION, SHIFT_EXPRESSION, RELATIONAL_EXPRESSION, EQUALITY_EXPRESSION, AND_EXPRESSION, EXCLUSIVE_EXPRESSION, INCLUSIVE_EXPRESSION, LOGICAL_AND_EXPRESSION, LOGICAL_OR_EXPRESSION, CONDITIONAL_EXPRESSION, ASSIGNMENT_EXPRESSION, ASSIGNMENT_OPERATOR, EXPRESSION, CONSTANT_EXPRESSION, DECLARATION, DECLARATION_SPECIFIERS, INIT_DECLARATOR, STORAGE_CLASS_SPECIFIER, TYPE_SPECIFIER, STRUCT_OR_UNION_SPECIFIER, STRUCT_OR_UNION, STRUCT_DECLARATION, STRUCT_DECLARATOR, ENUM_SPECIFIER, ENUMERATOR, CONST_QUALIFER, VOLATILE_QUALIFER, TYPE_QUALIFER, DECLARATOR, INITIALIZER, DIRECT_DECLARATOR, POINTER, PARAMETER_TYPE_LIST, PARAMETER_LIST, PARAMETER_DECLARATION, IDENTIFIER_LIST, TYPE_NAME, ABSTRACT_DECLARATOR, DIRECT_ABSTRACT_DECLARATOR, INITIALIZER_LIST, STATEMENT, LABELED_STATEMENT, COMPOUND_STATEMENT, STATEMENT_LIST, EXPRESSION_STATEMENT, SELECTION_STATEMENT, ITERATION_STATEMENT, JUMP_STATEMENT, FUNCTION_DEFINITION, EXTERNAL_DECLARATION, INVALID_TOKEN }

Public Member Functions
	TokenizerData ()

	~TokenizerData ()

void	addToken (const char start, const char end, OperationID id, std::string strVal)

void	showTokenizedSource ()

const TokenTreeNode *	tokenTree () const

Static Public Member Functions
static std::string	idString (OperationID id)

Private Types
typedef std::map< std::pair< const char , const char >, Token * >	TokenContainerType

Private Attributes
TokenContainerType	parsedTokens_

TokenTreeNode	root

Detailed Description

Class which is used by Spirit parser to create tokenized tree form of the source code.

Definition at line 148 of file OperationDAGLanguageParser.hh.

Member Typedef Documentation

◆ TokenContainerType

typedef std::map< std::pair<const char*, const char*>, Token*> TokenizerData::TokenContainerType

private

Definition at line 705 of file OperationDAGLanguageParser.hh.

Member Enumeration Documentation

◆ OperationID

enum TokenizerData::OperationID

List of different type tokens.

Enumerator
IDENTIFIER
STRING_LITERAL
INTEGER_LITERAL
FLOAT_LITERAL
PRIMARY_EXPRESSION
UNARY_EXPRESSION
UNARY_ADDROF
UNARY_STAR
UNARY_PLUS
UNARY_MINUS
UNARY_TILDE
UNARY_BANG
POSTFIX_EXPRESSION
CAST_EXPRESSION
MULTIPLICATIVE_EXPRESSION
ADDITIVE_EXPRESSION
SHIFT_EXPRESSION
RELATIONAL_EXPRESSION
EQUALITY_EXPRESSION
AND_EXPRESSION
EXCLUSIVE_EXPRESSION
INCLUSIVE_EXPRESSION
LOGICAL_AND_EXPRESSION
LOGICAL_OR_EXPRESSION
CONDITIONAL_EXPRESSION
ASSIGNMENT_EXPRESSION
ASSIGNMENT_OPERATOR
EXPRESSION
CONSTANT_EXPRESSION
DECLARATION
DECLARATION_SPECIFIERS
INIT_DECLARATOR
STORAGE_CLASS_SPECIFIER
TYPE_SPECIFIER
STRUCT_OR_UNION_SPECIFIER
STRUCT_OR_UNION
STRUCT_DECLARATION
STRUCT_DECLARATOR
ENUM_SPECIFIER
ENUMERATOR
CONST_QUALIFER
VOLATILE_QUALIFER
TYPE_QUALIFER
DECLARATOR
INITIALIZER
DIRECT_DECLARATOR
POINTER
PARAMETER_TYPE_LIST
PARAMETER_LIST
PARAMETER_DECLARATION
IDENTIFIER_LIST
TYPE_NAME
ABSTRACT_DECLARATOR
DIRECT_ABSTRACT_DECLARATOR
INITIALIZER_LIST
STATEMENT
LABELED_STATEMENT
COMPOUND_STATEMENT
STATEMENT_LIST
EXPRESSION_STATEMENT
SELECTION_STATEMENT
ITERATION_STATEMENT
JUMP_STATEMENT
FUNCTION_DEFINITION
EXTERNAL_DECLARATION
INVALID_TOKEN

Definition at line 154 of file OperationDAGLanguageParser.hh.

                      {        
         IDENTIFIER,
         STRING_LITERAL,
         INTEGER_LITERAL,
         FLOAT_LITERAL,
         PRIMARY_EXPRESSION,
         UNARY_EXPRESSION,
         UNARY_ADDROF,
         UNARY_STAR,
         UNARY_PLUS,
         UNARY_MINUS,
         UNARY_TILDE,
         UNARY_BANG,
         POSTFIX_EXPRESSION,
         CAST_EXPRESSION,
         MULTIPLICATIVE_EXPRESSION,
         ADDITIVE_EXPRESSION,
         SHIFT_EXPRESSION,
         RELATIONAL_EXPRESSION,
         EQUALITY_EXPRESSION,
         AND_EXPRESSION,
         EXCLUSIVE_EXPRESSION,
         INCLUSIVE_EXPRESSION,
         LOGICAL_AND_EXPRESSION,
         LOGICAL_OR_EXPRESSION,
         CONDITIONAL_EXPRESSION,
         ASSIGNMENT_EXPRESSION,
         ASSIGNMENT_OPERATOR,
         EXPRESSION,
         CONSTANT_EXPRESSION,
         DECLARATION,
         DECLARATION_SPECIFIERS,
         INIT_DECLARATOR,
         STORAGE_CLASS_SPECIFIER,
         TYPE_SPECIFIER,
         STRUCT_OR_UNION_SPECIFIER,
         STRUCT_OR_UNION,
         STRUCT_DECLARATION,
         STRUCT_DECLARATOR,
         ENUM_SPECIFIER,
         ENUMERATOR,
         CONST_QUALIFER,
         VOLATILE_QUALIFER,
         TYPE_QUALIFER,
         DECLARATOR,
         INITIALIZER,
         DIRECT_DECLARATOR,
         POINTER,
         PARAMETER_TYPE_LIST,
         PARAMETER_LIST,
         PARAMETER_DECLARATION,
         IDENTIFIER_LIST,
         TYPE_NAME,
         ABSTRACT_DECLARATOR,
         DIRECT_ABSTRACT_DECLARATOR,
         INITIALIZER_LIST,
         STATEMENT,
         LABELED_STATEMENT,
         COMPOUND_STATEMENT,
         STATEMENT_LIST,
         EXPRESSION_STATEMENT,
         SELECTION_STATEMENT,
         ITERATION_STATEMENT,
         JUMP_STATEMENT,
         FUNCTION_DEFINITION,
         EXTERNAL_DECLARATION,
         INVALID_TOKEN
     };

Constructor & Destructor Documentation

◆ TokenizerData()

TokenizerData::TokenizerData ( )

inline

Definition at line 613 of file OperationDAGLanguageParser.hh.

613 {}

◆ ~TokenizerData()

TokenizerData::~TokenizerData ( )

inline

Deletes all the parsed data.

Definition at line 618 of file OperationDAGLanguageParser.hh.

                      {
         while (!parsedTokens_.empty()) {
             delete (parsedTokens_.begin()->second);
             parsedTokens_.erase(parsedTokens_.begin());
         }
         // TODO: delete TokenTreeNodes
     }

Member Function Documentation

◆ addToken()

void TokenizerData::addToken	(	const char *	start,
		const char *	end,
		OperationID	id,
		std::string	strVal
	)

inline

Creates new token based on parsed data or updates new type info for old token.

Parameters

start	Start position of token.
end	End position of token.
id	Type of token.
strVal	String of token.

Definition at line 636 of file OperationDAGLanguageParser.hh.

                                                     {
         
         unsigned long startPos, endPos;
         
         if (parsedTokens_.empty()) {
             startPos = 0;
             endPos = end - start;
         } else {
             // in parsedTokens_.begin()->first.second is actually stored value
             // (-start) of first parsed token. So we can add it to current 
             // start address for getting relative end and start positions.
             startPos = 
                 reinterpret_cast<unsigned long>(start) + 
                 reinterpret_cast<unsigned long>(
                     parsedTokens_.begin()->first.second);
             
             endPos = 
                 reinterpret_cast<unsigned long>(end) + 
                 reinterpret_cast<unsigned long>(
                     parsedTokens_.begin()->first.second);
         }
                
         // key for map is selected (end, -start) for automatically organizase
         // tokens
         std::pair<const char*, const char*> 
             tokenKey(end, (const char*)(-reinterpret_cast<unsigned long>(start)));
         
         if (parsedTokens_.find(tokenKey) == parsedTokens_.end()) {
             parsedTokens_[tokenKey] = new Token(startPos, endPos, strVal);
             root.addToTokenTree(*parsedTokens_[tokenKey]);
         }
         
         parsedTokens_[tokenKey]->setProperty(id);
     }

Referenced by TokenizerActor::operator()().

◆ idString()

static std::string TokenizerData::idString ( OperationID id )

inlinestatic

Prints string value for each token.

Parameters

id	Token type.

Returns: String for token type.

Definition at line 229 of file OperationDAGLanguageParser.hh.

                                               {
         switch (id) {
         case IDENTIFIER: return "IDENTIFIER:";
         case STRING_LITERAL: return "STRING_LITERAL:";
         case INTEGER_LITERAL: return "INTEGER_LITERAL:";
         case FLOAT_LITERAL: return "FLOAT_LITERAL:";
         case PRIMARY_EXPRESSION: return "PRIMARY_EXPRESSION:";
         case UNARY_EXPRESSION: return "UNARY_EXPRESSION:";
         case UNARY_ADDROF: return "UNARY_ADDROF:";
         case UNARY_STAR: return "UNARY_STAR:";
         case UNARY_PLUS: return "UNARY_PLUS:";
         case UNARY_MINUS: return "UNARY_MINUS:";
         case UNARY_TILDE: return "UNARY_TILDE:";
         case UNARY_BANG: return "UNARY_BANG:";
         case POSTFIX_EXPRESSION: return "POSTFIX_EXPRESSION:";
         case CAST_EXPRESSION: return "CAST_EXPRESSION:";
         case MULTIPLICATIVE_EXPRESSION: return "MULTIPLICATIVE_EXPRESSION:";
         case ADDITIVE_EXPRESSION: return "ADDITIVE_EXPRESSION:";
         case SHIFT_EXPRESSION: return "SHIFT_EXPRESSION:";
         case RELATIONAL_EXPRESSION: return "RELATIONAL_EXPRESSION:";
         case EQUALITY_EXPRESSION: return "EQUALITY_EXPRESSION:";
         case AND_EXPRESSION: return "AND_EXPRESSION:";
         case EXCLUSIVE_EXPRESSION: return "EXCLUSIVE_EXPRESSION:";
         case INCLUSIVE_EXPRESSION: return "INCLUSIVE_EXPRESSION:";
         case LOGICAL_AND_EXPRESSION: return "LOGICAL_AND_EXPRESSION:";
         case LOGICAL_OR_EXPRESSION: return "LOGICAL_OR_EXPRESSION:";
         case CONDITIONAL_EXPRESSION: return "CONDITIONAL_EXPRESSION:";
         case ASSIGNMENT_EXPRESSION: return "ASSIGNMENT_EXPRESSION:";
         case ASSIGNMENT_OPERATOR: return "ASSIGNMENT_OPERATOR:";
         case EXPRESSION: return "EXPRESSION:";
         case CONSTANT_EXPRESSION: return "CONSTANT_EXPRESSION:";
         case DECLARATION: return "DECLARATION:";
         case DECLARATION_SPECIFIERS: return "DECLARATION_SPECIFIERS:";
         case INIT_DECLARATOR: return "INIT_DECLARATOR:";
         case STORAGE_CLASS_SPECIFIER: return "STORAGE_CLASS_SPECIFIER:";
         case TYPE_SPECIFIER: return "TYPE_SPECIFIER:";
         case STRUCT_OR_UNION_SPECIFIER: return "STRUCT_OR_UNION_SPECIFIER:";
         case STRUCT_OR_UNION: return "STRUCT_OR_UNION:";
         case STRUCT_DECLARATION: return "STRUCT_DECLARATION:";
         case STRUCT_DECLARATOR: return "STRUCT_DECLARATOR:";
         case ENUM_SPECIFIER: return "ENUM_SPECIFIER:";
         case ENUMERATOR: return "ENUMERATOR:";
         case CONST_QUALIFER: return "CONST_QUALIFER:";
         case VOLATILE_QUALIFER: return "VOLATILE_QUALIFER:";
         case TYPE_QUALIFER: return "TYPE_QUALIFER:";
         case DECLARATOR: return "DECLARATOR:";
         case INITIALIZER: return "INITIALIZER:";
         case DIRECT_DECLARATOR: return "DIRECT_DECLARATOR:";
         case POINTER: return "POINTER:";
         case PARAMETER_TYPE_LIST: return "PARAMETER_TYPE_LIST:";
         case PARAMETER_LIST: return "PARAMETER_LIST:";
         case PARAMETER_DECLARATION: return "PARAMETER_DECLARATION:";
         case IDENTIFIER_LIST: return "IDENTIFIER_LIST:";
         case TYPE_NAME: return "TYPE_NAME:";
         case ABSTRACT_DECLARATOR: return "ABSTRACT_DECLARATOR:";
         case DIRECT_ABSTRACT_DECLARATOR: return "DIRECT_ABSTRACT_DECLARATOR:";
         case INITIALIZER_LIST: return "INITIALIZER_LIST:";
         case STATEMENT: return "STATEMENT:";
         case LABELED_STATEMENT: return "LABELED_STATEMENT:";
         case COMPOUND_STATEMENT: return "COMPOUND_STATEMENT:";
         case STATEMENT_LIST: return "STATEMENT_LIST:";
         case EXPRESSION_STATEMENT: return "EXPRESSION_STATEMENT:";
         case SELECTION_STATEMENT: return "SELECTION_STATEMENT:";
         case ITERATION_STATEMENT: return "ITERATION_STATEMENT:";
         case JUMP_STATEMENT: return "JUMP_STATEMENT:";
         case FUNCTION_DEFINITION: return "FUNCTION_DEFINITION:";
         case EXTERNAL_DECLARATION: return "EXTERNAL_DECLARATION:";
         case INVALID_TOKEN: return "INVALID_TOKEN:";            
         default: return "UNKNOWN_ID:";
         }        
     }

Referenced by TokenizerData::TokenTreeNode::toStr().

◆ showTokenizedSource()

void TokenizerData::showTokenizedSource ( )

inline

Shows source code parsed to tokens.

Definition at line 675 of file OperationDAGLanguageParser.hh.

                                {
         
         for (TokenContainerType::iterator i = parsedTokens_.begin();
              i != parsedTokens_.end(); i++) {
             
             std::cerr << std::hex 
                       << std::setw(4) << i->second->start_
                       << " - " 
                       << std::setw(4) << i->second->end_ 
                 
                       << " : "                                 
                       << std::setw(30) << idString(i->second->type_) 
                 
                       << " : " 
                       << i->second->strValue_
                       << std::endl;
         }
     }

◆ tokenTree()

const TokenTreeNode* TokenizerData::tokenTree ( ) const

inline

Returns root node of token tree.

Returns: Root node of token tree.

Definition at line 699 of file OperationDAGLanguageParser.hh.

                                            {
         return &root;
     }

Referenced by OperationDAGConverter::createDAG().

Member Data Documentation

◆ parsedTokens_

TokenContainerType TokenizerData::parsedTokens_

private

Definition at line 707 of file OperationDAGLanguageParser.hh.

◆ root

TokenTreeNode TokenizerData::root

private

Definition at line 708 of file OperationDAGLanguageParser.hh.

The documentation for this class was generated from the following file:

OperationDAGLanguageParser.hh

Classes

Public Types

Public Member Functions

Static Public Member Functions

Private Types

Private Attributes

Detailed Description

Member Typedef Documentation

◆ TokenContainerType

Member Enumeration Documentation

◆ OperationID

Constructor & Destructor Documentation

◆ TokenizerData()

◆ ~TokenizerData()

Member Function Documentation

◆ addToken()

◆ idString()

◆ showTokenizedSource()

◆ tokenTree()

Member Data Documentation

◆ parsedTokens_

◆ root