int num_chars = 0;

%%
[ \t\n\r]+      printf("ignore: %s %d\n", yytext, yytext[0]);
.       ++num_chars; printf("count: %d %s\n", num_chars, yytext);

%%
main()
        {
        yylex();
        printf( "# of chars = %d\n", num_chars );
        }

count: 1 i
count: 2 n
count: 3 c
count: 4 l
count: 5 u
count: 6 d
count: 7 e
ignore:   32
count: 8 "
count: 9 .
count: 10 .
count: 11 /
count: 12 T
count: 13 a
count: 14 r
count: 15 g
count: 16 e
count: 17 t
count: 18 .
count: 19 t
count: 20 d
count: 21 "
ignore:
 10
# of chars = 21

/*===-- FileLexer.l - Scanner for TableGen Files ----------------*- C++ -*-===//
// 
//                     The LLVM Compiler Infrastructure
//
// This file was developed by the LLVM research group and is distributed under
// the University of Illinois Open Source License. See LICENSE.TXT for details.
// 
//===----------------------------------------------------------------------===//
//
// This file defines a simple flex scanner for TableGen files.  This is pretty
// straight-forward, except for the magic to handle file inclusion.
//
//===----------------------------------------------------------------------===*/

%option prefix="File"
%option yylineno
%option nostdinit
%option never-interactive
%option batch
%option nodefault
%option 8bit
%option outfile="Lexer.cpp"
%option ecs
%option noreject
%option noyymore

%x comment

%{
//hb:-->
#ifdef __INTERIX
# define _ALL_SOURCE
# define _GLIBCPP_USE_C99_DYNAMIC 1
# define _GLIBCPP_USE_C99_LONG_LONG_DYNAMIC 1
#endif
//hb:--<

#include "Record.h"
typedef std::pair<llvm::Record*, std::vector<llvm::Init*>*> SubClassRefTy;
#include "FileParser.h"

int Fileparse();

namespace llvm {

// Global variable recording the location of the include directory
std::string IncludeDirectory;

/// ParseInt - This has to handle the special case of binary numbers 0b0101
///
static int ParseInt(const char *Str) {
  if (Str[0] == '0' && Str[1] == 'b')
    return strtol(Str+2, 0, 2);
  return strtol(Str, 0, 0); 
}

static int CommentDepth = 0;

struct IncludeRec {
  std::string Filename;
  FILE *File;
  unsigned LineNo;
  YY_BUFFER_STATE Buffer;

  IncludeRec(const std::string &FN, FILE *F)
    : Filename(FN), File(F), LineNo(0){
  }
};

static std::vector<IncludeRec> IncludeStack;

std::ostream &err() {
  if (IncludeStack.empty())
    return std::cerr << "At end of input: ";

  for (unsigned i = 0, e = IncludeStack.size()-1; i != e; ++i)
    std::cerr << "Included from " << IncludeStack[i].Filename << ":"
              << IncludeStack[i].LineNo << ":\n";
  return std::cerr << "Parsing " << IncludeStack.back().Filename << ":"
                   << Filelineno << ": ";
}

/// ParseFile - this function begins the parsing of the specified tablegen file.
///
void ParseFile(const std::string &Filename, const std::string & IncludeDir) {
  FILE *F = stdin;
  if (Filename != "-") {
    F = fopen(Filename.c_str(), "r");

    if (F == 0) {
      std::cerr << "Could not open input file '" + Filename + "'!\n";
      exit (1);
    }
    IncludeStack.push_back(IncludeRec(Filename, F));
  } else {
    IncludeStack.push_back(IncludeRec("<stdin>", stdin));
  }

  // Record the location of the include directory so that the lexer can find
  // it later.
  IncludeDirectory = IncludeDir;
 
  Filein = F;
  Filelineno = 1;
  Fileparse();
  Filein = stdin;
}

/// HandleInclude - This function is called when an include directive is
/// encountered in the input stream...
///
static void HandleInclude(const char *Buffer) {
  unsigned Length = yyleng;
  //hb: -->
  std::cerr << "Length = " << Length << "\n";
  printf ("idx\thex\tchar\n");
  for (uint i=0; i < Length; i++)
  {
        printf ("%d\t%X\t%c\n", i, Buffer[i], Buffer[i]);
  }
  std::string WhatsInBuffer(Buffer, Buffer+Length);
  std::cerr << "WhatsInBuffer=" << WhatsInBuffer << "\n";
  if (Buffer[Length-1] == '"')
        std::cerr << "assert=true\n";
  else
        std::cerr << "assert=false\n";
  //hb: --<
  assert(Buffer[Length-1] == '"'); //hb: Don't you mean != "", because the line ends with "
  Buffer += strlen("include ");
  Length -= strlen("include ");
  while (*Buffer != '"') {
    ++Buffer;
    --Length;
  }
  assert(Length >= 2 && "Double quotes not found?");
  std::string Filename(Buffer+1, Buffer+Length-1);
  std::cerr << "Filename = '" << Filename << "'\n"; //hb:

  // Save the line number and lex buffer of the includer...
  IncludeStack.back().LineNo = Filelineno;
  IncludeStack.back().Buffer = YY_CURRENT_BUFFER;

  // Open the new input file...
  yyin = fopen(Filename.c_str(), "r");
  if (yyin == 0) {
    // If we couldn't find the file in the current directory, look for it in
    // the include directories.
    //
    // NOTE: Right now, there is only one directory.  We need to eventually add
    // support for more.
    std::string NextFilename = IncludeDirectory + "/" + Filename;
    yyin = fopen(NextFilename.c_str(), "r");
    if (yyin == 0) {
//hb:      err() << "Could not find include file '" << Filename << "'!\n";
      err() << "Could not find include file '" << NextFilename << "'!\n"; //hb:
      exit(1);
    }
    Filename = NextFilename;
  }

  // Add the file to our include stack...
  IncludeStack.push_back(IncludeRec(Filename, yyin));
  Filelineno = 1;  // Reset line numbering...
  //yyrestart(yyin);    // Start lexing the new file...

  yy_switch_to_buffer(yy_create_buffer(yyin, YY_BUF_SIZE));
}

/// yywrap - This is called when the lexer runs out of input in one of the
/// files. Switch back to an includer if an includee has run out of input.
///
extern "C"
int yywrap() {
  if (IncludeStack.back().File != stdin)
    fclose(IncludeStack.back().File);
  IncludeStack.pop_back();
  if (IncludeStack.empty()) return 1;  // Top-level file is done.

  // Otherwise, we need to switch back to a file which included the current one.
  Filelineno = IncludeStack.back().LineNo;  // Restore current line number
  yy_switch_to_buffer(IncludeStack.back().Buffer);
  return 0;
}

} // End llvm namespace

using namespace llvm;

%}

Comment      \/\/.*

Identifier   [a-zA-Z_][0-9a-zA-Z_]*
Integer      [-+]?[0-9]+|0x[0-9a-fA-F]+|0b[01]+
CodeFragment \[\{([^}]+|\}[^\]])*\}\]
StringVal    \"[^"]*\"
IncludeStr   include[ \t\n]+\"[^"]*\"

%%

{Comment}      { /* Ignore comments */ }

{IncludeStr}   { HandleInclude(yytext); }
{CodeFragment} { Filelval.StrVal = new std::string(yytext+2, yytext+yyleng-2);
                 return CODEFRAGMENT; }

int            { return INT; }
bit            { return BIT; }
bits           { return BITS; }
string         { return STRING; }
list           { return LIST; }
code           { return CODE; }
dag            { return DAG; }

class          { return CLASS; }
def            { return DEF; }
field          { return FIELD; }
let            { return LET; }
in             { return IN; }

{Identifier}   { Filelval.StrVal = new std::string(yytext, yytext+yyleng);
                 return ID; }
${Identifier}  { Filelval.StrVal = new std::string(yytext+1, yytext+yyleng);
                 return VARNAME; } 

{StringVal}    { Filelval.StrVal = new std::string(yytext+1, yytext+yyleng-1);
                 return STRVAL; }

{Integer}      { Filelval.IntVal = ParseInt(Filetext); return INTVAL; }

[ \t\n\r]+     { /* Ignore whitespace */ }


"/*"                    { BEGIN(comment); CommentDepth++; }
<comment>[^*/]*         /* eat anything that's not a '*' or '/' */
<comment>"*"+[^*/]*     /* eat up '*'s not followed by '/'s */
<comment>"/*"           { ++CommentDepth; }
<comment>"/"+[^*]*      /* eat up /'s not followed by *'s */
<comment>"*"+"/"        { if (!--CommentDepth) { BEGIN(INITIAL); } }
<comment><<EOF>>        { err() << "Unterminated comment!\n"; exit(1); }

.              { return Filetext[0]; }

%%

                int num_chars = 0;
                static int CommentDepth = 0;

Comment         \/\/.*
Identifier              [a-zA-Z_][0-9a-zA-Z_]*
Integer         [-+]?[0-9]+|0x[0-9a-fA-F]+|0b[01]+
CodeFragment    \[\{([^}]+|\}[^\]])*\}\]
StringVal               \"[^"]*\"
IncludeStr              include[ \t\n]+\"[^"]*\"

/*%%
[ \t\n\r]+      printf("ignore: %X\n", yytext[0]);
.               ++num_chars; printf("count: %d %c\n", num_chars, yytext[0]);

%%*/

%%

{Comment}      { /* Ignore comments */ }

{IncludeStr}    {
                                char *Buffer;
                                unsigned Length;
                                uint i;
                                
                                Buffer = yytext;
                                printf ("include: %s\n", yytext);

                                Length = yyleng;

                                printf ("Length = %d\n", Length);
                                printf ("idx\thex\tchar\n");
                                for (i=0; i < Length; i++)
                                {
                                        printf ("%d\t%X\t%c\n", i, Buffer[i], Buffer[i]);
                                }
                                printf ("WhatsInBuffer = %s\n", Buffer);
                                if (Buffer[Length-1] == '"')
                                        printf ("assert=true\n");
                                else
                                        printf ("assert=false\n");
                        }


{CodeFragment} { /*Filelval.StrVal = new std::string(yytext+2, yytext+yyleng-2);
                 return CODEFRAGMENT;*/ }

int            { /*return INT;*/ }
bit            { /*return BIT;*/ }
bits           { /*return BITS;*/ }
string         { /*return STRING;*/ }
list           { /*return LIST;*/ }
code           { /*return CODE;*/ }
dag            { /*return DAG;*/ }

class          { /*return CLASS;*/ }
def            { /*return DEF;*/ }
field          { /*return FIELD;*/ }
let            { /*return LET;*/ }
in             { /*return IN;*/ }

{Identifier}   { /*Filelval.StrVal = new std::string(yytext, yytext+yyleng);
                 return ID;*/ }
${Identifier}  { /*Filelval.StrVal = new std::string(yytext+1, yytext+yyleng);
                 return VARNAME;*/ } 

{StringVal}    { /*Filelval.StrVal = new std::string(yytext+1, yytext+yyleng-1);
                 return STRVAL;*/ }

{Integer}      { /*Filelval.IntVal = ParseInt(Filetext); return INTVAL;*/ }

[ \t\n\r]+     { printf("ignore: %X\n", yytext[0]); }

.              { ++num_chars; printf("count: %d %c\n", num_chars, yytext[0]); }

%%


main()
{
        yylex();
        printf( "# of chars = %d\n", num_chars );
}

include: include "../Target.td"
Length = 22
idx     hex     char
0       69      i
1       6E      n
2       63      c
3       6C      l
4       75      u
5       64      d
6       65      e
7       20
8       22      "
9       2E      .
10      2E      .
11      2F      /
12      54      T
13      61      a
14      72      r
15      67      g
16      65      e
17      74      t
18      2E      .
19      74      t
20      64      d
21      22      "
WhatsInBuffer = include "../Target.td"
assert=true
ignore: D
# of chars = 0

include: include "../Target.td"
Length = 23
idx     hex     char
0       69      i
1       6E      n
2       63      c
3       6C      l
4       75      u
5       64      d
6       65      e
7       20       
8       22      "
9       2E      .
10      2E      .
11      2F      /
12      54      T
13      61      a
14      72      r
15      67      g
16      65      e
17      74      t
18      2E      .
19      74      t
20      64      d
21      22      "
22      D       
WhatsInBuffer = include "../Target.td"
assert=false
# of chars = 0

lex t2; gcc lex.yy.c -ll"