Skip to content

Commit

Permalink
Use AnjutaTokenStream in all scanner
Browse files Browse the repository at this point in the history
  • Loading branch information
@sgranjoux
sgranjoux committed Nov 21, 2009
1 parent 784baf6 commit 0bd0c21
Showing 1 changed file with 30 additions and 213 deletions.
243 changes: 30 additions & 213 deletions src/ac-scanner.l
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,24 +23,35 @@
#include "ac-parser.h"

#include "libanjuta/anjuta-debug.h"
#include "libanjuta/anjuta-token-stream.h"

#include <stdlib.h>
#include <string.h>

#define YY_INPUT(buf, result, max_size) result = amp_ac_scanner_input (yyextra, buf, max_size)
#define YY_INPUT(buffer, result, max_size) result = anjuta_token_stream_read (yyextra->stream, buffer, max_size)

#define YY_EXTRA_TYPE AmpAcScanner*

#define YY_DECL static int ac_yylex (YYSTYPE * yylval_param,YYLTYPE * yylloc_param ,yyscan_t yyscanner)

//#define YY_USER_INIT {yy_flex_debug = 1;}

static AnjutaToken* amp_ac_scanner_append_lex_token (AmpAcScanner *scanner, gint token);
static gint amp_ac_scanner_input (AmpAcScanner *scanner, gchar *buffer, gsize max_size);
static gint amp_ac_scanner_parse_end (AmpAcScanner *scanner);

#define RETURN(tok) *yylval = amp_ac_scanner_append_lex_token (yyextra, tok); \
#define RETURN(tok) *yylval = anjuta_token_stream_tokenize (yyextra->stream, tok, yyleng); \
return tok

struct _AmpAcScanner
{
yyscan_t scanner;

AnjutaTokenStream *stream;

AmpProject *project;

AnjutaToken *parsed;
};


%}

Expand Down Expand Up @@ -140,192 +151,18 @@ AC_CONFIG_FILES\( { RETURN (AC_CONFIG_FILES); }

%%

typedef struct _AmpAcBuffer AmpAcBuffer;

struct _AmpAcBuffer
{
AnjutaToken *token;

/* Beginning of current token */
AnjutaToken *start;
gsize begin;

AnjutaToken *end; /* Last token */

/* Next data read buffer */
AnjutaToken *next;
gsize pos;

/* Place to put new token */
AnjutaToken *first;
AnjutaToken *last;

AmpAcBuffer *parent;
};

struct _AmpAcScanner
{
yyscan_t scanner;

AmpAcBuffer *buffer;

AmpProject *project;

AnjutaToken *parsed;
};


/* Private functions
*---------------------------------------------------------------------------*/

static void
amp_ac_scanner_append_token (AmpAcScanner *scanner, AnjutaToken *token)
{
if (scanner->buffer->last == NULL)
{
scanner->buffer->last = anjuta_token_insert_child (scanner->buffer->first, token);
}
else
{
while (anjuta_token_parent (scanner->buffer->last) != scanner->buffer->first)
{
scanner->buffer->last = anjuta_token_parent (scanner->buffer->last);
}
scanner->buffer->last = anjuta_token_insert_after (scanner->buffer->last, token);
}
}
static AnjutaToken*
amp_ac_scanner_append_lex_token (AmpAcScanner *scanner, gint token)
{
AnjutaToken *frag;
gsize length = yyget_leng (scanner->scanner);
AnjutaToken *end;

frag = anjuta_token_new_fragment (token, NULL, 0);

for (end = scanner->buffer->start; end != NULL;)
{
if (anjuta_token_get_type (end) < ANJUTA_TOKEN_PARSED)
{
gint toklen = anjuta_token_get_length (end);
AnjutaToken *copy = anjuta_token_cut (end, scanner->buffer->begin, length);

if (toklen >= (length + scanner->buffer->begin))
{

if (end == scanner->buffer->start)
{
/* Get whole token */
anjuta_token_free (frag);
anjuta_token_set_type (copy, token);
frag = copy;
}
else
{
/* Get several token */
anjuta_token_append_child (frag, copy);
}

if (toklen == (length + scanner->buffer->begin))
{
scanner->buffer->start = anjuta_token_next (end);
scanner->buffer->begin = 0;
}
else
{
scanner->buffer->start = end;
scanner->buffer->begin += length;
}
break;
}
else
{
anjuta_token_append_child (frag, copy);
length -= toklen;
end = anjuta_token_next (end);
scanner->buffer->begin = 0;
}
}
else
{
end = anjuta_token_next (end);
scanner->buffer->begin = 0;
}
}

amp_ac_scanner_append_token (scanner, frag);

return frag;
}

static gint
amp_ac_scanner_input (AmpAcScanner *scanner, gchar *buffer, gsize max_size)
{
gint result = YY_NULL;

if (scanner->buffer->next != NULL)
{
gsize length = anjuta_token_get_length (scanner->buffer->next);

if ((anjuta_token_get_type (scanner->buffer->next) >= ANJUTA_TOKEN_PARSED) || (scanner->buffer->pos >= length))
{
for (;;)
{
/* Last token */
if (scanner->buffer->next == scanner->buffer->end) return 0;

if (anjuta_token_get_type (scanner->buffer->next) >= ANJUTA_TOKEN_PARSED)
{
scanner->buffer->next = anjuta_token_next (scanner->buffer->next);
}
else
{
scanner->buffer->next = anjuta_token_next (scanner->buffer->next);
}

if ((scanner->buffer->next == NULL) || (anjuta_token_get_type (scanner->buffer->next) == ANJUTA_TOKEN_EOV))
{
/* Last token */
return 0;
}
else if ((anjuta_token_get_length (scanner->buffer->next) != 0) && (anjuta_token_get_type (scanner->buffer->next) < ANJUTA_TOKEN_PARSED))
{
/* Find some data */
scanner->buffer->pos = 0;
length = anjuta_token_get_length (scanner->buffer->next);
break;
}
}
}

if (scanner->buffer->pos < length)
{
const gchar *start = anjuta_token_get_string (scanner->buffer->next);

length -= scanner->buffer->pos;

if (length > max_size) length = max_size;
memcpy (buffer, start + scanner->buffer->pos, length);
scanner->buffer->pos += length;
result = length;
}
}

return result;
}

static gint
amp_ac_scanner_parse_end (AmpAcScanner *scanner)
{
AmpAcBuffer *buffer;

scanner->parsed = anjuta_token_stream_get_root (scanner->stream);
yypop_buffer_state(scanner->scanner);
buffer = scanner->buffer;
scanner->buffer = buffer->parent;
scanner->parsed = buffer->first;
g_free (buffer);
scanner->stream = anjuta_token_stream_pop (scanner->stream);

if (scanner->buffer == NULL)
if (scanner->stream == NULL)
{
yyterminate();
}
Expand Down Expand Up @@ -379,50 +216,29 @@ amp_ac_scanner_load_properties (AmpAcScanner *scanner, AnjutaToken *macro, Anjut
AnjutaToken *
amp_ac_scanner_parse_token (AmpAcScanner *scanner, AnjutaToken *token, gint start, GError **error)
{
AmpAcBuffer *buffer;

buffer = g_new (AmpAcBuffer, 1);
buffer->token = token;
buffer->pos = 0;
buffer->begin = 0;
buffer->parent = NULL;
AnjutaToken *first;
AnjutaTokenStream *stream;

buffer->next = anjuta_token_next (token);
buffer->start = buffer->next;
buffer->end = anjuta_token_last (token);
if (buffer->end == token) buffer->end = NULL;
stream = anjuta_token_stream_push (scanner->stream, token);
first = anjuta_token_stream_get_root (stream);

if (scanner->buffer != NULL)
if (scanner->stream != NULL)
{
/* Parse an included file or a expanded variable */
buffer->parent = scanner->buffer;
scanner->buffer = buffer;

//anjuta_token_dump (token);

buffer->first = buffer->parent->last;
buffer->last = NULL;

scanner->stream = stream;
yypush_buffer_state(yy_create_buffer(NULL, YY_BUF_SIZE, scanner->scanner), scanner->scanner);

return scanner->parsed;
}
else
{
amp_ac_yypstate *ps;
gint status;
YYSTYPE yylval_param;
YYSTYPE yylval_param;
YYLTYPE yylloc_param;
AnjutaToken* yytok;

buffer->last = NULL;
buffer->first = anjuta_token_new_static (ANJUTA_TOKEN_FILE, NULL);

yytok = buffer->first;

scanner->buffer = buffer;

scanner->stream = stream;
ps = amp_ac_yypstate_new ();

yylval_param = NULL;
yylloc_param = NULL;
switch (start)
Expand All @@ -434,6 +250,7 @@ amp_ac_scanner_parse_token (AmpAcScanner *scanner, AnjutaToken *token, gint star
default:
break;
}

do
{
gint yychar = ac_yylex (&yylval_param, &yylloc_param, scanner->scanner);
Expand All @@ -443,9 +260,9 @@ amp_ac_scanner_parse_token (AmpAcScanner *scanner, AnjutaToken *token, gint star

} while (status == YYPUSH_MORE);
amp_ac_yypstate_delete (ps);

return scanner->parsed;
}

return first;
}

/* Constructor & Destructor
Expand Down

0 comments on commit 0bd0c21

Please sign in to comment.