My project compiles fine, but when running a push_back on a vector, I'm getting an access violation when the vector attempts to resize itself. I have lots of code, but hopefully I can boil it down to just relevant classes.
I am sorry about the HUGE amount of code, but I want my example to be compilable, and the error is likely some sort of copy constructor or assignment operator error and not the immediate use of vector, the error appears when line 513 of parsetokenizer.cpp runs during the third loop through the main function. It happens when the vector re-sizes itself it calls a method called _Orphan_all() which causes the access violation Unhandled exception at 0x6d8cad54 (msvcp100d.dll) in FloatingTests.exe: 0xC0000005: Access violation writing location 0x00000065.
> msvcp100d.dll!std::_Container_base12::_Orphan_all() Line 202 + 0x5 bytes C++
FloatingTests.exe!std::vector<parse::Token,std::allocator<parse::Token> >::reserve(unsigned int _Count) Line 768 + 0xb bytes C++
FloatingTests.exe!std::vector<parse::Token,std::allocator<parse::Token> >::_Reserve(unsigned int _Count) Line 1298 C++
FloatingTests.exe!std::vector<parse::Token,std::allocator<parse::Token> >::push_back(const parse::Token & _Val) Line 992 C++
FloatingTests.exe!parse::TokenStringSource::nextVector() Line 514 C++
FloatingTests.exe!parse::TokenStringSource::loadChache() Line 531 + 0xc bytes C++
FloatingTests.exe!parse::TokenStringSource::ensureChacheIsLoaded() Line 545 C++
FloatingTests.exe!parse::TokenStringSource::hasMoreTokenStrings() Line 540 C++
FloatingTests.exe!TokenStringSourceExample::operator()(char * * argv, int argc) Line 188 + 0xb bytes C++
FloatingTests.exe!main(char * * argc, int argv) Line 40 + 0x31 bytes C++
FloatingTests.exe!__tmainCRTStartup() Line 555 + 0x19 bytes C
FloatingTests.exe!mainCRTStartup() Line 371 C
kernel32.dll!7515339a()
[Frames below may be incorrect and/or missing, no symbols loaded for kernel32.dll]
ntdll.dll!77a29ed2()
ntdll.dll!77a29ea5()
prseautil.h
#ifndef PARSEUTIL
#define PARSEUTIL
#include <stdio.h>
#include <memory>
#include <string>
namespace parse{
class Cloneable {
public:
virtual std::shared_ptr<Cloneable> clone()=0;
virtual std::string toString() const {return "Coneable";};
virtual ~Cloneable(){};
};
typedef std::shared_ptr<Cloneable> CloneablePtr;
class PushbackReader{
private:
std::unique_ptr<std::istream> reader;
char* buf;
int pos;
int bsize;
public:
PushbackReader(std::unique_ptr<std::istream> reader, int pushbacksize);
int read();
void unread(int ch);
~PushbackReader();
};
typedef std::shared_ptr<PushbackReader> PushbackPtr;
class Stack : public Cloneable {
protected:
CloneablePtr * arr;
int arrlen;
int len;
void ensuresize(int size);
public:
Stack();
explicit Stack(int i);
bool empty() const {return len==0;}
int size() const {return len;}
std::string toString () const;
operator std::string(){return toString();}
CloneablePtr elementAt(int i){return arr[i];}
CloneablePtr& operator[](int i){return arr[i];}
void removeAllElements();
void push(CloneablePtr c);
CloneablePtr pop();
CloneablePtr clone();
virtual ~Stack();
};
typedef std::shared_ptr<Stack> StackPtr;
std::string stringFromFile(std::string filename);
}
#endif
parseutli.cpp
#include "parseutil.h"
#include <sstream>
#include <fstream>
std::string parse::stringFromFile(std::string filename){
std::ostringstream oss;
std::ifstream file(filename);
if(file.is_open()){
while(file.good()){
std::string line;
std::getline(file,line);
oss<<line;
}
}
return oss.str();
}
parse::PushbackReader::PushbackReader(std::unique_ptr<std::istream> stream, int pushbacksize){
reader=std::move(stream);
pos=pushbacksize;
bsize=pushbacksize;
buf=new char[bsize];
}
parse::PushbackReader::~PushbackReader(){
if(buf) delete[] buf;
}
int parse::PushbackReader::read(){
if(pos<bsize) return buf[pos++];
else {
int c = reader->get();
if(reader->good()){
return c;
} else{
return -1;
}
}
}
void parse::PushbackReader::unread(int c){
buf[--pos]=(char)c;
}
void parse::Stack::ensuresize(int s){
if(arrlen<=s){
int nlen = arrlen*2;
nlen=nlen<s ? s : nlen;
CloneablePtr * nbuf = new CloneablePtr[nlen];
for(int i =0; i<arrlen; i++){
nbuf[i]=arr[i];
}
if(arr)
delete[] arr;
arr=nbuf;
arrlen=nlen;
}
}
parse::Stack::Stack(){
arr=NULL;
len=0;
ensuresize(10);
}
parse::Stack::Stack(int s){
arr=NULL;
len=0;
ensuresize(s);
}
std::string parse::Stack::toString() const {
std::ostringstream oss;
oss<<"[";
for(int i = 0; i<len; i++){
if(i!=0) oss<<", ";
oss<<arr[i]->toString();
}
oss<<"]";
return oss.str();
}
void parse::Stack::removeAllElements(){
len=0;
}
void parse::Stack::push(std::shared_ptr<parse::Cloneable> c){
ensuresize(len+1);
arr[len++]=c;
}
std::shared_ptr<parse::Cloneable> parse::Stack::pop(){
if(len!=0){
return arr[--len];
}
return NULL;
}
std::shared_ptr<parse::Cloneable> parse::Stack::clone(){
std::shared_ptr<parse::Stack> ns(new parse::Stack(len));
for(int i = 0; i<len; i++){
ns->push(arr[i]->clone());
}
return ns;
}
parse::Stack::~Stack(){
delete[] arr;
}
parsetokenizer.h
#ifndef PARSETOKENIZER_H
#define PARSETOKENIZER_H
#include "parseutil.h"
#include <vector>
namespace parse{
class TokenType{
protected:
std::string name;
public:
TokenType(std::string nname) : name(nname){}
TokenType(){name="";}
std::string getName() const {return name;}
};
static bool operator==(const TokenType& l, const TokenType& r){
return l.getName()==r.getName();
}
static bool operator!=(const TokenType& l, const TokenType& r){
return l.getName()!=r.getName();
}
static bool operator< (const TokenType& l, const TokenType& r){
return l.getName()<r.getName();
}
static bool operator> (const TokenType& l, const TokenType& r){
return l.getName()>r.getName();
}
static bool operator<=(const TokenType& l, const TokenType& r){
return l.getName()<=r.getName();
}
static bool operator>=(const TokenType& l, const TokenType& r){
return l.getName()>=r.getName();
}
class Token : public Cloneable {
protected:
TokenType ttype;
std::string sval;
double nval;
public:
static const TokenType TT_EOF;
static const TokenType TT_NUMBER;
static const TokenType TT_WORD;
static const TokenType TT_SYMBOL;
static const TokenType TT_QUOTED;
static const Token T_EOF;
TokenType type() const {return ttype;}
std::string toString () const;
operator std::string(){return toString();}
CloneablePtr clone(){
return CloneablePtr(new Token(*this));
}
std::string strval()const {return sval;}
double numval() const {return nval;}
bool isWord(){return ttype==TT_WORD;}
bool isSymbol(){return ttype==TT_SYMBOL;}
bool isQuotedString(){return ttype==TT_QUOTED;}
bool isNumber(){return ttype==TT_NUMBER;}
bool equals(Token t);
bool equalsIgnoreCase(Token t);
Token() : ttype(TT_SYMBOL),sval(""),nval(0){}
Token(char c) : ttype(TT_SYMBOL), sval(1,c),nval(0){}
Token(double d) : ttype(TT_NUMBER), sval(""),nval(d){}
Token(std::string s) : ttype(TT_WORD),sval(s),nval(0){}
Token(TokenType tt, std::string s, double d) : ttype(tt),sval(s),nval(d){}
Token(const Token& t):ttype(t.ttype){
sval=t.sval;
nval=t.nval;
}
Token& operator=(const Token& t){
ttype=t.ttype;
sval=t.sval;
nval=t.nval;
return *this;
}
};
typedef std::shared_ptr<Token> TokenPtr;
static bool operator==(const Token& l, const Token& r){
return (l.strval()==r.strval()&&l.numval()==r.numval()&&r.type()==l.type());
}
class Tokenizer;
class TokenizerState{
public:
virtual Token nextToken(
PushbackPtr r, int c, Tokenizer& t
)=0;
virtual ~TokenizerState(){}
};
class WhitespaceState : public TokenizerState {
protected:
bool whitespaceChar[256];
public:
WhitespaceState();
Token nextToken(
PushbackPtr r, int c, Tokenizer& t
);
void setWhitespaceChars(int from, int to, bool b);
};
class SymbolNode {
protected:
char myChar;
std::vector<SymbolNode*> children;
bool valid;
SymbolNode* parent;
void addDescendantLine(std::string s);
SymbolNode* unreadToValid(PushbackPtr r);
void setValid(bool b){valid=b;}
SymbolNode* findDescendant(std::string s);
virtual SymbolNode* findChildWithChar(char c);
SymbolNode* ensureChildWithChar(char c);
SymbolNode* deepestRead(PushbackPtr r);
public:
SymbolNode(SymbolNode * const parent, char c) : parent(parent),myChar(c){children.clear();}
SymbolNode() : parent(NULL), myChar(0){children.clear();}
SymbolNode(const SymbolNode& n);
SymbolNode& operator=(const SymbolNode& n);
virtual std::string ancestry() const;
std::string toString() const;
operator std::string(){return toString();}
virtual ~SymbolNode();
friend class SymbolRootNode;
};
class SymbolRootNode : public SymbolNode {
public:
SymbolRootNode();
void add(std::string s);
std::string ancestry() const;
std::string nextSymbol(PushbackPtr r, int first);
protected:
SymbolNode rchildren[256];
SymbolNode* findChildWithChar(char c);
void init();
};
class SymbolState : public TokenizerState {
private:
SymbolRootNode symbols;
public:
void add(std::string s);
SymbolState();
Token nextToken(PushbackPtr r, int first, Tokenizer& t);
};
class WordState : public TokenizerState {
protected:
char * charbuf;
int buflen;
bool wordchar[256];
bool wordChar(int c);
void checkBufLength(int i);
public:
WordState();
Token nextToken(PushbackPtr r, int c, Tokenizer& t);
void setWordChars(int from, int to, bool b);
virtual ~WordState();
};
class NumberState : public TokenizerState {
protected:
int c;
double value;
bool absorbedLeadingMinus;
bool absorbedDot;
bool gotAdigit;
double absorbDigits(PushbackPtr r, bool frac);
void parseLeft(PushbackPtr r);
virtual void parseRight(PushbackPtr r);
virtual void reset(int cin);
virtual Token getValue(PushbackPtr r, Tokenizer& t);
public:
NumberState(){reset(0);}
Token nextToken(PushbackPtr r, int cin, Tokenizer& t);
~NumberState(){
int i = 0;
}
};
class SlashStarState : public TokenizerState {
public:
Token nextToken(PushbackPtr, int theStar, Tokenizer &t);
};
class SlashSlashState : public TokenizerState {
public:
Token nextToken(PushbackPtr r, int theSlash, Tokenizer& t);
};
class SlashState : public TokenizerState{
protected:
SlashSlashState slashSlashState;
SlashStarState slashStarState;
public:
Token nextToken(PushbackPtr r, int theslash, Tokenizer& t);
};
class QuoteState : public TokenizerState{
protected:
char * charbuf;
int buflen;
void checkBufLength(int i);
public:
QuoteState();
virtual ~QuoteState();
Token nextToken(PushbackPtr r, int cin, Tokenizer & t);
};
class Tokenizer{
protected:
PushbackPtr reader;
static const int DEFAULT_SYMBOL_MAX=4;
TokenizerState * characterState[256];
NumberState numberState;
QuoteState quoteState;
SlashState slashState;
SymbolState symbolState;
WhitespaceState whitespaceState;
WordState wordState;
public:
Tokenizer();
Tokenizer(std::string s);
PushbackPtr getReader(){return reader;}
Token nextToken();
SymbolState* getSymbolState(){return &symbolState;}
NumberState* getNumberState(){return &numberState;}
QuoteState* getQuoteState(){return "eState;}
SlashState* getSlashState(){return &slashState;}
WhitespaceState* getWhitespaceState(){return &whitespaceState;}
WordState* getWordState(){return &wordState;}
void setCharacterState(int from, int to, TokenizerState * state);
void setReader(PushbackPtr r){reader=r;}
void setString(std::string s);
void setString(std::string s, int symbolMax);
};
class TokenString{
protected:
Token * tokens;
int numTokens;
public:
TokenString(Token * tokens, int size);
TokenString(const TokenString &t);
TokenString(std::string s);
TokenString(Tokenizer t);
virtual ~TokenString();
int length(){return numTokens;}
Token tokenAt(int i){return tokens[i];}
Token& operator[](int i){return tokens[i];}
std::string toString();
operator std::string(){return toString();}
TokenString& operator=(const TokenString& t);
};
class TokenStringSource {
protected:
Tokenizer tokenizer;
std::string delimiter;
TokenString * cachedTokenString;
void ensureChacheIsLoaded();
void loadChache();
std::vector<Token> nextVector();
public:
TokenStringSource(Tokenizer t, std::string delim);
TokenStringSource(const TokenStringSource& t);
TokenStringSource& operator=(const TokenStringSource & t);
virtual ~TokenStringSource();
bool hasMoreTokenStrings();
TokenString nextTokenString();
};
};
#endif
parsetokenizer.cpp
#include "parsetokenizer.h"
#include <algorithm>
#include <sstream>
using namespace parse;
using namespace std;
const TokenType Token::TT_EOF("eof");
const TokenType Token::TT_NUMBER("number");
const TokenType Token::TT_WORD("word");
const TokenType Token::TT_SYMBOL("symbol");
const TokenType Token::TT_QUOTED("quoted");
const Token Token::T_EOF(Token::TT_EOF,"",0);
bool Token::equals(Token t){
if(ttype!=t.ttype)return false;
if(ttype==TT_NUMBER) return nval==t.nval;
return sval==t.sval;
}
bool Token::equalsIgnoreCase(Token t){
if(ttype!=t.ttype)return false;
if(ttype==TT_NUMBER) return nval==t.nval;
string tsval=sval;
string ttsval=t.sval;
transform(tsval.begin(),tsval.end(),tsval.begin(),::toupper);
transform(ttsval.begin(),ttsval.end(),ttsval.begin(),::toupper);
return tsval==ttsval;
}
string Token::toString() const {
ostringstream oss;
oss<<"TOKEN type="<<ttype.getName()<<" str="<<sval<<" double="<<nval;
return oss.str();
}
SymbolNode* SymbolNode::unreadToValid(shared_ptr<PushbackReader> r){
if(valid) return this;
r->unread(myChar);
return parent->unreadToValid(r);
}
string SymbolNode::toString() const {
ostringstream oss;
oss<<"SymbolNode char="<<myChar<<"("<<valid<<")";
return oss.str();
}
SymbolNode* SymbolNode::findDescendant(string s){
char c = s[0];
SymbolNode* n = findChildWithChar(c);
if(s.length()==1){
return n;
}
return n->findDescendant(s.substr(1,s.length()));
}
SymbolNode* SymbolNode::findChildWithChar(char c){
vector<SymbolNode*>::iterator it = children.begin();
for(;it!=children.end();it++){
if((*it)->myChar==c){
return (*it);
}
}
return NULL;
}
SymbolNode* SymbolNode::ensureChildWithChar(char c){
SymbolNode* n = findChildWithChar(c);
if(n==NULL){
n=new SymbolNode(this,c);
children.push_back(n);
}
return n;
}
SymbolNode* SymbolNode::deepestRead(shared_ptr<PushbackReader> r){
char c = (char) r->read();
SymbolNode* n = findChildWithChar(c);
if(n==NULL){
r->unread(c);
return this;
}
return n->deepestRead(r);
}
SymbolNode::SymbolNode(const SymbolNode& n){
myChar=n.myChar;
valid=n.valid;
parent=n.parent;
for(int i = 0; i<n.children.size(); i++){
children.push_back(new SymbolNode(*n.children[i]));
}
}
SymbolNode& SymbolNode::operator=(const SymbolNode& n){
for(int i = 0; i<children.size(); i++){
delete children[i];
}
for(int i = 0; i<n.children.size(); i++){
children.push_back(new SymbolNode(*n.children[i]));
}
myChar=n.myChar;
valid=n.valid;
parent=n.parent;
return *this;
}
string SymbolNode::ancestry() const {
return parent->ancestry().append(1,myChar);
}
void SymbolNode::addDescendantLine(string s){
if(s.length() > 0){
char c = s[0];
SymbolNode* n = ensureChildWithChar(c);
n->addDescendantLine(s.substr(1,s.length()));
}
}
SymbolNode::~SymbolNode(){
vector<SymbolNode*>::iterator it = children.begin();
for(;it!=children.end();it++){
delete *it;
}
}
string SymbolRootNode::nextSymbol(shared_ptr<PushbackReader> r, int first){
SymbolNode* n1 = findChildWithChar((char)first);
SymbolNode* n2 = n1->deepestRead(r);
SymbolNode* n3 = n2->unreadToValid(r);
return n3->ancestry();
}
SymbolRootNode::SymbolRootNode()
{
init();
}
void SymbolRootNode::init(){
int len = 256;
for(int i = 0; i<len; i++){
rchildren[i] = SymbolNode(this,i);
rchildren[i].setValid(true);
}
}
SymbolNode* SymbolRootNode::findChildWithChar(char c){
return &rchildren[c];
}
string SymbolRootNode::ancestry() const{
return "";
}
void SymbolRootNode::add(string s){
char c = s[0];
SymbolNode* n = ensureChildWithChar(c);
n->addDescendantLine(s.substr(1,s.length()));
findDescendant(s)->setValid(true);
}
SymbolState::SymbolState(){
add("!=");
add(":-");
add("<=");
add(">=");
}
void SymbolState::add(string s){
symbols.add(s);
}
Token SymbolState::nextToken(shared_ptr<PushbackReader> r, int first, Tokenizer &t){
string s = symbols.nextSymbol(r,first);
return Token(Token::TT_SYMBOL,s,0);
}
bool WordState::wordChar(int c){
if(c>=0&&c<256) {
return wordchar[c];
}
return false;
}
void WordState::setWordChars(int from, int to, bool b){
for(int i =from; i<=to; i++){
if(i>=0&&i<256) wordchar[i]=b;
}
}
Token WordState::nextToken(shared_ptr<PushbackReader>r, int c, Tokenizer& t){
int i = 0;
do {
checkBufLength(i);
charbuf[i++]=(char)c;
c=r->read();
}while(wordChar(c));
if(c>=0) r->unread(c);
string sval(charbuf,i);
return Token(Token::TT_WORD,sval,0);
}
WordState::WordState(){
buflen=16;
charbuf = new char[buflen];
setWordChars(0,255,false);
setWordChars('a', 'z', true);
setWordChars('A', 'Z', true);
setWordChars('0', '9', true);
setWordChars('-', '-', true);
setWordChars('_', '_', true);
setWordChars('\'', '\'', true);
setWordChars(0xc0, 0xff, true);
}
WordState::~WordState(){
delete[] charbuf;
}
void WordState::checkBufLength(int in){
if(in>=buflen){
int nlen = buflen*2;
char* nb = new char[nlen];
for(int i = 0; i<buflen; i++){
nb[i]=charbuf[i];
}
delete[] charbuf;
buflen=nlen;
charbuf=nb;
}
}
void WhitespaceState::setWhitespaceChars(int from, int to, bool b){
for(int i = from; i<=to; i++){
if(i>=0 && i<256) whitespaceChar[i]=b;
}
}
Token WhitespaceState::nextToken(shared_ptr<PushbackReader> r, int aWhitespaceChar, Tokenizer &t){
int c;
do{
c=r->read();
} while (
c>=0 &&
c<256 &&
whitespaceChar[c]);
if(c>=0) r->unread(c);
return t.nextToken();
}
WhitespaceState::WhitespaceState(){
setWhitespaceChars(0,255,false);
setWhitespaceChars(0,' ', true);
}
Token NumberState::getValue(shared_ptr<PushbackReader> r, Tokenizer& t){
if(!gotAdigit){
if(absorbedLeadingMinus && absorbedDot){
r->unread('.');
return ((t.getSymbolState())->nextToken(r,'-',t));
}
if(absorbedLeadingMinus){
return ((t.getSymbolState())->nextToken(r,'-',t));
}
if(absorbedDot){
return ((t.getSymbolState())->nextToken(r,'.',t));
}
}
if(absorbedLeadingMinus){
value=-value;
}
return Token(Token::TT_NUMBER,"",value);
}
void NumberState::reset(int cin){
c=cin;
value=0;
absorbedLeadingMinus=false;
absorbedDot=false;
gotAdigit=false;
}
void NumberState::parseRight(shared_ptr<PushbackReader> r){
if(c=='.'){
c=r->read();
absorbedDot=true;
value+=absorbDigits(r,true);
}
}
void NumberState::parseLeft(shared_ptr<PushbackReader> r){
if(c=='-'){
c=r->read();
absorbedLeadingMinus=true;
}
value=absorbDigits(r,false);
}
Token NumberState::nextToken(shared_ptr<PushbackReader> r, int cin, Tokenizer& t){
reset(cin);
parseLeft(r);
parseRight(r);
r->unread(c);
return getValue(r,t);
}
double NumberState::absorbDigits(shared_ptr<PushbackReader> r, bool fraction){
int divideBy=1;
double v = 0;
while('0'<=c && c<='9'){
gotAdigit=true;
v=v*10+(c-'0');
c=r->read();
if(fraction) divideBy*=10;
}
if(fraction) v=v/divideBy;
return v;
}
Token SlashStarState::nextToken(shared_ptr<PushbackReader> r, int theStar, Tokenizer &t){
int c = 0;
int lastc=0;
while(c>=0){
if((lastc=='*') && (c=='/')){
break;
}
lastc=c;
c=r->read();
}
return t.nextToken();
}
Token SlashSlashState::nextToken(shared_ptr<PushbackReader> r, int theSlash, Tokenizer &t){
int c;
while((c=r->read())!='\n' &&c!='\r'&&c>=0){}
return t.nextToken();
}
Token SlashState::nextToken(shared_ptr<PushbackReader> r, int theslash, Tokenizer& t){
int c = r->read();
if(c=='*') return slashStarState.nextToken(r,'*',t);
if(c=='/') return slashSlashState.nextToken(r,'/',t);
if(c>=0) r->unread(c);
return Token(Token::TT_SYMBOL,"/",0);
}
QuoteState::QuoteState(){
buflen=16;
charbuf=new char[buflen];
}
QuoteState::~QuoteState(){
delete[] charbuf;
}
Token QuoteState::nextToken(shared_ptr<PushbackReader> r, int cin, Tokenizer & t){
int i = 0;
charbuf[i++]=(char)cin;
int c;
do{
c=r->read();
if(c<0){
c=cin;
}
checkBufLength(i);
charbuf[i++]=(char)c;
}while(c!=cin);
string sval(charbuf,i);
return Token(Token::TT_QUOTED,sval,0);
}
void QuoteState::checkBufLength(int in){
if(in>=buflen){
int nlen = buflen*2;
char* nbuf = new char[nlen];
for(int i =0;i<buflen; i++){
nbuf[i]=charbuf[i];
}
delete[] charbuf;
charbuf=nbuf;
buflen=nlen;
}
}
void Tokenizer::setString(string s, int symbolMax){
setReader(shared_ptr<PushbackReader>(new PushbackReader(unique_ptr<istream>(new istringstream(s)),symbolMax)));
}
void Tokenizer::setString(string s){
setString(s,DEFAULT_SYMBOL_MAX);
}
void Tokenizer::setCharacterState(int from, int to, TokenizerState *state){
for(int i = from; i<=to; i++){
if(i>=0&&i<256) characterState[i]=state;
}
}
Token Tokenizer::nextToken(){
int c = reader->read();
if(c>=0 &&c<256){
return characterState[c]->nextToken(reader,c,*this);
}
return Token::T_EOF;
}
Tokenizer::Tokenizer(){
setCharacterState(0, 255, getSymbolState()); // the default
setCharacterState( 0, ' ', getWhitespaceState());
setCharacterState( 'a', 'z', getWordState());
setCharacterState( 'A', 'Z', getWordState());
setCharacterState(0xc0, 0xff, getWordState());
setCharacterState( '0', '9', getNumberState());
setCharacterState( '-', '-', getNumberState());
setCharacterState( '.', '.', getNumberState());
setCharacterState( '"', '"', getQuoteState());
setCharacterState( '\'', '\'', getQuoteState());
setCharacterState( '/', '/', getSlashState());
}
Tokenizer::Tokenizer(string s){
setCharacterState(0, 255, getSymbolState()); // the default
setCharacterState( 0, ' ', getWhitespaceState());
setCharacterState( 'a', 'z', getWordState());
setCharacterState( 'A', 'Z', getWordState());
setCharacterState(0xc0, 0xff, getWordState());
setCharacterState( '0', '9', getNumberState());
setCharacterState( '-', '-', getNumberState());
setCharacterState( '.', '.', getNumberState());
setCharacterState( '"', '"', getQuoteState());
setCharacterState( '\'', '\'', getQuoteState());
setCharacterState( '/', '/', getSlashState());
setString(s);
}
string TokenString::toString(){
ostringstream oss;
for(int i = 0; i<numTokens; i++){
if(i>0) oss<<" ";
oss<<tokens[i].toString()<<endl;
}
return oss.str();
}
TokenString::TokenString(Tokenizer t){
vector<Token> v;
while(true){
Token tok = t.nextToken();
if(tok.type() == Token::TT_EOF) break;
v.push_back(tok);
}
numTokens=v.size();
tokens = new Token[v.size()];
for(int i = 0;i<v.size();i++){
tokens[i]=v[i];
}
}
TokenString::TokenString(string s){
Tokenizer t(s);
vector<Token> v;
while(true){
Token tok = t.nextToken();
if(tok.type() == Token::TT_EOF) break;
v.push_back(tok);
}
numTokens=v.size();
tokens = new Token[v.size()];
for(int i = 0;i<v.size();i++){
tokens[i]=v[i];
}
}
TokenString::TokenString(Token * t, int size){
tokens = new Token[size];
numTokens=size;
for(int i = 0; i <size; i++){
tokens[i]=t[i];
}
}
TokenString::TokenString(const TokenString &t){
numTokens=t.numTokens;
tokens=new Token[numTokens];
for(int i = 0; i<numTokens; i++){
tokens[i]=t.tokens[i];
}
}
TokenString& TokenString::operator=(const TokenString& t){
if(tokens) delete[] tokens;
numTokens=t.numTokens;
tokens=new Token[numTokens];
for(int i = 0; i<numTokens; i++){
tokens[i]=t.tokens[i];
}
return *this;
}
TokenString::~TokenString(){
delete[] tokens;
}
vector<Token> TokenStringSource::nextVector(){
vector<Token> v;
while(true){
Token tok = tokenizer.nextToken();
if(tok.type()==Token::TT_EOF ||tok.strval()==delimiter)break;
v.push_back(tok);
}
return v;
}
TokenString TokenStringSource::nextTokenString(){
ensureChacheIsLoaded();
TokenString returnTokenString(*cachedTokenString);
delete cachedTokenString;
cachedTokenString=NULL;
return returnTokenString;
}
TokenStringSource::~TokenStringSource(){
if(cachedTokenString) delete cachedTokenString;
}
void TokenStringSource::loadChache(){
vector<Token> tokenvec = nextVector();
if(tokenvec.empty()) cachedTokenString=NULL;
else{
cachedTokenString = new TokenString(&tokenvec[0],tokenvec.size());
}
}
bool TokenStringSource::hasMoreTokenStrings(){
ensureChacheIsLoaded();
return cachedTokenString!=NULL;
}
void TokenStringSource::ensureChacheIsLoaded(){
if(cachedTokenString==NULL) loadChache();
}
TokenStringSource::TokenStringSource(Tokenizer t, string delim){
cachedTokenString=NULL;
tokenizer=t;
delimiter=delim;
}
TokenStringSource::TokenStringSource(const TokenStringSource& t){
tokenizer=t.tokenizer;
delimiter=t.delimiter;
cachedTokenString=NULL;
}
TokenStringSource& TokenStringSource::operator=(const TokenStringSource& t){
tokenizer=t.tokenizer;
delimiter=t.delimiter;
cachedTokenString=NULL;
return *this;
}
main.cpp
#include <string>
#include <iostream>
#include <fstream>
#include "parsetokens.h"
#include "parsetokenizer.h"
#include "parseutil.h"
int main(){
std::string s = "I came; I saw; I left in peace;";
parse::Tokenizer tok(s);
parse::TokenStringSource tss(tok,";");
while(tss.hasMoreTokenStrings()){
std::cout<<tss.nextTokenString().toString()<<std::endl;
}
return 0;
}
Again sorry about the codes size, anything you can see that could be the problem would be helpful. Runtime errors like this drive me crazy and I always have trouble figuring out where the real problem is.
