Skip to content

GitLab

Commit 7a40e4ba authored by Simon Morlat's avatar Simon Morlat
Browse files

good optimization parsing time below 2 ms, thanks to algorithm to detect exclusive selectors.

parent e53b79ec
Hide whitespace changes
Inline Side-by-side
Showing with 274 additions and 46 deletions
src/belr.cc
View file @ 7a40e4ba
......@@ -7,6 +7,26 @@
namespace belr{
TransitionMap::TransitionMap(){
for(size_t i=0;i<sizeof(mPossibleChars)/sizeof(bool);++i)
mPossibleChars[i]=false;
}
bool TransitionMap::intersect(const TransitionMap* other){
for(size_t i=0;i<sizeof(mPossibleChars)/sizeof(bool);++i){
if (mPossibleChars[i] && other->mPossibleChars[i]) return true;
}
return false;
}
void TransitionMap::merge(const TransitionMap* other){
for(size_t i=0;i<sizeof(mPossibleChars)/sizeof(bool);++i){
if (other->mPossibleChars[i]) mPossibleChars[i]=true;
}
}
Recognizer::Recognizer(){
}
......@@ -24,7 +44,8 @@ const string &Recognizer::getName()const{
size_t Recognizer::feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos){
size_t match;
ParserLocalContext hctx=ctx->beginParse(shared_from_this());
ParserLocalContext hctx;
if (ctx) ctx->beginParse(hctx, shared_from_this());
match=_feed(ctx, input, pos);
if (match!=string::npos && match>0){
if (0 && mName.size()>0){
......@@ -32,11 +53,45 @@ size_t Recognizer::feed(const shared_ptr<ParserContextBase> &ctx, const string &
cout<<"Matched recognizer '"<<mName<<"' with sequence '"<<matched<<"'."<<endl;
}
}
ctx->endParse(hctx, input, pos, match);
if (ctx) ctx->endParse(hctx, input, pos, match);
return match;
}
bool Recognizer::getTransitionMap(TransitionMap* mask){
bool ret=_getTransitionMap(mask);
if (0 /*!mName.empty()*/){
cout<<"TransitionMap after "<<mName<<endl;
for(int i=0;i<256;++i){
if (mask->mPossibleChars[i]) cout<<(char)i;
}
cout<<endl;
}
return ret;
}
bool Recognizer::_getTransitionMap(TransitionMap* mask){
string input;
input.resize(2,'\0');
for(int i=0;i<256;++i){
input[0]=i;
if (feed(NULL,input,0)==1)
mask->mPossibleChars[i]=true;
}
return true;
}
void Recognizer::optimize(){
optimize(0);
}
void Recognizer::optimize(int recursionLevel){
if (recursionLevel!=0 && mId!=0) return; /*stop on rule except at level 0*/
_optimize(++recursionLevel);
}
CharRecognizer::CharRecognizer(int to_recognize, bool caseSensitive) : mToRecognize(to_recognize), mCaseSensitive(caseSensitive){
if (::tolower(to_recognize)==::toupper(to_recognize)){
/*not a case caseSensitive character*/
......@@ -53,7 +108,12 @@ size_t CharRecognizer::_feed(const shared_ptr<ParserContextBase> &ctx, const str
return ::tolower(input[pos])==mToRecognize ? 1 : string::npos;
}
Selector::Selector(){
void CharRecognizer::_optimize(int recursionLevel){
}
Selector::Selector() : mIsExclusive(false){
}
shared_ptr<Selector> Selector::addRecognizer(const shared_ptr<Recognizer> &r){
......@@ -61,42 +121,83 @@ shared_ptr<Selector> Selector::addRecognizer(const shared_ptr<Recognizer> &r){
return static_pointer_cast<Selector> (shared_from_this());
}
bool Selector::_getTransitionMap(TransitionMap* mask){
for (auto it=mElements.begin(); it!=mElements.end(); ++it){
(*it)->getTransitionMap(mask);
}
return true;
}
size_t Selector::_feedExclusive(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos){
size_t matched=0;
for (auto it=mElements.begin(); it!=mElements.end(); ++it){
matched=(*it)->feed(ctx, input, pos);
if (matched!=string::npos && matched>0) {
return matched;
}
}
return string::npos;
}
size_t Selector::_feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos){
if (mIsExclusive) return _feedExclusive(ctx, input, pos);
size_t matched=0;
size_t bestmatch=0;
shared_ptr<HandlerContextBase> bestBranch;
for (auto it=mElements.begin(); it!=mElements.end(); ++it){
auto br=ctx->branch();
shared_ptr<HandlerContextBase> br;
if (ctx) br=ctx->branch();
matched=(*it)->feed(ctx, input, pos);
if (matched!=string::npos && matched>bestmatch) {
bestmatch=matched;
if (bestBranch) ctx->removeBranch(bestBranch);
bestBranch=br;
}else{
ctx->removeBranch(br);
if (ctx)
ctx->removeBranch(br);
}
}
if (bestmatch==0) return string::npos;
if (bestmatch!=string::npos){
if (ctx && bestmatch!=string::npos){
ctx->merge(bestBranch);
}
return bestmatch;
}
void Selector::_optimize(int recursionLevel){
for (auto it=mElements.begin(); it!=mElements.end(); ++it){
(*it)->optimize(recursionLevel);
}
TransitionMap *all=NULL;
bool intersectionFound=false;
for (auto it=mElements.begin(); it!=mElements.end() && !intersectionFound; ++it){
TransitionMap *cur=new TransitionMap();
(*it)->getTransitionMap(cur);
if (all){
if (cur->intersect(all)){
intersectionFound=true;
}
all->merge(cur);
delete cur;
}else all=cur;
}
if (all) delete all;
if (!intersectionFound){
//cout<<"Selector '"<<getName()<<"' is exclusive."<<endl;
mIsExclusive=true;
}
}
ExclusiveSelector::ExclusiveSelector(){
mIsExclusive=true;
}
size_t ExclusiveSelector::_feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos){
size_t matched=0;
for (auto it=mElements.begin(); it!=mElements.end(); ++it){
matched=(*it)->feed(ctx, input, pos);
if (matched!=string::npos && matched>0) {
return matched;
}
}
return string::npos;
return Selector::_feedExclusive(ctx, input, pos);
}
......@@ -108,6 +209,18 @@ shared_ptr<Sequence> Sequence::addRecognizer(const shared_ptr<Recognizer> &eleme
return static_pointer_cast<Sequence>( shared_from_this());
}
bool Sequence::_getTransitionMap(TransitionMap* mask){
bool isComplete=false;
for (auto it=mElements.begin(); it!=mElements.end(); ++it){
if ((*it)->getTransitionMap(mask)) {
isComplete=true;
break;
}
}
return isComplete;
}
size_t Sequence::_feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos){
size_t matched=0;
size_t total=0;
......@@ -123,6 +236,12 @@ size_t Sequence::_feed(const shared_ptr<ParserContextBase> &ctx, const string &i
return total;
}
void Sequence::_optimize(int recursionLevel){
for (auto it=mElements.begin(); it!=mElements.end(); ++it)
(*it)->optimize(recursionLevel);
}
Loop::Loop(){
mMin=0;
mMax=-1;
......@@ -151,6 +270,16 @@ size_t Loop::_feed(const shared_ptr<ParserContextBase> &ctx, const string &input
return total;
}
bool Loop::_getTransitionMap(TransitionMap* mask){
mRecognizer->getTransitionMap(mask);
return mMin!=0; //we must say to upper layer that this loop recognizer is allowed to be optional by returning FALSE
}
void Loop::_optimize(int recursionLevel){
mRecognizer->optimize(recursionLevel);
}
CharRange::CharRange(int begin, int end) : mBegin(begin), mEnd(end){
}
......@@ -160,6 +289,11 @@ size_t CharRange::_feed(const shared_ptr<ParserContextBase> &ctx, const string &
return string::npos;
}
void CharRange::_optimize(int recursionLevel){
}
shared_ptr<CharRecognizer> Foundation::charRecognizer(int character, bool caseSensitive){
return make_shared<CharRecognizer>(character, caseSensitive);
}
......@@ -188,6 +322,16 @@ size_t Literal::_feed(const shared_ptr< ParserContextBase >& ctx, const string&
return mLiteralSize;
}
void Literal::_optimize(int recursionLevel){
}
bool Literal::_getTransitionMap(TransitionMap* mask){
mask->mPossibleChars[::tolower(mLiteral[0])]=true;
mask->mPossibleChars[::toupper(mLiteral[0])]=true;
return true;
}
shared_ptr<Recognizer> Utils::literal(const string & lt){
return make_shared<Literal>(lt);
}
......@@ -217,6 +361,12 @@ void RecognizerPointer::setPointed(const shared_ptr<Recognizer> &r){
mRecognizer=r;
}
void RecognizerPointer::_optimize(int recursionLevel){
/*do not call optimize() on the pointed value to avoid a loop.
* The grammar will do it for all rules anyway*/
}
Grammar::Grammar(const string& name) : mName(name){
}
......@@ -309,6 +459,13 @@ bool Grammar::isComplete()const{
return ret;
}
void Grammar::optimize(){
for(auto it=mRules.begin(); it!=mRules.end(); ++it){
(*it).second->optimize();
}
}
int Grammar::getNumRules() const{
return mRules.size();
}
......
src/belr.hh
View file @ 7a40e4ba
......@@ -14,6 +14,13 @@ string tolower(const string &str);
class ParserContextBase;
struct TransitionMap{
TransitionMap();
bool intersect(const TransitionMap *other);
void merge(const TransitionMap *other);
bool mPossibleChars[256];
};
class Recognizer : public enable_shared_from_this<Recognizer>{
public:
void setName(const string &name);
......@@ -22,7 +29,13 @@ public:
unsigned int getId()const{
return mId;
}
bool getTransitionMap(TransitionMap *mask);
void optimize();
void optimize(int recursionLevel);
protected:
/*returns true if the transition map is complete, false otherwise*/
virtual bool _getTransitionMap(TransitionMap *mask);
virtual void _optimize(int recursionLevel)=0;
Recognizer();
virtual size_t _feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos)=0;
string mName;
......@@ -33,6 +46,7 @@ class CharRecognizer : public Recognizer{
public:
CharRecognizer(int to_recognize, bool caseSensitive=false);
private:
virtual void _optimize(int recursionLevel);
virtual size_t _feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos);
int mToRecognize;
bool mCaseSensitive;
......@@ -43,8 +57,12 @@ public:
Selector();
shared_ptr<Selector> addRecognizer(const shared_ptr<Recognizer> &element);
protected:
virtual void _optimize(int recursionLevel);
virtual size_t _feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos);
size_t _feedExclusive(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos);
virtual bool _getTransitionMap(TransitionMap *mask);
list<shared_ptr<Recognizer>> mElements;
bool mIsExclusive;
};
/**This is an optimization of the first one for the case where there can be only a single match*/
......@@ -59,6 +77,9 @@ class Sequence : public Recognizer{
public:
Sequence();
shared_ptr<Sequence> addRecognizer(const shared_ptr<Recognizer> &element);
virtual bool _getTransitionMap(TransitionMap *mask);
protected:
virtual void _optimize(int recursionLevel);
private:
virtual size_t _feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos);
list<shared_ptr<Recognizer>> mElements;
......@@ -68,6 +89,9 @@ class Loop : public Recognizer{
public:
Loop();
shared_ptr<Loop> setRecognizer(const shared_ptr<Recognizer> &element, int min=0, int max=-1);
virtual bool _getTransitionMap(TransitionMap *mask);
protected:
virtual void _optimize(int recursionLevel);
private:
virtual size_t _feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos);
shared_ptr<Recognizer> mRecognizer;
......@@ -88,6 +112,7 @@ class CharRange : public Recognizer{
public:
CharRange(int begin, int end);
private:
virtual void _optimize(int recursionLevel);
virtual size_t _feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos);
int mBegin,mEnd;
};
......@@ -95,7 +120,9 @@ private:
class Literal : public Recognizer{
public:
Literal(const string &lit);
virtual bool _getTransitionMap(TransitionMap *mask);
private:
virtual void _optimize(int recursionLevel);
virtual size_t _feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos);
string mLiteral;
size_t mLiteralSize;
......@@ -113,6 +140,7 @@ public:
shared_ptr<Recognizer> getPointed();
void setPointed(const shared_ptr<Recognizer> &r);
private:
virtual void _optimize(int recursionLevel);
virtual size_t _feed(const shared_ptr<ParserContextBase> &ctx, const string &input, size_t pos);
shared_ptr<Recognizer> mRecognizer;
};
......@@ -140,6 +168,7 @@ public:
**/
shared_ptr<Recognizer> getRule(const string &name);
bool isComplete()const;
void optimize();
int getNumRules()const;
private:
void assignRule(const string &name, const shared_ptr<Recognizer> &rule);
......
src/grammarbuilder.cc
View file @ 7a40e4ba
......@@ -92,8 +92,12 @@ shared_ptr< Recognizer > ABNFElement::buildRecognizer(const shared_ptr< Grammar
return mElement->buildRecognizer(grammar);
if (!mRulename.empty())
return grammar->getRule(mRulename);
if (!mCharVal.empty())
return Utils::literal(mCharVal);
if (!mCharVal.empty()){
if (mCharVal.size()==1)
return Foundation::charRecognizer(mCharVal[0],false);
else
return Utils::literal(mCharVal);
}
cerr<<"ABNFElement::buildRecognizer is empty, should not happen!"<<endl;
abort();
return NULL;
......@@ -323,6 +327,8 @@ shared_ptr<Grammar> ABNFGrammarBuilder::createFromAbnf(const string &path, const
cout<<"Succesfully created grammar with "<<retGram->getNumRules()<<" rules."<<endl;
if (retGram->isComplete()){
cout<<"Grammar is complete."<<endl;
retGram->optimize();
cout<<"Grammar has been optimized."<<endl;
}else{
cout<<"WARNING: grammar is not complete."<<endl;
}
......
src/parser-impl.cc
View file @ 7a40e4ba
......@@ -49,17 +49,17 @@ void Assignment<_parserElementT>::invoke(_parserElementT parent, const string &i
mCollector->invokeWithChild(parent, mChild->realize(input,mBegin,mCount));
}else{
string value=input.substr(mBegin, mCount);
shared_ptr<CollectorBase<_parserElementT,const string&>> cc1=dynamic_pointer_cast<CollectorBase<_parserElementT,const string&>>(mCollector);
CollectorBase<_parserElementT,const string&>* cc1=dynamic_cast<CollectorBase<_parserElementT,const string&>*>(mCollector);
if (cc1){
cc1->invoke(parent, value);
return;
}
shared_ptr<CollectorBase<_parserElementT,const char*>> cc2=dynamic_pointer_cast<CollectorBase<_parserElementT,const char*>>(mCollector);
CollectorBase<_parserElementT,const char*>* cc2=dynamic_cast<CollectorBase<_parserElementT,const char*>*>(mCollector);
if (cc2){
cc2->invoke(parent, value.c_str());
return;
}
shared_ptr<CollectorBase<_parserElementT,int>> cc3=dynamic_pointer_cast<CollectorBase<_parserElementT,int>>(mCollector);
CollectorBase<_parserElementT,int> *cc3=dynamic_cast<CollectorBase<_parserElementT,int>*>(mCollector);
if (cc3){
cc3->invoke(parent, atoi(value.c_str()));
return;
......@@ -74,12 +74,12 @@ void Assignment<_parserElementT>::invoke(_parserElementT parent, const string &i
template <typename _parserElementT>
HandlerContext<_parserElementT>::HandlerContext(const shared_ptr<ParserHandlerBase<_parserElementT>> &handler) :
mHandler(handler){
mHandler(*handler.get()){
}
template <typename _parserElementT>
void HandlerContext<_parserElementT>::setChild(unsigned int subrule_id, size_t begin, size_t count, const shared_ptr<HandlerContext<_parserElementT>> &child){
auto collector=mHandler->getCollector(subrule_id);
auto collector=mHandler.getCollector(subrule_id);
if (collector){
mAssignments.push_back(Assignment<_parserElementT>(collector, begin, count, child));
}
......@@ -87,7 +87,7 @@ void HandlerContext<_parserElementT>::setChild(unsigned int subrule_id, size_t b
template <typename _parserElementT>
_parserElementT HandlerContext<_parserElementT>::realize(const string &input, size_t begin, size_t count){
_parserElementT ret=mHandler->invoke(input, begin, count);
_parserElementT ret=mHandler.invoke(input, begin, count);
for (auto it=mAssignments.begin(); it!=mAssignments.end(); ++it){
(*it).invoke(ret,input);
}
......@@ -96,7 +96,7 @@ _parserElementT HandlerContext<_parserElementT>::realize(const string &input, si
template <typename _parserElementT>
shared_ptr<HandlerContext<_parserElementT>> HandlerContext<_parserElementT>::branch(){
return make_shared<HandlerContext>(mHandler);
return mHandler.createContext();
}
template <typename _parserElementT>
......@@ -116,6 +116,12 @@ void HandlerContext<_parserElementT>::undoAssignments(size_t pos){
mAssignments.erase(mAssignments.begin()+pos,mAssignments.end());
}
template <typename _parserElementT>
void HandlerContext< _parserElementT >::recycle(){
mAssignments.clear();
mHandler.releaseContext(static_pointer_cast<HandlerContext< _parserElementT >>(shared_from_this()));
}
//
// ParserHandlerBase template class implementation
//
......@@ -141,6 +147,21 @@ const shared_ptr<AbstractCollector<_parserElementT>> & ParserHandlerBase<_parser
return mParser.mNullCollector;
}
template <typename _parserElementT>
void ParserHandlerBase< _parserElementT >::releaseContext(const shared_ptr<HandlerContext<_parserElementT>> &ctx){
mCachedContext=ctx;
}
template <typename _parserElementT>
shared_ptr<HandlerContext<_parserElementT>> ParserHandlerBase<_parserElementT>::createContext(){
if (mCachedContext) {
shared_ptr<HandlerContext<_parserElementT>> ret=mCachedContext;
mCachedContext.reset();
return ret;
}
return make_shared<HandlerContext<_parserElementT>>(this->shared_from_this());
}
//
// ParserHandler template implementation
//
......@@ -154,12 +175,17 @@ _parserElementT ParserHandler<_derivedParserElementT,_parserElementT>::invoke(co
return NULL;
}
//
// ParserContext template class implementation
//
template <typename _parserElementT>
ParserContext<_parserElementT>::ParserContext(Parser<_parserElementT> &parser) : mParser(parser){
}
template <typename _parserElementT>
ParserLocalContext ParserContext<_parserElementT>::_beginParse(const shared_ptr<Recognizer> &rec){
void ParserContext<_parserElementT>::_beginParse(ParserLocalContext & lctx, const shared_ptr<Recognizer> &rec){
shared_ptr<HandlerContextBase> ctx;
auto h=mParser.getHandler(rec->getId());
......@@ -167,7 +193,7 @@ ParserLocalContext ParserContext<_parserElementT>::_beginParse(const shared_ptr<
ctx=h->createContext();
mHandlerStack.push_back(static_pointer_cast<HandlerContext<_parserElementT>>(ctx));
}
return ParserLocalContext(ctx,rec,mHandlerStack.back()->getLastIterator());
lctx.set(ctx,rec,mHandlerStack.back()->getLastIterator());
}
template <typename _parserElementT>
......@@ -184,11 +210,15 @@ void ParserContext<_parserElementT>::_endParse(const ParserLocalContext &localct
/*no parent, this is our root object*/
mRoot=static_pointer_cast<HandlerContext< _parserElementT >>(localctx.mHandlerContext);
}
}else{
//no match
static_pointer_cast<HandlerContext< _parserElementT >>(localctx.mHandlerContext)->recycle();
}
}else{
if (count!=string::npos && count>0){
/*assign string to parent */
mHandlerStack.back()->setChild(localctx.mRecognizer->getId(), begin, count, NULL);
if (localctx.mRecognizer->getId()!=0)
mHandlerStack.back()->setChild(localctx.mRecognizer->getId(), begin, count, NULL);
}else{
mHandlerStack.back()->undoAssignments(localctx.mAssignmentPos);
}
......@@ -217,7 +247,8 @@ void ParserContext<_parserElementT>::_merge(const shared_ptr<HandlerContext<_par
abort();
}
mHandlerStack.pop_back();
return mHandlerStack.back()->merge(other);
mHandlerStack.back()->merge(other);
other->recycle();
}
template <typename _parserElementT>
......@@ -230,11 +261,12 @@ void ParserContext<_parserElementT>::_removeBranch(const shared_ptr<HandlerConte
advance(it,1);
mHandlerStack.erase(it.base());
}
other->recycle();
}
template <typename _parserElementT>
ParserLocalContext ParserContext<_parserElementT>::beginParse(const shared_ptr<Recognizer> &rec){
return _beginParse(rec);
void ParserContext<_parserElementT>::beginParse(ParserLocalContext &ctx, const shared_ptr<Recognizer> &rec){
_beginParse(ctx, rec);
}
template <typename _parserElementT>
......@@ -257,11 +289,6 @@ void ParserContext<_parserElementT>::removeBranch(const shared_ptr<HandlerContex
_removeBranch(static_pointer_cast<HandlerContext<_parserElementT>>(other));
}
template <typename _parserElementT>
shared_ptr<HandlerContext<_parserElementT>> ParserHandlerBase<_parserElementT>::createContext(){
return make_shared<HandlerContext<_parserElementT>>(this->shared_from_this());
}
//
// Parser template class implementation
//
......
src/parser.hh
View file @ 7a40e4ba
......@@ -50,23 +50,27 @@ class HandlerContext;
template <typename _parserElementT>
class Parser;
class HandlerContextBase;
template <typename _parserElementT>
class ParserHandlerBase : public enable_shared_from_this<ParserHandlerBase<_parserElementT>>{
friend class HandlerContext<_parserElementT>;
public:
ParserHandlerBase(const Parser<_parserElementT> &parser, const string &name);
virtual _parserElementT invoke(const string &input, size_t begin, size_t count)=0;
shared_ptr<HandlerContext<_parserElementT>> createContext();
const string &getRulename()const{
return mRulename;
}
protected:
void releaseContext(const shared_ptr<HandlerContext<_parserElementT>> &ctx);
ParserHandlerBase(const Parser<_parserElementT> &parser, const string &name);
void installCollector(const string &rulename, const shared_ptr<AbstractCollector<_parserElementT>> &collector);
const shared_ptr<AbstractCollector<_parserElementT>> &getCollector(unsigned int rule_id)const;
private:
map<unsigned int, shared_ptr<AbstractCollector<_parserElementT>> > mCollectors;
const Parser<_parserElementT> &mParser;
string mRulename;
shared_ptr<HandlerContext<_parserElementT>> mCachedContext;
};
template <typename _derivedParserElementT, typename _parserElementT>
......@@ -100,19 +104,19 @@ private:
template <typename _parserElementT>
class Assignment{
private: