Changeset 656

Timestamp:

04/04/06 17:26:07 (4 years ago)

Author:

ahu

Message:

move cache to being pruned 'CLOCK' fashion

Location:

trunk/pdns/pdns

Files:

• pdns_recursor.cc (modified) (2 diffs)
• rec_channel_rec.cc (modified) (2 diffs)
• recursor_cache.cc (modified) (6 diffs)
• recursor_cache.hh (modified) (4 diffs)

trunk/pdns/pdns/pdns_recursor.cc

@@ -290 +290 @@
     if(res<0) {
       pw.getHeader()->rcode=RCode::ServFail;
-      pw.commit();
+      // no commit here, because no record
       g_stats.servFails++;
     }
@@ -679 +679 @@
     ::arg().set("max-tcp-clients","Maximum number of simultaneous TCP clients")="128";
     ::arg().set("hint-file", "If set, load root hints from this file")="";
+    ::arg().set("max-cache-entries", "If set, maximum number of entries in the main cache")="0";
 
     ::arg().setCmd("help","Provide a helpful message");

trunk/pdns/pdns/rec_channel_rec.cc

@@ -92 +92 @@
 }
 
+template<typename T>
+string doSlashCache(T begin, T end)
+{
+  RC.doSlash(10);
+
+  return "done\n";
+}
 
 uint32_t getQueryRate()
@@ -187 +194 @@
     return doDumpCache(begin, end);
 
+  if(cmd=="slash-cache") 
+    return doSlashCache(begin, end);
+
   if(cmd=="wipe-cache") 
     return doWipeCache(begin, end);

trunk/pdns/pdns/recursor_cache.cc

@@ -4 +4 @@
 #include <boost/shared_ptr.hpp>
 #include "dnsrecords.hh"
+#include <malloc.h>
+#include "arguments.hh"
 
 using namespace std;
@@ -100 +102 @@
     if(res) {
       for(cache_t::const_iterator i=d_cachecache.first; i != d_cachecache.second; ++i) 
-        if(i->d_qtype == qt.getCode()) 
+        if(i->d_qtype == qt.getCode()) {
+          typedef cache_t::nth_index<1>::type sequence_t;
+          sequence_t& sidx=d_cache.get<1>();
+          sequence_t::iterator si=d_cache.project<1>(i);
+
           for(vector<StoredRecord>::const_iterator k=i->d_records.begin(); k != i->d_records.end(); ++k) {
             if(k->d_ttd > (uint32_t) now) {
@@ -107 +113 @@
             }
           }
+          if(res->empty())
+            sidx.relocate(sidx.begin(), si); 
+          else
+            sidx.relocate(sidx.end(), si); 
+          break;
+        }
     }
 
@@ -124 +136 @@
   tuple<string, uint16_t> key=make_tuple(toLowerCanonic(qname), qt.getCode());
   cache_t::iterator stored=d_cache.find(key);
+
+  //  cerr<<"storing "<< toLowerCanonic(qname)+"|"+qt.getName()<<"\n";
   
   bool isNew=false;
@@ -180 +194 @@
     return;
   }
+
+  typedef cache_t::nth_index<1>::type sequence_t;
+  sequence_t& sidx=d_cache.get<1>();
+
   time_t now=time(0);
-  for(cache_t::const_iterator i=d_cache.begin(); i!=d_cache.end(); ++i) {
+  for(sequence_t::const_iterator i=sidx.begin(); i != sidx.end(); ++i) {
     for(vector<StoredRecord>::const_iterator j=i->d_records.begin(); j != i->d_records.end(); ++j) {
       DNSResourceRecord rr=String2DNSRR(i->d_qname, QType(i->d_qtype), j->d_string, j->d_ttd - now);
@@ -190 +208 @@
 }
 
+void MemRecursorCache::doSlash(int perc)
+{
+  doPrune();
+}
+
 void MemRecursorCache::doPrune(void)
 {
   uint32_t now=(uint32_t)time(0);
   d_cachecachevalid=false;
-//  cout<<"Going to prune!\n";
-
-  typedef cache_t::nth_index<1>::type cache_by_ttd_t;
-  cache_by_ttd_t& ttdindex=d_cache.get<1>();
-
-  uint32_t looked(0), quickZonk(0), fullZonk(0), partialZonk(0), noZonk(0);
-  DTime dt;
-  dt.set(); 
-  cache_by_ttd_t::iterator j;
-  for(j=ttdindex.begin();j!=ttdindex.end();){
-    if(j->getTTD() > now) {
-//      cout<<"Done pruning, this record ("<<j->d_name<<") only needs to get killed in "<< j->getTTD() - now <<" seconds, rest will be later\n";
+
+  unsigned int maxCached=::arg().asNum("max-cache-entries");
+  unsigned int toTrim=0;
+  
+  unsigned int cacheSize=d_cache.size();
+
+  if(maxCached && cacheSize > maxCached)
+    toTrim = cacheSize - maxCached;
+
+  //  cout<<"Need to trim "<<toTrim<<" from cache to meet target!\n";
+
+  typedef cache_t::nth_index<1>::type sequence_t;
+  sequence_t& sidx=d_cache.get<1>();
+
+
+  unsigned int tried=0, lookAt, erased=0;
+
+  // two modes - if toTrim is 0, just look through 10000 records and nuke everything that is expired
+  // otherwise, scan first 5*toTrim records, and stop once we've nuked enough
+  if(toTrim)
+    lookAt=5*toTrim;
+  else
+    lookAt=10000;
+
+
+  sequence_t::iterator iter=sidx.begin(), eiter;
+  for(; iter != sidx.end() && tried < lookAt ; ++tried) {
+    if(iter->getTTD() < now) {
+      sidx.erase(iter++);
+      erased++;
+    }
+    else
+      ++iter;
+
+    if(toTrim && erased > toTrim)
       break;
-    }
-    else 
-        ;
-//      cout<<"Looking at '"<<j->d_name<<"', "<<now - j->getTTD()<<" seconds overdue!\n";
-    looked++;
-    if(j->d_records.size()==1) {
-      j++;
-      quickZonk++;
-      continue;
-    }
-    predicate p(now);
-    CacheEntry ce=*j;
-
-    size_t before=ce.d_records.size();
-    ce.d_records.erase(remove_if(ce.d_records.begin(), ce.d_records.end(), p), ce.d_records.end());
-
-    if(ce.d_records.empty()) { // everything is gone
-//      cout<<"Zonked it entirely!\n";
-      j++;
-      fullZonk++;
-    }
-    else {
-      if(ce.d_records.size()!=before) {
-//      cout<<"Zonked partially, putting back, new TTD: "<< ce.getTTD() - now<<endl;;
-        cache_by_ttd_t::iterator here=j++;
-        ttdindex.replace(here, ce);
-        partialZonk++;
-      }
-      else {
-        ++j;
-        noZonk++;
-        break;
-      }
-    }
-  }
-  
-  //  cout<<"Walk took "<< dt.udiff()<<"usec\n";
-  //  dt.set();
-  ttdindex.erase(ttdindex.begin(), j);
-//    cout<<"Erase took "<< dt.udiff()<<" usec, looked: "<<looked<<", quick: "<<quickZonk<<", full: ";
- //   cout<<fullZonk<<", partial: "<<partialZonk<<", no: "<<noZonk<<"\n";
-  //  cache_t(d_cache).swap(d_cache);
-}
-
+  }
+
+  //  cout<<"erased "<<erased<<" records based on ttd\n";
+  
+  if(erased >= toTrim)
+    return;
+
+  //  if(toTrim)
+  //    cout<<"Still have "<<toTrim - erased<<" entries left to erase to meet target\n";
+
+
+  eiter=iter=sidx.begin();
+  advance(eiter, toTrim);
+  sidx.erase(iter, eiter);
+}
+

trunk/pdns/pdns/recursor_cache.hh

@@ -12 +12 @@
 #include <boost/tuple/tuple_comparison.hpp>
 #include <boost/multi_index/key_extractors.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
 #include <boost/version.hpp>
 #if BOOST_VERSION >= 103300
@@ -32 +33 @@
   void replace(const string &qname, const QType& qt,  const set<DNSResourceRecord>& content);
   void doPrune(void);
+  void doSlash(int perc);
   void doDumpAndClose(int fd);
   void doWipeCache(const string& name);
@@ -103 +105 @@
                       >
                 >,
-                ordered_non_unique<const_mem_fun<CacheEntry,uint32_t,&CacheEntry::getTTD> >
+               sequenced<>
                >
   > cache_t;
@@ -109 +111 @@
 private:
   cache_t d_cache;
-  pair<cache_t::const_iterator, cache_t::const_iterator> d_cachecache;
+  pair<cache_t::iterator, cache_t::iterator> d_cachecache;
   string d_cachedqname;
   bool d_cachecachevalid;