Changeset 535

Timestamp:

11/02/05 15:27:31 (8 years ago)

Author:

ahu

Message:

fix pruning of negcache
specialize the common case where we see identical information coming in, and only update the ttd

Location:

trunk/pdns/pdns

Files:

• Makefile.am (modified) (1 diff)
• dnspbench.cc (modified) (2 diffs)
• dnsreplay-mindex.cc (modified) (2 diffs)
• pdns_recursor.cc (modified) (2 diffs)
• recursor_cache.cc (modified) (1 diff)
• recursor_cache.hh (modified) (1 diff)
• syncres.cc (modified) (6 diffs)
• syncres.hh (modified) (2 diffs)

trunk/pdns/pdns/Makefile.am

@@ -48 +48 @@
 
 dnspbench_SOURCES=dnspbench.cc sstuff.hh dnsparser.cc dnsparser.hh dnsrecords.cc dnswriter.cc dnswriter.hh \
-        misc.cc misc.hh rcpgenerator.cc rcpgenerator.hh base64.cc base64.hh
+        misc.cc misc.hh rcpgenerator.cc rcpgenerator.hh base64.cc base64.hh unix_utility.cc logger.cc \
+        statbag.cc
 
 

trunk/pdns/pdns/dnspbench.cc

@@ -4 +4 @@
 #include "dnswriter.hh"
 #include "dnsrecords.hh"
+#include "logger.hh"
+#include "statbag.hh"
 
+Logger L("dnspbench");
+StatBag S;
 
 
@@ -10 +14 @@
 try
 {
+  dnsheader dnsheader;
+  dnsheader.qdcount=htons(1);
+  dnsheader.ancount=htons(1);
+  Socket s(InterNetwork, Datagram);
+  string spacket;
+  char* p=(char*)&dnsheader;
+  spacket.assign(p, p+sizeof(dnsheader));
+  IPEndpoint rem("127.0.0.1",5300);
+  s.sendTo(spacket, rem);
+  
+  return 0;
+
   reportAllTypes();
 
-  cerr<<"sizeof(optString): "<<sizeof(struct optString)<<endl;
+  vector<uint8_t> packet;
 
-  optString os("hallo!");
-
-  cerr<<"optString: '"<<(string)os<<"'\n";
-
-  vector<uint8_t> packet;
-  
   uint16_t type=DNSRecordContent::TypeToNumber(argv[2]);
 

trunk/pdns/pdns/dnsreplay-mindex.cc

@@ -512 +512 @@
         }
         
-        double factor=20;
+        double factor=10;
         
         seconds/=factor;
@@ -624 +624 @@
     gettimeofday(&now, 0);
 
-    mental_time= mental_time + 2*(now-then);
+    mental_time= mental_time + 1*(now-then);
   }
  out:;

trunk/pdns/pdns/pdns_recursor.cc

@@ -501 +501 @@
   if(qcounter) {
     L<<Logger::Error<<"stats: "<<qcounter<<" questions, "<<RC.size()<<" cache entries, "<<SyncRes::s_negcache.size()<<" negative entries, "
-     <<(int)((RC.cacheHits*100.0)/(RC.cacheHits+RC.cacheMisses))<<"% cache hits";
-    L<<Logger::Error<<", outpacket/query ratio "<<(int)(SyncRes::s_outqueries*100.0/SyncRes::s_queries)<<"%";
+     <<(int)((RC.cacheHits*100.0)/(RC.cacheHits+RC.cacheMisses))<<"% cache hits"<<endl;
+    L<<Logger::Error<<"stats: throttle map: "<<SyncRes::s_throttle.size()<<", ns speeds: "<<SyncRes::s_nsSpeeds.size()<<endl;
+    L<<Logger::Error<<"stats: outpacket/query ratio "<<(int)(SyncRes::s_outqueries*100.0/SyncRes::s_queries)<<"%";
     L<<Logger::Error<<", "<<(int)(SyncRes::s_throttledqueries*100.0/(SyncRes::s_outqueries+SyncRes::s_throttledqueries))<<"% throttled, "
      <<SyncRes::s_nodelegated<<" no-delegation drops"<<endl;
@@ -519 +520 @@
   if(now - last_prune > 60) { 
     RC.doPrune();
+    int pruned=0;
+    for(map<string, NegCacheEntry>::iterator i = SyncRes::s_negcache.begin(); i != SyncRes::s_negcache.end();) 
+      if(i->second.ttd > now) {
+        SyncRes::s_negcache.erase(i++);
+        pruned++;
+      }
+      else
+        ++i;
+    //    cerr<<"Pruned "<<pruned<<" records, left "<<SyncRes::s_negcache.size()<<"\n";
     last_prune=time(0);
   }

trunk/pdns/pdns/recursor_cache.cc

@@ -59 +59 @@
 }
   
+/* the code below is rather tricky - it basically replaces the stuff cached for qname by content, but it is special
+   cased for when inserting identical records with only differing ttls, in which case the entry is not
+   touched, but only given a new ttd */
 void MemRecursorCache::replace(const string &qname, const QType& qt,  const set<DNSResourceRecord>& content)
 {
   set<StoredRecord>& stored=d_cache[toLowerCanonic(qname)+"|"+qt.getName()];
 
-  for(set<StoredRecord>::iterator k=stored.begin();k!=stored.end();++k) 
-    k->d_string.prune();
+  set<StoredRecord>::iterator k;
+  typedef vector<set<StoredRecord>::iterator> touched_t;
+  touched_t touched;
 
-  stored.clear();
+  // walk through new content, encode it as new
+  StoredRecord dr;
 
   for(set<DNSResourceRecord>::const_iterator i=content.begin(); i != content.end(); ++i) {
-    StoredRecord dr;
     dr.d_ttd=i->ttl;
     dr.d_string=DNSRR2String(*i);
-    stored.insert(dr);
-    //cerr<<"Storing: "<< toLowerCanonic(qname)+"|"+qt.getName() << " <=> '"<<i->content<<"', ttd="<<i->ttl<<endl;
+    k=stored.find(dr);
+    if(k!=stored.end()) {           // was it there already?
+      //      cerr<<"updating record '"<<qname<<"' -> '"<<i->content<<"'\n";
+      k->d_ttd=i->ttl;              // update ttl
+      touched.push_back(k);         // note that this record is here to stay
+    }
+    else {
+      //      cerr<<"inserting record '"<<qname<<"' -> '"<<i->content<<"'\n";
+      touched.push_back(stored.insert(dr).first);  // same thing
+    }
+  }
+
+  for(k=stored.begin(); k!=stored.end(); ) {                     // walk over the stored set of records
+    touched_t::const_iterator j;                                
+    for(j=touched.begin(); j!=touched.end() && *j != k ; ++j);   // walk over touched iterators
+    if(j==touched.end()) {                                       // this record was not there
+      //      DNSResourceRecord rr=String2DNSRR(qname, qt,  k->d_string, 0); 
+      //      cerr<<"removing from record '"<<qname<<"' '"<<rr.content<<"'\n";
+      k->d_string.prune();                                      
+      stored.erase(k++);                                         // cleanup
+    }
+    else
+      ++k;
   }
 }

trunk/pdns/pdns/recursor_cache.hh

@@ -72 +72 @@
   struct StoredRecord
   {
-    uint32_t d_ttd;
+    mutable uint32_t d_ttd;
     optString<> d_string;
     bool operator<(const StoredRecord& rhs) const
     {
-      return make_pair(d_ttd, d_string) < make_pair(rhs.d_ttd, rhs.d_string);
+      return d_string < rhs.d_string;
+      //      return make_pair(d_ttd, d_string) < make_pair(rhs.d_ttd, rhs.d_string);
     }
   };
   typedef map<string, set<StoredRecord> > cache_t;
-
+private:
   cache_t d_cache;
 };

trunk/pdns/pdns/syncres.cc

@@ -38 +38 @@
 
 map<string,NegCacheEntry> SyncRes::s_negcache;    
+map<string,DecayingEwma> SyncRes::s_nsSpeeds;
 unsigned int SyncRes::s_queries;
 unsigned int SyncRes::s_outgoingtimeouts;
@@ -230 +231 @@
       sqname=ni->second.name;
       sqt="SOA";
-
     }
     else {
@@ -305 +305 @@
 }
 
-static map<string,DecayingEwma> nsSpeeds;
+
 
 struct speedOrder
@@ -325 +325 @@
   for(set<string>::const_iterator i=nameservers.begin();i!=nameservers.end();++i) {
     rnameservers.push_back(*i);
-    DecayingEwma& temp=nsSpeeds[toLower(*i)];
+    DecayingEwma& temp=s_nsSpeeds[toLower(*i)];
     speeds[*i]=temp.get(&d_now);
   }
@@ -406 +406 @@
             LOG<<prefix<<qname<<": error resolving"<<endl;
 
-          nsSpeeds[toLower(*tns)].submit(1000000, &d_now); // 1 sec
+          s_nsSpeeds[toLower(*tns)].submit(1000000, &d_now); // 1 sec
           
           s_throttle.throttle(d_now.tv_sec, remoteIP+"|"+qname+"|"+qtype.getName(),20,5);
@@ -432 +432 @@
       }
       LOG<<prefix<<qname<<": Got "<<result.size()<<" answers from "<<*tns<<" ("<<remoteIP<<"), rcode="<<d_lwr.d_rcode<<", in "<<d_lwr.d_usec/1000<<"ms"<<endl;
-      nsSpeeds[toLower(*tns)].submit(d_lwr.d_usec, &d_now);
+      s_nsSpeeds[toLower(*tns)].submit(d_lwr.d_usec, &d_now);
 
       map<string,set<DNSResourceRecord> > tcache;

trunk/pdns/pdns/syncres.hh

@@ -65 +65 @@
     else if(i->second.ttd > e.ttd || (i->second.count) < e.count) 
       d_cont[t]=e;
-
+  }
+  
+  unsigned int size()
+  {
+    return d_cont.size();
   }
 private:
@@ -165 +169 @@
   unsigned int d_timeouts;
   static map<string,NegCacheEntry> s_negcache;    
+  static map<string,DecayingEwma> s_nsSpeeds;
   static Throttle<string> s_throttle;
 private: