kmail

kmmsgdict.cpp

00001 /* kmail message dictionary */
00002 /* Author: Ronen Tzur <rtzur@shani.net> */
00003 
00004 #include "kmfolderindex.h"
00005 #include "kmfolder.h"
00006 #include "kmmsgdict.h"
00007 #include "kmdict.h"
00008 #include "globalsettings.h"
00009 #include "folderstorage.h"
00010 
00011 #include <qfileinfo.h>
00012 
00013 #include <kdebug.h>
00014 #include <kstaticdeleter.h>
00015 
00016 #include <stdio.h>
00017 #include <unistd.h>
00018 
00019 #include <string.h>
00020 #include <errno.h>
00021 
00022 #include <config.h>
00023 
00024 #ifdef HAVE_BYTESWAP_H
00025 #include <byteswap.h>
00026 #endif
00027 
00028 // We define functions as kmail_swap_NN so that we don't get compile errors
00029 // on platforms where bswap_NN happens to be a function instead of a define.
00030 
00031 /* Swap bytes in 32 bit value.  */
00032 #ifdef bswap_32
00033 #define kmail_swap_32(x) bswap_32(x)
00034 #else
00035 #define kmail_swap_32(x) \
00036      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) |           \
00037       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
00038 #endif
00039 
00040 
00041 //-----------------------------------------------------------------------------
00042 
00043 // Current version of the .index.ids files
00044 #define IDS_VERSION 1002
00045 
00046 // The asterisk at the end is important
00047 #define IDS_HEADER "# KMail-Index-IDs V%d\n*"
00048 
00053 class KMMsgDictEntry : public KMDictItem
00054 {
00055 public:
00056   KMMsgDictEntry(const KMFolder *aFolder, int aIndex)
00057   : folder( aFolder ), index( aIndex )
00058     {}
00059 
00060   const KMFolder *folder;
00061   int index;
00062 };
00063 
00071 class KMMsgDictREntry
00072 {
00073 public:
00074   KMMsgDictREntry(int size = 0)
00075   {
00076     array.resize(size);
00077     memset(array.data(), 0, array.size() * sizeof(KMMsgDictEntry *));  // faster than a loop
00078     fp = 0;
00079     swapByteOrder = false;
00080     baseOffset = 0;
00081   }
00082 
00083   ~KMMsgDictREntry()
00084   {
00085     array.resize(0);
00086     if (fp)
00087       fclose(fp);
00088   }
00089 
00090   void set(int index, KMMsgDictEntry *entry)
00091   {
00092     if (index >= 0) {
00093       int size = array.size();
00094       if (index >= size) {
00095         int newsize = QMAX(size + 25, index + 1);
00096         array.resize(newsize);
00097         for (int j = size; j < newsize; j++)
00098           array.at(j) = 0;
00099       }
00100       array.at(index) = entry;
00101     }
00102   }
00103 
00104   KMMsgDictEntry *get(int index)
00105   {
00106     if (index >= 0 && (unsigned)index < array.size())
00107       return array.at(index);
00108     return 0;
00109   }
00110 
00111   ulong getMsn(int index)
00112   {
00113     KMMsgDictEntry *entry = get(index);
00114     if (entry)
00115       return entry->key;
00116     return 0;
00117   }
00118 
00119   int getRealSize()
00120   {
00121     int count = array.size() - 1;
00122     while (count >= 0) {
00123       if (array.at(count))
00124         break;
00125       count--;
00126     }
00127     return count + 1;
00128   }
00129 
00130   void sync()
00131   {
00132     fflush(fp);
00133   }
00134 
00135 public:
00136   QMemArray<KMMsgDictEntry *> array;
00137   FILE *fp;
00138   bool swapByteOrder;
00139   off_t baseOffset;
00140 };
00141 
00142 
00143 static KStaticDeleter<KMMsgDict> msgDict_sd;
00144 KMMsgDict * KMMsgDict::m_self = 0;
00145 
00146 //-----------------------------------------------------------------------------
00147 
00148 KMMsgDict::KMMsgDict()
00149 {
00150   int lastSizeOfDict = GlobalSettings::self()->msgDictSizeHint();
00151   lastSizeOfDict = ( lastSizeOfDict * 11 ) / 10;
00152   GlobalSettings::self()->setMsgDictSizeHint( 0 );
00153   dict = new KMDict( lastSizeOfDict );
00154   nextMsgSerNum = 1;
00155   m_self = this;
00156 }
00157 
00158 //-----------------------------------------------------------------------------
00159 
00160 KMMsgDict::~KMMsgDict()
00161 {
00162   delete dict;
00163 }
00164 
00165 //-----------------------------------------------------------------------------
00166 
00167 const KMMsgDict* KMMsgDict::instance()
00168 {
00169   if ( !m_self ) {
00170     msgDict_sd.setObject( m_self, new KMMsgDict() );
00171   }
00172   return m_self;
00173 }
00174 
00175 KMMsgDict* KMMsgDict::mutableInstance()
00176 {
00177   if ( !m_self ) {
00178     msgDict_sd.setObject( m_self, new KMMsgDict() );
00179   }
00180   return m_self;
00181 }
00182 
00183 //-----------------------------------------------------------------------------
00184 
00185 unsigned long KMMsgDict::getNextMsgSerNum() {
00186   unsigned long msn = nextMsgSerNum;
00187   nextMsgSerNum++;
00188   return msn;
00189 }
00190 
00191 void KMMsgDict::deleteRentry(KMMsgDictREntry *entry)
00192 {
00193   delete entry;
00194 }
00195 
00196 unsigned long KMMsgDict::insert(unsigned long msgSerNum,
00197                                 const KMMsgBase *msg, int index)
00198 {
00199   unsigned long msn = msgSerNum;
00200   if (!msn) {
00201     msn = getNextMsgSerNum();
00202   } else {
00203     if (msn >= nextMsgSerNum)
00204       nextMsgSerNum = msn + 1;
00205   }
00206 
00207   KMFolderIndex* folder = static_cast<KMFolderIndex*>( msg->storage() );
00208   if ( !folder ) {
00209     kdDebug(5006) << "KMMsgDict::insert: Cannot insert the message, "
00210       << "null pointer to storage. Requested serial: " << msgSerNum
00211       << endl;
00212     kdDebug(5006) << "  Message info: Subject: " << msg->subject() << ", To: "
00213       << msg->toStrip() << ", Date: " << msg->dateStr() << endl;
00214     return 0;
00215   }
00216 
00217   if (index == -1)
00218     index = folder->find(msg);
00219 
00220   // Should not happen, indicates id file corruption
00221   while (dict->find((long)msn)) {
00222     msn = getNextMsgSerNum();
00223     folder->setDirty( true ); // rewrite id file
00224   }
00225 
00226   // Insert into the dict. Don't use dict->replace() as we _know_
00227   // there is no entry with the same msn, we just made sure.
00228   KMMsgDictEntry *entry = new KMMsgDictEntry(folder->folder(), index);
00229   dict->insert((long)msn, entry);
00230 
00231   KMMsgDictREntry *rentry = folder->rDict();
00232   if (!rentry) {
00233     rentry = new KMMsgDictREntry();
00234     folder->setRDict(rentry);
00235   }
00236   rentry->set(index, entry);
00237 
00238   return msn;
00239 }
00240 
00241 unsigned long KMMsgDict::insert(const KMMsgBase *msg, int index)
00242 {
00243   unsigned long msn = msg->getMsgSerNum();
00244   return insert(msn, msg, index);
00245 }
00246 
00247 //-----------------------------------------------------------------------------
00248 
00249 void KMMsgDict::replace(unsigned long msgSerNum,
00250                const KMMsgBase *msg, int index)
00251 {
00252   KMFolderIndex* folder = static_cast<KMFolderIndex*>( msg->storage() );
00253   if ( !folder ) {
00254     kdDebug(5006) << "KMMsgDict::replace: Cannot replace the message serial "
00255       << "number, null pointer to storage. Requested serial: " << msgSerNum
00256       << endl;
00257     kdDebug(5006) << "  Message info: Subject: " << msg->subject() << ", To: "
00258       << msg->toStrip() << ", Date: " << msg->dateStr() << endl;
00259     return;
00260   }
00261 
00262   if ( index == -1 )
00263     index = folder->find( msg );
00264 
00265   remove( msgSerNum );
00266   KMMsgDictEntry *entry = new KMMsgDictEntry( folder->folder(), index );
00267   dict->insert( (long)msgSerNum, entry );
00268 
00269   KMMsgDictREntry *rentry = folder->rDict();
00270   if (!rentry) {
00271     rentry = new KMMsgDictREntry();
00272     folder->setRDict(rentry);
00273   }
00274   rentry->set(index, entry);
00275 }
00276 
00277 //-----------------------------------------------------------------------------
00278 
00279 void KMMsgDict::remove(unsigned long msgSerNum)
00280 {
00281   long key = (long)msgSerNum;
00282   KMMsgDictEntry *entry = (KMMsgDictEntry *)dict->find(key);
00283   if (!entry)
00284     return;
00285 
00286   if (entry->folder) {
00287     KMMsgDictREntry *rentry = entry->folder->storage()->rDict();
00288     if (rentry)
00289       rentry->set(entry->index, 0);
00290   }
00291 
00292   dict->remove((long)key);
00293 }
00294 
00295 unsigned long KMMsgDict::remove(const KMMsgBase *msg)
00296 {
00297   unsigned long msn = msg->getMsgSerNum();
00298   remove(msn);
00299   return msn;
00300 }
00301 
00302 //-----------------------------------------------------------------------------
00303 
00304 void KMMsgDict::update(const KMMsgBase *msg, int index, int newIndex)
00305 {
00306   KMMsgDictREntry *rentry = msg->parent()->storage()->rDict();
00307   if (rentry) {
00308     KMMsgDictEntry *entry = rentry->get(index);
00309     if (entry) {
00310       entry->index = newIndex;
00311       rentry->set(index, 0);
00312       rentry->set(newIndex, entry);
00313     }
00314   }
00315 }
00316 
00317 //-----------------------------------------------------------------------------
00318 
00319 void KMMsgDict::getLocation(unsigned long key,
00320                             KMFolder **retFolder, int *retIndex) const
00321 {
00322   KMMsgDictEntry *entry = (KMMsgDictEntry *)dict->find((long)key);
00323   if (entry) {
00324     *retFolder = (KMFolder *)entry->folder;
00325     *retIndex = entry->index;
00326   } else {
00327     *retFolder = 0;
00328     *retIndex = -1;
00329   }
00330 }
00331 
00332 void KMMsgDict::getLocation(const KMMsgBase *msg,
00333                             KMFolder **retFolder, int *retIndex) const
00334 {
00335   getLocation(msg->getMsgSerNum(), retFolder, retIndex);
00336 }
00337 
00338 void KMMsgDict::getLocation( const KMMessage * msg, KMFolder * *retFolder, int * retIndex ) const
00339 {
00340   getLocation( msg->toMsgBase().getMsgSerNum(), retFolder, retIndex );
00341 }
00342 
00343 //-----------------------------------------------------------------------------
00344 
00345 unsigned long KMMsgDict::getMsgSerNum(KMFolder *folder, int index) const
00346 {
00347   unsigned long msn = 0;
00348   if ( folder ) {
00349     KMMsgDictREntry *rentry = folder->storage()->rDict();
00350     if (rentry)
00351       msn = rentry->getMsn(index);
00352   }
00353   return msn;
00354 }
00355 
00356 //-----------------------------------------------------------------------------
00357 
00358 //static
00359 QValueList<unsigned long> KMMsgDict::serNumList(QPtrList<KMMsgBase> msgList)
00360 {
00361   QValueList<unsigned long> ret;
00362   for ( unsigned int i = 0; i < msgList.count(); i++ ) {
00363     unsigned long serNum = msgList.at(i)->getMsgSerNum();
00364     assert( serNum );
00365     ret.append( serNum );
00366   }
00367   return ret;
00368 }
00369 
00370 //-----------------------------------------------------------------------------
00371 
00372 QString KMMsgDict::getFolderIdsLocation( const FolderStorage &storage )
00373 {
00374   return storage.indexLocation() + ".ids";
00375 }
00376 
00377 //-----------------------------------------------------------------------------
00378 
00379 bool KMMsgDict::isFolderIdsOutdated( const FolderStorage &storage )
00380 {
00381   bool outdated = false;
00382 
00383   QFileInfo indexInfo( storage.indexLocation() );
00384   QFileInfo idsInfo( getFolderIdsLocation( storage ) );
00385 
00386   if (!indexInfo.exists() || !idsInfo.exists())
00387     outdated = true;
00388   if (indexInfo.lastModified() > idsInfo.lastModified())
00389     outdated = true;
00390 
00391   return outdated;
00392 }
00393 
00394 //-----------------------------------------------------------------------------
00395 
00396 int KMMsgDict::readFolderIds( FolderStorage& storage )
00397 {
00398   if ( isFolderIdsOutdated( storage ) )
00399     return -1;
00400 
00401   QString filename = getFolderIdsLocation( storage );
00402   FILE *fp = fopen(QFile::encodeName(filename), "r+");
00403   if (!fp)
00404     return -1;
00405 
00406   int version = 0;
00407   fscanf(fp, IDS_HEADER, &version);
00408   if (version != IDS_VERSION) {
00409     fclose(fp);
00410     return -1;
00411   }
00412 
00413   bool swapByteOrder;
00414   Q_UINT32 byte_order;
00415   if (!fread(&byte_order, sizeof(byte_order), 1, fp)) {
00416     fclose(fp);
00417     return -1;
00418   }
00419   swapByteOrder = (byte_order == 0x78563412);
00420 
00421   Q_UINT32 count;
00422   if (!fread(&count, sizeof(count), 1, fp)) {
00423     fclose(fp);
00424     return -1;
00425   }
00426   if (swapByteOrder)
00427      count = kmail_swap_32(count);
00428 
00429   // quick consistency check to avoid allocating huge amount of memory
00430   // due to reading corrupt file (#71549)
00431   long pos = ftell(fp);       // store current position
00432   fseek(fp, 0, SEEK_END);
00433   long fileSize = ftell(fp);  // how large is the file ?
00434   fseek(fp, pos, SEEK_SET);   // back to previous position
00435 
00436   // the file must at least contain what we try to read below
00437   if ( (fileSize - pos) < (long)(count * sizeof(Q_UINT32)) ) {
00438     fclose(fp);
00439     return -1;
00440   }
00441 
00442   KMMsgDictREntry *rentry = new KMMsgDictREntry(count);
00443 
00444   for (unsigned int index = 0; index < count; index++) {
00445     Q_UINT32 msn;
00446 
00447     bool readOk = fread(&msn, sizeof(msn), 1, fp);
00448     if (swapByteOrder)
00449        msn = kmail_swap_32(msn);
00450 
00451     if (!readOk || dict->find(msn)) {
00452       for (unsigned int i = 0; i < index; i++) {
00453         msn = rentry->getMsn(i);
00454         dict->remove((long)msn);
00455       }
00456       delete rentry;
00457       fclose(fp);
00458       return -1;
00459     }
00460 
00461     // We found a serial number that is zero. This is not allowed, and would
00462     // later cause problems like in bug 149715.
00463     // Therefore, use a fresh serial number instead
00464     if ( msn == 0 ) {
00465       kdWarning(5006) << "readFolderIds(): Found serial number zero at index " << index
00466                       << " in folder " << filename << endl;
00467       msn = getNextMsgSerNum();
00468       Q_ASSERT( msn != 0 );
00469     }
00470 
00471     // Insert into the dict. Don't use dict->replace() as we _know_
00472     // there is no entry with the same msn, we just made sure.
00473     KMMsgDictEntry *entry = new KMMsgDictEntry( storage.folder(), index);
00474     dict->insert((long)msn, entry);
00475     if (msn >= nextMsgSerNum)
00476       nextMsgSerNum = msn + 1;
00477 
00478     rentry->set(index, entry);
00479   }
00480   // Remember how many items we put into the dict this time so we can create
00481   // it with an appropriate size next time.
00482   GlobalSettings::setMsgDictSizeHint( GlobalSettings::msgDictSizeHint() + count );
00483 
00484   fclose(fp);
00485   storage.setRDict(rentry);
00486 
00487   return 0;
00488 }
00489 
00490 //-----------------------------------------------------------------------------
00491 
00492 KMMsgDictREntry *KMMsgDict::openFolderIds( const FolderStorage& storage, bool truncate)
00493 {
00494   KMMsgDictREntry *rentry = storage.rDict();
00495   if (!rentry) {
00496     rentry = new KMMsgDictREntry();
00497     storage.setRDict(rentry);
00498   }
00499 
00500   if (!rentry->fp) {
00501     QString filename = getFolderIdsLocation( storage );
00502     FILE *fp = truncate ? 0 : fopen(QFile::encodeName(filename), "r+");
00503     if (fp)
00504     {
00505       int version = 0;
00506       fscanf(fp, IDS_HEADER, &version);
00507       if (version == IDS_VERSION)
00508       {
00509          Q_UINT32 byte_order = 0;
00510          fread(&byte_order, sizeof(byte_order), 1, fp);
00511          rentry->swapByteOrder = (byte_order == 0x78563412);
00512       }
00513       else
00514       {
00515          fclose(fp);
00516          fp = 0;
00517       }
00518     }
00519 
00520     if (!fp)
00521     {
00522       fp = fopen(QFile::encodeName(filename), "w+");
00523       if (!fp)
00524       {
00525         kdDebug(5006) << "Dict '" << filename
00526                       << "' cannot open with folder " << storage.label() << ": "
00527                       << strerror(errno) << " (" << errno << ")" << endl;
00528          delete rentry;
00529          rentry = 0;
00530          return 0;
00531       }
00532       fprintf(fp, IDS_HEADER, IDS_VERSION);
00533       Q_UINT32 byteOrder = 0x12345678;
00534       fwrite(&byteOrder, sizeof(byteOrder), 1, fp);
00535       rentry->swapByteOrder = false;
00536     }
00537     rentry->baseOffset = ftell(fp);
00538     rentry->fp = fp;
00539   }
00540 
00541   return rentry;
00542 }
00543 
00544 //-----------------------------------------------------------------------------
00545 
00546 int KMMsgDict::writeFolderIds( const FolderStorage &storage )
00547 {
00548   KMMsgDictREntry *rentry = openFolderIds( storage, true );
00549   if (!rentry)
00550     return 0;
00551   FILE *fp = rentry->fp;
00552 
00553   fseek(fp, rentry->baseOffset, SEEK_SET);
00554   // kdDebug(5006) << "Dict writing for folder " << storage.label() << endl;
00555   Q_UINT32 count = rentry->getRealSize();
00556   if (!fwrite(&count, sizeof(count), 1, fp)) {
00557     kdDebug(5006) << "Dict cannot write count with folder " << storage.label() << ": "
00558                   << strerror(errno) << " (" << errno << ")" << endl;
00559     return -1;
00560   }
00561 
00562   for (unsigned int index = 0; index < count; index++) {
00563     Q_UINT32 msn = rentry->getMsn(index);
00564     if (!fwrite(&msn, sizeof(msn), 1, fp))
00565       return -1;
00566     if ( msn == 0 ) {
00567       kdWarning(5006) << "writeFolderIds(): Serial number of message at index "
00568                       << index << " is zero in folder " << storage.label() << endl;
00569     }
00570   }
00571 
00572   rentry->sync();
00573 
00574   off_t eof = ftell(fp);
00575   QString filename = getFolderIdsLocation( storage );
00576   truncate(QFile::encodeName(filename), eof);
00577   fclose(rentry->fp);
00578   rentry->fp = 0;
00579 
00580   return 0;
00581 }
00582 
00583 //-----------------------------------------------------------------------------
00584 
00585 int KMMsgDict::touchFolderIds( const FolderStorage &storage )
00586 {
00587   KMMsgDictREntry *rentry = openFolderIds( storage, false);
00588   if (rentry) {
00589     rentry->sync();
00590     fclose(rentry->fp);
00591     rentry->fp = 0;
00592   }
00593   return 0;
00594 }
00595 
00596 //-----------------------------------------------------------------------------
00597 
00598 int KMMsgDict::appendToFolderIds( FolderStorage& storage, int index)
00599 {
00600   KMMsgDictREntry *rentry = openFolderIds( storage, false);
00601   if (!rentry)
00602     return 0;
00603   FILE *fp = rentry->fp;
00604 
00605 //  kdDebug(5006) << "Dict appending for folder " << storage.label() << endl;
00606 
00607   fseek(fp, rentry->baseOffset, SEEK_SET);
00608   Q_UINT32 count;
00609   if (!fread(&count, sizeof(count), 1, fp)) {
00610     kdDebug(5006) << "Dict cannot read count for folder " << storage.label() << ": "
00611                   << strerror(errno) << " (" << errno << ")" << endl;
00612     return 0;
00613   }
00614   if (rentry->swapByteOrder)
00615      count = kmail_swap_32(count);
00616 
00617   count++;
00618 
00619   if (rentry->swapByteOrder)
00620      count = kmail_swap_32(count);
00621   fseek(fp, rentry->baseOffset, SEEK_SET);
00622   if (!fwrite(&count, sizeof(count), 1, fp)) {
00623     kdDebug(5006) << "Dict cannot write count for folder " << storage.label() << ": "
00624                   << strerror(errno) << " (" << errno << ")" << endl;
00625     return 0;
00626   }
00627 
00628   long ofs = (count - 1) * sizeof(ulong);
00629   if (ofs > 0)
00630     fseek(fp, ofs, SEEK_CUR);
00631 
00632   Q_UINT32 msn = rentry->getMsn(index);
00633   if (rentry->swapByteOrder)
00634      msn = kmail_swap_32(msn);
00635   if (!fwrite(&msn, sizeof(msn), 1, fp)) {
00636     kdDebug(5006) << "Dict cannot write count for folder " << storage.label() << ": "
00637                   << strerror(errno) << " (" << errno << ")" << endl;
00638     return 0;
00639   }
00640 
00641   rentry->sync();
00642   fclose(rentry->fp);
00643   rentry->fp = 0;
00644 
00645   return 0;
00646 }
00647 
00648 //-----------------------------------------------------------------------------
00649 
00650 bool KMMsgDict::hasFolderIds( const FolderStorage& storage )
00651 {
00652   return storage.rDict() != 0;
00653 }
00654 
00655 //-----------------------------------------------------------------------------
00656 
00657 bool KMMsgDict::removeFolderIds( FolderStorage& storage )
00658 {
00659   storage.setRDict( 0 );
00660   QString filename = getFolderIdsLocation( storage );
00661   return unlink( QFile::encodeName( filename) );
00662 }