acquire.h

Go to the documentation of this file.

// -*- mode: cpp; mode: fold -*-
// Description                                                          /*{{{*/
/* ######################################################################
 
   Acquire - File Acquiration
   
   This module contains the Acquire system. It is responsible for bringing
   files into the local pathname space. It deals with URIs for files and
   URI handlers responsible for downloading or finding the URIs.
   
   Each file to download is represented by an Acquire::Item class subclassed
   into a specialization. The Item class can add itself to several URI
   acquire queues each prioritized by the download scheduler. When the
   system is run the proper URI handlers are spawned and the acquire
   queues are fed into the handlers by the schedular until the queues are
   empty. This allows for an Item to be downloaded from an alternate source
   if the first try turns out to fail. It also allows concurrent downloading
   of multiple items from multiple sources as well as dynamic balancing
   of load between the sources.
   
   Scheduling of downloads is done on a first ask first get basis. This
   preserves the order of the download as much as possible. And means the
   fastest source will tend to process the largest number of files.
   
   Internal methods and queues for performing gzip decompression,
   md5sum hashing and file copying are provided to allow items to apply
   a number of transformations to the data files they are working with.
   
   ##################################################################### */
                                                                        /*}}}*/
 
 
#ifndef PKGLIB_ACQUIRE_H
#define PKGLIB_ACQUIRE_H
 
#include <apt-pkg/hashes.h>
#include <apt-pkg/macros.h>
#include <apt-pkg/weakptr.h>
 
#include <chrono>
#include <string>
#include <vector>
 
#include <cstddef>
#include <sys/select.h>
#include <sys/time.h>
 
 
 
class pkgAcquireStatus;
class metaIndex;
 
class APT_PUBLIC pkgAcquire
{   
   private:
   using clock = std::chrono::steady_clock;
   using time_point = std::chrono::time_point<clock>;
   int LockFD;
   void * const d;
 
   public:
   
   class Item;
   class Queue;
   class Worker;
   struct MethodConfig;
   struct ItemDesc;
   friend class Item;
   friend class pkgAcqMetaBase;
   friend class Queue;
 
   typedef std::vector<Item *>::iterator ItemIterator;
   typedef std::vector<Item *>::const_iterator ItemCIterator;
 
   protected:
   
   std::vector<Item *> Items;
   
   Queue *Queues;
 
   Worker *Workers;
 
   MethodConfig *Configs;
 
   pkgAcquireStatus *Log;
 
   unsigned long ToFetch;
 
   // Configurable parameters for the scheduler
 
   enum QueueStrategy {
     QueueHost,
     QueueAccess} QueueMode;
 
   bool const Debug;
   bool Running;
 
   void Add(Item *Item);
 
   void Remove(Item *Item);
 
   void Add(Worker *Work);
 
   void Remove(Worker *Work);
   
   void Enqueue(ItemDesc &Item);
 
   void Dequeue(Item *Item);
 
   std::string QueueName(std::string URI,MethodConfig const *&Config);
 
   virtual void SetFds(int &Fd,fd_set *RSet,fd_set *WSet);
 
   virtual bool RunFds(fd_set *RSet,fd_set *WSet);
 
   void Bump();
   
   public:
 
   MethodConfig *GetConfig(std::string Access);
 
   enum RunResult {
     Continue,
 
     Failed,
 
     Cancelled};
 
   RunResult Run(int PulseInterval=500000);
 
   void Shutdown();
   
   inline Worker *WorkersBegin() {return Workers;};
 
   Worker *WorkerStep(Worker *I) APT_PURE;
 
   inline ItemIterator ItemsBegin() {return Items.begin();};
   inline ItemCIterator ItemsBegin() const {return Items.begin();};
 
   inline ItemIterator ItemsEnd() {return Items.end();};
   inline ItemCIterator ItemsEnd() const {return Items.end();};
   
   // Iterate over queued Item URIs
   class UriIterator;
   UriIterator UriBegin();
   UriIterator UriEnd();
   
   bool Clean(std::string Dir);
   
   bool CleanLists(std::string const &Dir);
 
   unsigned long long TotalNeeded();
 
   unsigned long long FetchNeeded();
 
   unsigned long long PartialPresent();
 
   void SetLog(pkgAcquireStatus *Progress) { Log = Progress; }
 
   bool GetLock(std::string const &Lock);
 
   explicit pkgAcquire(pkgAcquireStatus *Log);
   pkgAcquire();
 
   virtual ~pkgAcquire();
 
   APT_HIDDEN static std::string URIEncode(std::string const &part);
 
   private:
   APT_HIDDEN void Initialize();
};
 
struct APT_PUBLIC pkgAcquire::ItemDesc : public WeakPointable
{
   std::string URI;
   std::string Description;
   std::string ShortDesc;
   Item *Owner;
};
                                                                        /*}}}*/
class APT_PUBLIC pkgAcquire::Queue
{
   friend class pkgAcquire;
   friend class pkgAcquire::UriIterator;
   friend class pkgAcquire::Worker;
 
   void * const d;
 
   Queue *Next;
   
   protected:
 
   struct QItem : public ItemDesc
   {
      QItem *Next;
      pkgAcquire::Worker *Worker;
 
      std::vector<Item*> Owners;
 
      unsigned long long CurrentSize = 0;
 
      unsigned long long TotalSize = 0;
 
      unsigned long long ResumePoint = 0;
 
      typedef std::vector<Item*>::const_iterator owner_iterator;
 
      void operator =(pkgAcquire::ItemDesc const &I)
      {
         URI = I.URI;
         Description = I.Description;
         ShortDesc = I.ShortDesc;
         Owners.clear();
         Owners.push_back(I.Owner);
         Owner = I.Owner;
      };
 
      HashStringList GetExpectedHashes() const;
 
      unsigned long long GetMaximumSize() const;
 
      void SyncDestinationFiles() const;
 
      std::string Custom600Headers() const;
      int APT_HIDDEN GetPriority() const;
      time_point APT_HIDDEN GetFetchAfter() const;
   };
 
   std::string Name;
 
   QItem *Items;
 
   pkgAcquire::Worker *Workers;
 
   pkgAcquire *Owner;
 
   signed long PipeDepth;
 
   unsigned long MaxPipeDepth;
   
   public:
   
   bool Enqueue(ItemDesc &Item);
 
   bool Dequeue(Item *Owner);
 
   QItem *FindItem(std::string URI,pkgAcquire::Worker *Owner) APT_PURE;
 
   bool ItemStart(QItem *Itm,unsigned long long Size);
 
   bool ItemDone(QItem *Itm);
   
   bool Startup();
 
   bool Shutdown(bool Final);
 
   bool Cycle();
 
   void Bump();
   
   Queue(std::string const &Name,pkgAcquire * const Owner);
 
   virtual ~Queue();
};
                                                                        /*}}}*/
class APT_PUBLIC pkgAcquire::UriIterator
{
   void * const d;
 
   pkgAcquire::Queue *CurQ;
   pkgAcquire::Queue::QItem *CurItem;
   
   public:
   
   inline void operator ++() {operator ++(0);};
 
   void operator ++(int)
   {
      CurItem = CurItem->Next;
      while (CurItem == 0 && CurQ != 0)
      {
         CurItem = CurQ->Items;
         CurQ = CurQ->Next;
      }
   };
   
   inline pkgAcquire::Queue::QItem const *operator ->() const {return CurItem;};
   inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;};
   inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;};
   
   explicit UriIterator(pkgAcquire::Queue *Q);
   virtual ~UriIterator();
};
                                                                        /*}}}*/
struct APT_PUBLIC pkgAcquire::MethodConfig
{
   class Private;
   Private *const d;
 
   MethodConfig *Next;
   
   std::string Access;
 
   std::string Version;
 
   bool SingleInstance;
 
   bool Pipeline;
 
   bool SendConfig;
 
   bool LocalOnly;
 
   bool NeedsCleanup;
 
   bool Removable;
   
   MethodConfig();
 
   APT_HIDDEN bool GetAuxRequests() const;
   APT_HIDDEN void SetAuxRequests(bool const value);
   APT_HIDDEN bool GetSendURIEncoded() const;
   APT_HIDDEN void SetSendURIEncoded(bool const value);
 
   virtual ~MethodConfig();
};
                                                                        /*}}}*/
class APT_PUBLIC pkgAcquireStatus
{
   void * const d;
 
   protected:
   
   struct timeval Time;
 
   struct timeval StartTime;
 
   unsigned long long LastBytes;
 
   unsigned long long CurrentCPS;
 
   unsigned long long CurrentBytes;
 
   unsigned long long TotalBytes;
 
   unsigned long long FetchedBytes;
 
   unsigned long long ElapsedTime;
 
   unsigned long TotalItems;
 
   unsigned long CurrentItems;
   
   double Percent;
 
   public:
 
   bool Update;
 
   bool MorePulses;
      
   virtual void Fetched(unsigned long long Size,unsigned long long ResumePoint);
   
   virtual bool MediaChange(std::string Media,std::string Drive) = 0;
 
   struct ReleaseInfoChange
   {
      std::string Type; 
      std::string From; 
      std::string To; 
      std::string Message; 
      bool DefaultAction; 
   };
   virtual bool ReleaseInfoChanges(metaIndex const * const LastRelease, metaIndex const * const CurrentRelease, std::vector<ReleaseInfoChange> &&Changes);
   APT_HIDDEN static bool ReleaseInfoChangesAsGlobalErrors(std::vector<ReleaseInfoChange> &&Changes);
 
   virtual void IMSHit(pkgAcquire::ItemDesc &/*Itm*/) {};
 
   virtual void Fetch(pkgAcquire::ItemDesc &/*Itm*/) {};
 
   virtual void Done(pkgAcquire::ItemDesc &/*Itm*/) {};
 
   virtual void Fail(pkgAcquire::ItemDesc &/*Itm*/) {};
 
   virtual bool Pulse(pkgAcquire *Owner);
 
   virtual void Start();
 
   virtual void Stop();
   
   pkgAcquireStatus();
   virtual ~pkgAcquireStatus();
};
                                                                        /*}}}*/
#endif