libzypp  17.36.1
Fetcher.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
12 #include <iostream>
13 #include <fstream>
14 #include <list>
15 #include <map>
16 
17 #include <zypp/base/Easy.h>
18 #include <zypp/base/LogControl.h>
19 #include <zypp/base/LogTools.h>
20 #include <zypp/base/PtrTypes.h>
21 #include <zypp-core/base/DefaultIntegral>
22 #include <zypp/base/String.h>
23 #include <zypp-media/MediaException>
24 #include <zypp/Fetcher.h>
25 #include <zypp/ZYppFactory.h>
26 #include <zypp/CheckSum.h>
27 #include <zypp-core/base/UserRequestException>
30 
31 #undef ZYPP_BASE_LOGGER_LOGGROUP
32 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp::fetcher"
33 
35 namespace zypp
36 {
37 
43  struct FetcherIndex
44  {
46  : location(loc)
47  {}
52  };
53 
54  using FetcherIndex_Ptr = shared_ptr<FetcherIndex>;
55 
58  {
59  bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs ) const
60  {
61  if ( lhs == rhs )
62  return false; // incl. NULL == NULL
63  if ( ! lhs )
64  return true; // NULL < nonNULL
65  if ( ! rhs )
66  return false; // nonNULL > NULL
67  // both nonNULL ==> compare medianr and path
68  if ( lhs->location.medianr() == rhs->location.medianr() )
69  return lhs->location.filename() < rhs->location.filename();
70  //else
71  return lhs->location.medianr() < rhs->location.medianr();
72  }
73  };
74 
79  struct FetcherJob
80  {
81  enum Flag
82  {
83  None = 0x0000,
84  Directory = 0x0001,
85  Recursive = 0x0002,
87  // check checksums even if there is no such
88  // checksum (warns of no checksum)
90  };
91  ZYPP_DECLARE_FLAGS(Flags, Flag);
92 
93  FetcherJob(const FetcherJob &) = default;
94  FetcherJob(FetcherJob &&) = default;
95  FetcherJob &operator=(const FetcherJob &) = default;
96  FetcherJob &operator=(FetcherJob &&) = default;
97 
98  FetcherJob(const OnMediaLocation &loc) : location(loc), flags(None) {
99  //MIL << location << endl;
100  }
101 
103  {
104  //MIL << location << " | * " << checkers.size() << endl;
105  }
106 
108  //CompositeFileChecker checkers;
109  std::list<FileChecker> checkers;
110  Flags flags;
111  };
112 
113  ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
114  using FetcherJob_Ptr = shared_ptr<FetcherJob>;
115 
116  std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
117  {
118  return str << obj->location;
119  }
120 
122  //
123  // CLASS NAME : Fetcher::Impl
124  //
127  {
128  friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
129 
130  public:
131  Impl();
132 
133  Impl(const Impl &) = default;
134  Impl(Impl &&) = delete;
135  Impl &operator=(const Impl &) = delete;
136  Impl &operator=(Impl &&) = delete;
137 
138  ~Impl() {}
139 
140  void setOptions( Fetcher::Options options );
141  Fetcher::Options options() const;
142 
143  void addIndex( const OnMediaLocation &resource );
144 
145  void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
146  void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
147 
148  void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
149  void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
150  void addCachePath( const Pathname &cache_dir );
151  void reset();
152  void setMediaSetAccess ( MediaSetAccess &media );
153  void start( const Pathname &dest_dir,
154  const ProgressData::ReceiverFnc &progress );
155  void start( const Pathname &dest_dir,
156  MediaSetAccess &media,
157  const ProgressData::ReceiverFnc & progress_receiver );
158 
160  static shared_ptr<Impl> nullimpl()
161  {
162  static shared_ptr<Impl> _nullimpl( new Impl );
163  return _nullimpl;
164  }
165  private:
169  void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
170 
174  void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
175 
183  void readIndex( const Pathname &index, const Pathname &basedir );
184 
186  void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
187 
189  void readContentFileIndex( const Pathname &index, const Pathname &basedir );
190 
192  void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
193 
199  Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r );
204  void validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r );
205 
209  void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
210  const Pathname &dest_dir, FetcherJob::Flags flags );
211 
215  void autoaddIndexes( const filesystem::DirContent &content,
216  MediaSetAccess &media,
217  const OnMediaLocation &resource,
218  const Pathname &dest_dir );
222  void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r );
223 
224  private:
225  friend Impl * rwcowClone<Impl>( const Impl * rhs );
227  Impl * clone() const
228  { return new Impl( *this ); }
229 
230  std::list<FetcherJob_Ptr> _resources;
231  std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
232  std::set<Pathname> _caches;
233  // checksums read from the indexes
234  std::map<std::string, CheckSum> _checksums;
235  // cache of dir contents
236  std::map<std::string, filesystem::DirContent> _dircontent;
237 
239 
240  Fetcher::Options _options;
241  };
243 
245  {
246  if ( _mediaSetAccess )
247  _mediaSetAccess->precacheFiles( {resource} );
248 
249  FetcherJob_Ptr job;
250  job.reset(new FetcherJob(resource));
251  job->flags |= FetcherJob:: AlwaysVerifyChecksum;
252  _resources.push_back(job);
253  }
254 
256  : _options(0)
257  {
258  }
259 
260  void Fetcher::Impl::setOptions( Fetcher::Options options )
261  { _options = options; }
262 
263  Fetcher::Options Fetcher::Impl::options() const
264  { return _options; }
265 
267  bool recursive,
268  const FileChecker &checker )
269  {
270  FetcherJob_Ptr job;
271  job.reset(new FetcherJob(resource));
272  if ( checker )
273  job->checkers.push_back(checker);
274  if ( recursive )
275  job->flags |= FetcherJob::Recursive;
276  job->flags |= FetcherJob::Directory;
277 
278  _resources.push_back(job);
279  }
280 
282  bool recursive,
283  const FileChecker &checker )
284  {
285  FetcherJob_Ptr job;
286  job.reset(new FetcherJob(resource));
287  if ( checker )
288  job->checkers.push_back(checker);
289  if ( recursive )
290  job->flags |= FetcherJob::Recursive;
291  job->flags |= FetcherJob::Directory;
292  job->flags |= FetcherJob::AlwaysVerifyChecksum;
293 
294  _resources.push_back(job);
295 
296  }
297 
298  void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
299  {
300  if ( _mediaSetAccess )
301  _mediaSetAccess->precacheFiles( {resource} );
302 
303  FetcherJob_Ptr job;
304  job.reset(new FetcherJob(resource));
305  if ( checker )
306  job->checkers.push_back(checker);
307  _resources.push_back(job);
308  }
309 
310  void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
311  {
312  MIL << "adding index " << resource << endl;
313  _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
314  }
315 
316 
318  {
319  _resources.clear();
320  _indexes.clear();
321  _checksums.clear();
322  _dircontent.clear();
323  }
324 
326  {
327  _mediaSetAccess = &media;
328  }
329 
330  void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
331  {
332  PathInfo info(cache_dir);
333  if ( info.isExist() )
334  {
335  if ( info.isDir() )
336  {
337  DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
338  _caches.insert(cache_dir);
339  }
340  else
341  {
342  // don't add bad cache directory, just log the error
343  ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
344  }
345  }
346  else
347  {
348  ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
349  }
350 
351  }
352 
353  Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r )
354  {
355  Pathname ret;
356  // No checksum - no match
357  if ( resource_r.checksum().empty() )
358  return ret;
359 
360  // first check in the destination directory
361  Pathname cacheLocation = destDir_r / resource_r.filename();
362  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
363  {
364  swap( ret, cacheLocation );
365  return ret;
366  }
367 
368  MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
369  for( const Pathname & cacheDir : _caches )
370  {
371  cacheLocation = cacheDir / resource_r.filename();
372  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
373  {
374  MIL << "file " << resource_r.filename() << " found in cache " << cacheDir << endl;
375  swap( ret, cacheLocation );
376  return ret;
377  }
378  }
379 
380  return ret;
381  }
382 
383  void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r )
384  {
385  try
386  {
387  MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl;
388 
389  for ( const FileChecker & chkfnc : checkers_r )
390  {
391  if ( chkfnc )
392  chkfnc( localfile_r );
393  else
394  ERR << "Invalid checker for '" << localfile_r << "'" << endl;
395  }
396 
397  }
398  catch ( const FileCheckException &e )
399  {
400  throw; // let known exceptions bypass silently
401  }
402  catch ( const Exception &e )
403  {
404  throw; // slet known exceptions bypass silently
405  }
406  catch (...)
407  {
408  ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString()));
409  }
410  }
411 
413  MediaSetAccess &media,
414  const OnMediaLocation &resource,
415  const Pathname &dest_dir )
416  {
417  auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
418  {
419  if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
420  return false;
421  // add the index of this directory
422  OnMediaLocation indexloc( resource );
423  indexloc.changeFilename( resource.filename() + index_r );
424  addIndex( indexloc );
425  // we need to read it now
426  downloadAndReadIndexList( media, dest_dir );
427  return true;
428  } );
429 
431  {
432  fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
433  }
435  {
436  fnc_addIfInContent( "content" );
437  }
438  }
439 
441  const OnMediaLocation &resource,
442  filesystem::DirContent &content )
443  {
444  if ( _dircontent.find(resource.filename().asString())
445  != _dircontent.end() )
446  {
447  filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
448 
449  std::copy(filled.begin(), filled.end(), std::back_inserter(content));
450  }
451  else
452  {
453  filesystem::DirContent tofill;
454  media.dirInfo( tofill,
455  resource.filename(),
456  false /* dots */,
457  resource.medianr());
458  std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
459  _dircontent[resource.filename().asString()] = tofill;
460  }
461  }
462 
464  const OnMediaLocation &resource,
465  const Pathname &dest_dir, FetcherJob::Flags flags )
466  {
467  // first get the content of the directory so we can add
468  // individual transfer jobs
469  MIL << "Adding directory " << resource.filename() << endl;
470  filesystem::DirContent content;
471  try {
472  getDirectoryContent(media, resource, content);
473  }
474  catch ( media::MediaFileNotFoundException & exception )
475  {
476  ZYPP_CAUGHT( exception );
477  WAR << "Skipping subtree hidden at " << resource.filename() << endl;
478  return;
479  }
480 
481  // this method test for the option flags so indexes are added
482  // only if the options are enabled
483  autoaddIndexes(content, media, resource, dest_dir);
484 
485  for ( filesystem::DirContent::const_iterator it = content.begin();
486  it != content.end();
487  ++it )
488  {
489  // skip CHECKSUMS* as they were already retrieved
490  if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
491  continue;
492 
493  Pathname filename = resource.filename() + it->name;
494 
495  switch ( it->type )
496  {
497  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
498  case filesystem::FT_FILE:
499  {
500  CheckSum chksm(resource.checksum());
501  if ( _checksums.find(filename.asString()) != _checksums.end() )
502  {
503  // the checksum can be replaced with the one in the index.
504  chksm = _checksums[filename.asString()];
505  //MIL << "resource " << filename << " has checksum in the index file." << endl;
506  }
507  else
508  WAR << "Resource " << filename << " has no checksum in the index either." << endl;
509 
510  if ( flags & FetcherJob::AlwaysVerifyChecksum )
511  enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
512  else
513  enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
514  break;
515  }
516  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
517  if ( flags & FetcherJob::Recursive )
518  addDirJobs(media, filename, dest_dir, flags);
519  break;
520  default:
521  // don't provide devices, sockets, etc.
522  break;
523  }
524  }
525  }
526 
527  void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r )
528  {
529  const OnMediaLocation & resource( jobp_r->location );
530 
531  try
532  {
533  scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard; // will take care provided files get released
534 
535  // get cached file (by checksum) or provide from media
536  Pathname tmpFile = locateInCache( resource, destDir_r );
537  if ( tmpFile.empty() )
538  {
539  MIL << "Not found in cache, retrieving..." << endl;
541  releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block
542  }
543 
544  // The final destination: locateInCache also checks destFullPath!
545  // If we find a cache match (by checksum) at destFullPath, take
546  // care it gets deleted, in case the validation fails.
547  ManagedFile destFullPath( destDir_r / resource.filename() );
548  if ( tmpFile == destFullPath )
549  destFullPath.setDispose( filesystem::unlink );
550 
551  // validate the file (throws if not valid)
552  validate( tmpFile, jobp_r->checkers );
553 
554  // move it to the final destination
555  if ( tmpFile == destFullPath )
556  destFullPath.resetDispose(); // keep it!
557  else
558  {
559  if ( assert_dir( destFullPath->dirname() ) != 0 )
560  ZYPP_THROW( Exception( "Can't create " + destFullPath->dirname().asString() ) );
561 
562  if ( filesystem::hardlinkCopy( tmpFile, destFullPath ) != 0 )
563  ZYPP_THROW( Exception( "Can't hardlink/copy " + tmpFile.asString() + " to " + destDir_r.asString() ) );
564  }
565  }
566  catch ( Exception & excpt )
567  {
568  if ( resource.optional() )
569  {
570  ZYPP_CAUGHT( excpt );
571  WAR << "optional resource " << resource << " could not be transferred." << endl;
572  return;
573  }
574  else
575  {
576  excpt.remember( "Can't provide " + resource.filename().asString() );
577  ZYPP_RETHROW( excpt );
578  }
579  }
580  }
581 
582  // helper class to consume a content file
584  {
586  {
587  setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
588  }
589 
590  void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
591  { _repoindex = data_r; }
592 
593  parser::susetags::RepoIndex_Ptr _repoindex;
594  };
595 
596  // generic function for reading indexes
597  void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
598  {
599  if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
600  readChecksumsIndex(index, basedir);
601  else if ( index.basename() == "content" )
602  readContentFileIndex(index, basedir);
603  else
604  WAR << index << ": index file format not known" << endl;
605  }
606 
607  // reads a content file index
608  void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
609  {
610  ContentReaderHelper reader;
611  reader.parse(index);
612  MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
613  for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
614  {
615  // content file entries don't start with /
616  _checksums[(basedir + it->first).asString()] = it->second;
617  }
618  }
619 
620  // reads a CHECKSUMS (old SHA1SUMS) file index
621  void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
622  {
623  std::ifstream in( index.c_str() );
624  if ( ! in.fail() )
625  {
626  std::string buffer;
627  while ( getline( in, buffer ) )
628  {
629 
630  if ( buffer[0] == '#' )
631  continue; // simple comment
632 
633  CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
634  if ( checksum.empty() )
635  continue; // empty line | unknown cheksum format
636 
637  if ( buffer.empty() )
638  {
639  WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
640  continue;
641  }
642 
643  _checksums[(basedir/buffer).asString()] = checksum;
644  }
645  }
646  else
647  ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
648  }
649 
650  void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
651  {
652  MIL << "downloading index " << resource << endl;
653  // create a new fetcher with a different state to transfer the
654  // file containing checksums and its signature
655  Fetcher fetcher;
656  // signature checker for index. We havent got the signature from
657  // the nextwork yet.
658  SignatureFileChecker sigchecker;
659 
660  // build the name of the index and the signature
661  const OnMediaLocation& idxloc(resource);
662  OnMediaLocation sigloc(resource);
663  OnMediaLocation keyloc(resource);
664 
665  // we should not fail the download if those don't exists
666  // the checking will warn later
667  sigloc.setOptional(true);
668  keyloc.setOptional(true);
669 
670  // calculate signature and key name
671  sigloc.changeFilename( sigloc.filename().extend(".asc") );
672  keyloc.changeFilename( keyloc.filename().extend(".key") );
673 
674  //assert_dir(dest_dir + idxloc.filename().dirname());
675 
676  // transfer the signature
677  fetcher.enqueue(sigloc);
678  fetcher.start( dest_dir, media );
679  // if we get the signature, update the checker
680  if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
681  sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
682 
683  fetcher.reset();
684 
685  // now the key
686  fetcher.enqueue(keyloc);
687  fetcher.start( dest_dir, media );
688  fetcher.reset();
689 
690  // try to import the key
691  if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
692  getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
693  else
694  WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
695 
696  // now the index itself
697  fetcher.enqueue( idxloc, FileChecker(sigchecker) );
698  fetcher.start( dest_dir, media );
699  fetcher.reset();
700  }
701 
702  // this method takes all the user pointed indexes, gets them and also tries to
703  // download their signature, and verify them. After that, it parses each one
704  // to fill the checksum cache.
706  {
707  // if there is no indexes, then just return to avoid
708  // the directory listing
709  if ( _indexes.empty() )
710  {
711  MIL << "No indexes to read." << endl;
712  return;
713  }
714 
715  for_( it_idx, _indexes.begin(), _indexes.end() )
716  {
717  if ( (*it_idx)->read )
718  {
719  DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
720  }
721  else
722  {
723  // base::LogControl::TmpLineWriter shutUp;
724  downloadIndex( media, (*it_idx)->location, dest_dir );
725  // now we have the indexes in dest_dir
726  readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
727  // Take care we don't process it again
728  MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
729  (*it_idx)->read = true;
730  }
731  }
732  MIL << "done reading indexes" << endl;
733  }
734 
735  void Fetcher::Impl::start( const Pathname &dest_dir,
736  const ProgressData::ReceiverFnc & progress )
737  {
738  if ( !_mediaSetAccess )
739  ZYPP_THROW( zypp::Exception("Called Fetcher::start without setting MediaSetAccess before.") );
740  start( dest_dir, *_mediaSetAccess, progress );
741  }
742 
743  // start processing all fetcher jobs.
744  // it processes any user pointed index first
745  void Fetcher::Impl::start( const Pathname &dest_dir,
746  MediaSetAccess &media,
747  const ProgressData::ReceiverFnc & progress_receiver )
748  {
749  _mediaSetAccess = nullptr; //reset the internally stored MediaSetAccess
750 
751  ProgressData progress(_resources.size());
752  progress.sendTo(progress_receiver);
753 
754  downloadAndReadIndexList(media, dest_dir);
755 
756  for ( const FetcherJob_Ptr & jobp : _resources )
757  {
758  if ( jobp->flags & FetcherJob::Directory )
759  {
760  const OnMediaLocation location(jobp->location);
761  addDirJobs(media, location, dest_dir, jobp->flags);
762  continue;
763  }
764 
765  // may be this code can be factored out
766  // together with the autodiscovery of indexes
767  // of addDirJobs
768  if ( ( _options & AutoAddChecksumsIndexes ) ||
770  {
771  // if auto indexing is enabled, then we need to read the
772  // index for each file. We look only in the directory
773  // where the file is. this is expensive of course.
774  filesystem::DirContent content;
775  getDirectoryContent(media, jobp->location.filename().dirname(), content);
776  // this method test for the option flags so indexes are added
777  // only if the options are enabled
778  MIL << "Autodiscovering signed indexes on '"
779  << jobp->location.filename().dirname() << "' for '"
780  << jobp->location.filename() << "'" << endl;
781 
782  autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
783 
784  // also look in the root of the media
785  content.clear();
786  getDirectoryContent(media, Pathname("/"), content);
787  // this method test for the option flags so indexes are added
788  // only if the options are enabled
789  MIL << "Autodiscovering signed indexes on '"
790  << "/" << "' for '"
791  << jobp->location.filename() << "'" << endl;
792 
793  autoaddIndexes(content, media, Pathname("/"), dest_dir);
794  }
795 
796  // if the checksum is empty, but the checksum is in one of the
797  // indexes checksum, then add a checker
798  if ( jobp->location.checksum().empty() )
799  {
800  if ( _checksums.find(jobp->location.filename().asString())
801  != _checksums.end() )
802  {
803  CheckSum chksm = _checksums[jobp->location.filename().asString()];
804  ChecksumFileChecker digest_check(chksm);
805  jobp->checkers.push_back(digest_check);
806  }
807  else
808  {
809  // if the index checksum is empty too, we only add the checker
810  // if the AlwaysVerifyChecksum option is set on
811  if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum )
812  {
813  // add the checker with the empty checksum
814  ChecksumFileChecker digest_check(jobp->location.checksum());
815  jobp->checkers.push_back(digest_check);
816  }
817  }
818  }
819  else
820  {
821  // checksum is not empty, so add a checksum checker
822  ChecksumFileChecker digest_check(jobp->location.checksum());
823  jobp->checkers.push_back(digest_check);
824  }
825 
826  // Provide and validate the file. If the file was not transferred
827  // and no exception was thrown, it was an optional file.
828  provideToDest( media, dest_dir, jobp );
829 
830  if ( ! progress.incr() )
831  ZYPP_THROW(AbortRequestException());
832  } // for each job
833  }
834 
836  inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
837  {
838  for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
839  {
840  str << *it_res;
841  }
842  return str;
843  }
844 
846  : _pimpl( new Impl() )
847  {}
848 
850  {}
851 
852  void Fetcher::setOptions( Fetcher::Options options )
853  {
855  }
856 
857  Fetcher::Options Fetcher::options() const
858  {
859  return _pimpl->options();
860  }
861 
862  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
863  {
864  enqueueDigested( OnMediaLocation(resource).setDeltafile(deltafile), checker );
865  }
866 
867  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
868  {
869  _pimpl->enqueueDigested( resource, checker );
870  }
871 
872  void Fetcher::enqueueDir( const OnMediaLocation &resource,
873  bool recursive,
874  const FileChecker &checker )
875  {
876  _pimpl->enqueueDir(resource, recursive, checker);
877  }
878 
880  bool recursive,
881  const FileChecker &checker )
882  {
883  _pimpl->enqueueDigestedDir(resource, recursive, checker);
884  }
885 
886 
887  void Fetcher::addIndex( const OnMediaLocation &resource )
888  {
889  _pimpl->addIndex(resource);
890  }
891 
892 
893  void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
894  {
895  _pimpl->enqueue(resource, checker);
896  }
897 
898 
899  void Fetcher::addCachePath( const Pathname &cache_dir )
900  {
901  _pimpl->addCachePath(cache_dir);
902  }
903 
905  {
906  _pimpl->reset();
907  }
908 
910  {
911  _pimpl->setMediaSetAccess( media );
912  }
913 
915  {
916  _pimpl->start( dest_dir, progress );
917  }
918 
919  void Fetcher::start( const Pathname &dest_dir,
920  MediaSetAccess &media,
921  const ProgressData::ReceiverFnc & progress_receiver )
922  {
923  _pimpl->start(dest_dir, media, progress_receiver);
924  }
925 
926  std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
927  {
928  return str << *obj._pimpl;
929  }
930 
932 } // namespace zypp
934 
std::string asString(const Patch::Category &obj)
Definition: Patch.cc:122
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Definition: Fetcher.cc:590
#define MIL
Definition: Logger.h:100
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Definition: Fetcher.cc:608
Pathname deltafile
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:324
void setOptions(Options options)
Set the Fetcher options.
Definition: Fetcher.cc:852
Listentry returned by readdir.
Definition: PathInfo.h:509
Pathname provideFile(const OnMediaLocation &resource, ProvideFileOptions options=PROVIDE_DEFAULT)
Provides a file from a media location.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:244
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
Definition: progressdata.h:229
void precacheFiles(const std::vector< OnMediaLocation > &files)
Tries to fetch the given files and precaches them.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:424
Describes a resource file located on a medium.
Checks for the validity of a signature.
Definition: FileChecker.h:70
FetcherJob & operator=(const FetcherJob &)=default
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Definition: Fetcher.cc:887
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Fetcher.cc:593
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: progressdata.h:140
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:281
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory)...
Definition: Fetcher.cc:353
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:175
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:27
const char * c_str() const
String representation.
Definition: Pathname.h:112
String related utilities and Regular expression matching.
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
Definition: Fetcher.cc:527
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:298
std::string getline(std::istream &str)
Read one line from stream.
Definition: IOStream.cc:33
std::ostream & operator<<(std::ostream &str, const SerialNumber &obj)
Definition: SerialNumber.cc:52
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
Definition: Fetcher.cc:899
If a content file is found, it is downloaded and read.
Definition: Fetcher.h:123
const CheckSum & checksum() const
The checksum of the resource on the server.
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:130
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Definition: Fetcher.cc:879
void addCachePath(const Pathname &cache_dir)
Definition: Fetcher.cc:330
OnMediaLocation location
Index localtion.
Definition: Fetcher.cc:49
zypp::RepoManager::RefreshServiceOptions _options
Definition: serviceswf.cc:724
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:526
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:888
std::list< FileChecker > checkers
Definition: Fetcher.cc:109
#define ERR
Definition: Logger.h:102
shared_ptr< FetcherJob > FetcherJob_Ptr
Definition: Fetcher.cc:114
unsigned medianr() const
The media number the resource is located on.
static shared_ptr< Impl > nullimpl()
Offer default Impl.
Definition: Fetcher.cc:160
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
Definition: Fetcher.h:346
void remember(const Exception &old_r)
Store an other Exception as history.
Definition: Exception.cc:141
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Definition: Fetcher.cc:909
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:914
Impl & operator=(const Impl &)=delete
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Definition: Fetcher.cc:59
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
Definition: Fetcher.cc:231
bool empty() const
Test for an empty path.
Definition: Pathname.h:116
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:444
FetcherJob(const FetcherJob &)=default
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Definition: String.cc:264
Fetcher::Options _options
Definition: Fetcher.cc:240
const std::string & asString() const
String representation.
Definition: Pathname.h:93
void setMediaSetAccess(MediaSetAccess &media)
Definition: Fetcher.cc:325
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:286
void dirInfo(filesystem::DirContent &retlist, const Pathname &dirname, bool dots=true, unsigned media_nr=1)
Fills retlist with directory information.
void reset()
Reset the transfer (jobs) list.
Definition: Fetcher.cc:904
MediaSetAccess * _mediaSetAccess
Definition: Fetcher.cc:238
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:126
#define WAR
Definition: Logger.h:101
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Definition: Fetcher.cc:597
Parse repoindex part from a content file.
FetcherJob(const OnMediaLocation &loc)
Definition: Fetcher.cc:98
std::set ordering (less semantic)
Definition: Fetcher.cc:57
Fetcher::Options options() const
Definition: Fetcher.cc:263
Maintain [min,max] and counter (value) for progress counting.
Definition: progressdata.h:131
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:266
DefaultIntegral< bool, false > read
Whether we read this index.
Definition: Fetcher.cc:51
std::map< std::string, filesystem::DirContent > _dircontent
Definition: Fetcher.cc:236
OnMediaLocation location
Definition: Fetcher.cc:107
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:867
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:650
const Pathname & filename() const
The path to the resource on the medium.
std::list< FetcherJob_Ptr > _resources
Definition: Fetcher.cc:230
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress)
Definition: Fetcher.cc:735
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:705
Fetcher()
Default ctor.
Definition: Fetcher.cc:845
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
Definition: Fetcher.cc:872
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Definition: Fetcher.cc:383
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
Definition: IOStream.h:51
ZYPP_DECLARE_FLAGS(Flags, Flag)
shared_ptr< FetcherIndex > FetcherIndex_Ptr
Definition: Fetcher.cc:54
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
Definition: Fetcher.cc:440
If a CHECKSUMS file is found, it is downloaded and read.
Definition: Fetcher.h:128
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Definition: AutoDispose.h:167
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:440
Class to encapsulate the OnMediaLocation object and the FileChecker together.
Definition: Fetcher.cc:79
void setOptions(Fetcher::Options options)
Definition: Fetcher.cc:260
zypp::parser::susetags::RepoIndex_Ptr _repoindex
Definition: susetags.cc:324
FetcherIndex(const OnMediaLocation &loc)
Definition: Fetcher.cc:45
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
Definition: Fetcher.cc:621
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:705
Fetcher implementation.
Definition: Fetcher.cc:126
Base class for Exception.
Definition: Exception.h:146
Built in file checkers.
Definition: FileChecker.h:47
bool empty() const
Definition: CheckSum.cc:173
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Definition: PathInfo.cc:1056
std::map< std::string, CheckSum > _checksums
Definition: Fetcher.cc:234
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
Definition: Fetcher.cc:43
std::set< Pathname > _caches
Definition: Fetcher.cc:232
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:29
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Definition: Fetcher.cc:836
Options options() const
Get current options.
Definition: Fetcher.cc:857
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:893
virtual ~Fetcher()
Dtor.
Definition: Fetcher.cc:849
Easy-to use interface to the ZYPP dependency resolver.
Definition: Application.cc:19
OnMediaLocation & setOptional(bool val)
Set whether the resource is optional.
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Definition: Fetcher.cc:412
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1027
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
bool optional() const
Whether this is an optional resource.
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Definition: Fetcher.h:105
void addIndex(const OnMediaLocation &resource)
Definition: Fetcher.cc:310
#define DBG
Definition: Logger.h:99
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
Definition: PathInfo.cc:1068
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Definition: Fetcher.cc:463
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Impl * clone() const
clone for RWCOW_pointer
Definition: Fetcher.cc:227
The user is not asked anything, and the error exception is just propagated.