libzypp 17.35.1
Fetcher.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
12#include <iostream>
13#include <fstream>
14#include <list>
15#include <map>
16
17#include <zypp/base/Easy.h>
19#include <zypp/base/LogTools.h>
20#include <zypp/base/PtrTypes.h>
21#include <zypp-core/base/DefaultIntegral>
22#include <zypp/base/String.h>
23#include <zypp-media/MediaException>
24#include <zypp/Fetcher.h>
25#include <zypp/ZYppFactory.h>
26#include <zypp/CheckSum.h>
27#include <zypp-core/base/UserRequestException>
30
31#undef ZYPP_BASE_LOGGER_LOGGROUP
32#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::fetcher"
33
35namespace zypp
36{
37
53
54 using FetcherIndex_Ptr = shared_ptr<FetcherIndex>;
55
58 {
59 bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs ) const
60 {
61 if ( lhs == rhs )
62 return false; // incl. NULL == NULL
63 if ( ! lhs )
64 return true; // NULL < nonNULL
65 if ( ! rhs )
66 return false; // nonNULL > NULL
67 // both nonNULL ==> compare medianr and path
68 if ( lhs->location.medianr() == rhs->location.medianr() )
69 return lhs->location.filename() < rhs->location.filename();
70 //else
71 return lhs->location.medianr() < rhs->location.medianr();
72 }
73 };
74
80 {
81 enum Flag
82 {
83 None = 0x0000,
84 Directory = 0x0001,
85 Recursive = 0x0002,
87 // check checksums even if there is no such
88 // checksum (warns of no checksum)
90 };
92
93 FetcherJob(const FetcherJob &) = default;
94 FetcherJob(FetcherJob &&) = default;
95 FetcherJob &operator=(const FetcherJob &) = default;
97
99 //MIL << location << endl;
100 }
101
103 {
104 //MIL << location << " | * " << checkers.size() << endl;
105 }
106
108 //CompositeFileChecker checkers;
109 std::list<FileChecker> checkers;
110 Flags flags;
111 };
112
114 using FetcherJob_Ptr = shared_ptr<FetcherJob>;
115
116 std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
117 {
118 return str << obj->location;
119 }
120
122 //
123 // CLASS NAME : Fetcher::Impl
124 //
127 {
128 friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
129
130 public:
131 Impl();
132
133 Impl(const Impl &) = default;
134 Impl(Impl &&) = delete;
135 Impl &operator=(const Impl &) = delete;
136 Impl &operator=(Impl &&) = delete;
137
138 ~Impl() {}
139
140 void setOptions( Fetcher::Options options );
141 Fetcher::Options options() const;
142
143 void addIndex( const OnMediaLocation &resource );
144
145 void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
146 void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
147
148 void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
149 void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
150 void addCachePath( const Pathname &cache_dir );
151 void reset();
152 void setMediaSetAccess ( MediaSetAccess &media );
153 void start( const Pathname &dest_dir,
154 const ProgressData::ReceiverFnc &progress );
155 void start( const Pathname &dest_dir,
156 MediaSetAccess &media,
157 const ProgressData::ReceiverFnc & progress_receiver );
158
160 static shared_ptr<Impl> nullimpl()
161 {
162 static shared_ptr<Impl> _nullimpl( new Impl );
163 return _nullimpl;
164 }
165 private:
169 void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
170
174 void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
175
183 void readIndex( const Pathname &index, const Pathname &basedir );
184
186 void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
187
189 void readContentFileIndex( const Pathname &index, const Pathname &basedir );
190
192 void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
193
199 Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r );
204 void validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r );
205
209 void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
210 const Pathname &dest_dir, FetcherJob::Flags flags );
211
215 void autoaddIndexes( const filesystem::DirContent &content,
216 MediaSetAccess &media,
217 const OnMediaLocation &resource,
218 const Pathname &dest_dir );
222 void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r );
223
224 private:
225 friend Impl * rwcowClone<Impl>( const Impl * rhs );
227 Impl * clone() const
228 { return new Impl( *this ); }
229
230 std::list<FetcherJob_Ptr> _resources;
231 std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
232 std::set<Pathname> _caches;
233 // checksums read from the indexes
234 std::map<std::string, CheckSum> _checksums;
235 // cache of dir contents
236 std::map<std::string, filesystem::DirContent> _dircontent;
237
239
240 Fetcher::Options _options;
241 };
243
245 {
246 if ( _mediaSetAccess )
247 _mediaSetAccess->precacheFiles( {resource} );
248
249 FetcherJob_Ptr job;
250 job.reset(new FetcherJob(resource));
252 _resources.push_back(job);
253 }
254
256 : _options(0)
257 {
258 }
259
260 void Fetcher::Impl::setOptions( Fetcher::Options options )
261 { _options = options; }
262
263 Fetcher::Options Fetcher::Impl::options() const
264 { return _options; }
265
267 bool recursive,
268 const FileChecker &checker )
269 {
270 FetcherJob_Ptr job;
271 job.reset(new FetcherJob(resource));
272 if ( checker )
273 job->checkers.push_back(checker);
274 if ( recursive )
275 job->flags |= FetcherJob::Recursive;
276 job->flags |= FetcherJob::Directory;
277
278 _resources.push_back(job);
279 }
280
282 bool recursive,
283 const FileChecker &checker )
284 {
285 FetcherJob_Ptr job;
286 job.reset(new FetcherJob(resource));
287 if ( checker )
288 job->checkers.push_back(checker);
289 if ( recursive )
290 job->flags |= FetcherJob::Recursive;
291 job->flags |= FetcherJob::Directory;
293
294 _resources.push_back(job);
295
296 }
297
298 void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
299 {
300 if ( _mediaSetAccess )
301 _mediaSetAccess->precacheFiles( {resource} );
302
303 FetcherJob_Ptr job;
304 job.reset(new FetcherJob(resource));
305 if ( checker )
306 job->checkers.push_back(checker);
307 _resources.push_back(job);
308 }
309
311 {
312 MIL << "adding index " << resource << endl;
313 _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
314 }
315
316
318 {
319 _resources.clear();
320 _indexes.clear();
321 _checksums.clear();
322 _dircontent.clear();
323 }
324
326 {
327 _mediaSetAccess = &media;
328 }
329
330 void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
331 {
332 PathInfo info(cache_dir);
333 if ( info.isExist() )
334 {
335 if ( info.isDir() )
336 {
337 DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
338 _caches.insert(cache_dir);
339 }
340 else
341 {
342 // don't add bad cache directory, just log the error
343 ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
344 }
345 }
346 else
347 {
348 ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
349 }
350
351 }
352
353 Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r )
354 {
355 Pathname ret;
356 // No checksum - no match
357 if ( resource_r.checksum().empty() )
358 return ret;
359
360 // first check in the destination directory
361 Pathname cacheLocation = destDir_r / resource_r.filename();
362 if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
363 {
364 swap( ret, cacheLocation );
365 return ret;
366 }
367
368 MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
369 for( const Pathname & cacheDir : _caches )
370 {
371 cacheLocation = cacheDir / resource_r.filename();
372 if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
373 {
374 MIL << "file " << resource_r.filename() << " found in cache " << cacheDir << endl;
375 swap( ret, cacheLocation );
376 return ret;
377 }
378 }
379
380 return ret;
381 }
382
383 void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r )
384 {
385 try
386 {
387 MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl;
388
389 for ( const FileChecker & chkfnc : checkers_r )
390 {
391 if ( chkfnc )
392 chkfnc( localfile_r );
393 else
394 ERR << "Invalid checker for '" << localfile_r << "'" << endl;
395 }
396
397 }
398 catch ( const FileCheckException &e )
399 {
400 throw; // let known exceptions bypass silently
401 }
402 catch ( const Exception &e )
403 {
404 throw; // slet known exceptions bypass silently
405 }
406 catch (...)
407 {
408 ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString()));
409 }
410 }
411
413 MediaSetAccess &media,
414 const OnMediaLocation &resource,
415 const Pathname &dest_dir )
416 {
417 auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
418 {
419 if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
420 return false;
421 // add the index of this directory
422 OnMediaLocation indexloc( resource );
423 indexloc.changeFilename( resource.filename() + index_r );
424 addIndex( indexloc );
425 // we need to read it now
426 downloadAndReadIndexList( media, dest_dir );
427 return true;
428 } );
429
431 {
432 fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
433 }
435 {
436 fnc_addIfInContent( "content" );
437 }
438 }
439
441 const OnMediaLocation &resource,
442 filesystem::DirContent &content )
443 {
444 if ( _dircontent.find(resource.filename().asString())
445 != _dircontent.end() )
446 {
447 filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
448
449 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
450 }
451 else
452 {
454 media.dirInfo( tofill,
455 resource.filename(),
456 false /* dots */,
457 resource.medianr());
458 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
459 _dircontent[resource.filename().asString()] = tofill;
460 }
461 }
462
464 const OnMediaLocation &resource,
465 const Pathname &dest_dir, FetcherJob::Flags flags )
466 {
467 // first get the content of the directory so we can add
468 // individual transfer jobs
469 MIL << "Adding directory " << resource.filename() << endl;
471 try {
472 getDirectoryContent(media, resource, content);
473 }
474 catch ( media::MediaFileNotFoundException & exception )
475 {
476 ZYPP_CAUGHT( exception );
477 WAR << "Skipping subtree hidden at " << resource.filename() << endl;
478 return;
479 }
480
481 // this method test for the option flags so indexes are added
482 // only if the options are enabled
483 autoaddIndexes(content, media, resource, dest_dir);
484
485 for ( filesystem::DirContent::const_iterator it = content.begin();
486 it != content.end();
487 ++it )
488 {
489 // skip CHECKSUMS* as they were already retrieved
490 if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
491 continue;
492
493 Pathname filename = resource.filename() + it->name;
494
495 switch ( it->type )
496 {
497 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
499 {
500 CheckSum chksm(resource.checksum());
501 if ( _checksums.find(filename.asString()) != _checksums.end() )
502 {
503 // the checksum can be replaced with the one in the index.
504 chksm = _checksums[filename.asString()];
505 //MIL << "resource " << filename << " has checksum in the index file." << endl;
506 }
507 else
508 WAR << "Resource " << filename << " has no checksum in the index either." << endl;
509
511 enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
512 else
513 enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
514 break;
515 }
516 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
517 if ( flags & FetcherJob::Recursive )
518 addDirJobs(media, filename, dest_dir, flags);
519 break;
520 default:
521 // don't provide devices, sockets, etc.
522 break;
523 }
524 }
525 }
526
527 void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r )
528 {
529 const OnMediaLocation & resource( jobp_r->location );
530
531 try
532 {
533 scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard; // will take care provided files get released
534
535 // get cached file (by checksum) or provide from media
536 Pathname tmpFile = locateInCache( resource, destDir_r );
537 if ( tmpFile.empty() )
538 {
539 MIL << "Not found in cache, retrieving..." << endl;
541 releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block
542 }
543
544 // The final destination: locateInCache also checks destFullPath!
545 // If we find a cache match (by checksum) at destFullPath, take
546 // care it gets deleted, in case the validation fails.
547 ManagedFile destFullPath( destDir_r / resource.filename() );
548 if ( tmpFile == destFullPath )
549 destFullPath.setDispose( filesystem::unlink );
550
551 // validate the file (throws if not valid)
552 validate( tmpFile, jobp_r->checkers );
553
554 // move it to the final destination
555 if ( tmpFile == destFullPath )
556 destFullPath.resetDispose(); // keep it!
557 else
558 {
559 if ( assert_dir( destFullPath->dirname() ) != 0 )
560 ZYPP_THROW( Exception( "Can't create " + destFullPath->dirname().asString() ) );
561
562 if ( filesystem::hardlinkCopy( tmpFile, destFullPath ) != 0 )
563 ZYPP_THROW( Exception( "Can't hardlink/copy " + tmpFile.asString() + " to " + destDir_r.asString() ) );
564 }
565 }
566 catch ( Exception & excpt )
567 {
568 if ( resource.optional() )
569 {
570 ZYPP_CAUGHT( excpt );
571 WAR << "optional resource " << resource << " could not be transferred." << endl;
572 return;
573 }
574 else
575 {
576 excpt.remember( "Can't provide " + resource.filename().asString() );
577 ZYPP_RETHROW( excpt );
578 }
579 }
580 }
581
582 // helper class to consume a content file
584 {
586 {
587 setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
588 }
589
590 void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
591 { _repoindex = data_r; }
592
593 parser::susetags::RepoIndex_Ptr _repoindex;
594 };
595
596 // generic function for reading indexes
597 void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
598 {
599 if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
600 readChecksumsIndex(index, basedir);
601 else if ( index.basename() == "content" )
602 readContentFileIndex(index, basedir);
603 else
604 WAR << index << ": index file format not known" << endl;
605 }
606
607 // reads a content file index
608 void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
609 {
610 ContentReaderHelper reader;
611 reader.parse(index);
612 MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
613 for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
614 {
615 // content file entries don't start with /
616 _checksums[(basedir + it->first).asString()] = it->second;
617 }
618 }
619
620 // reads a CHECKSUMS (old SHA1SUMS) file index
621 void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
622 {
623 std::ifstream in( index.c_str() );
624 if ( ! in.fail() )
625 {
626 std::string buffer;
627 while ( getline( in, buffer ) )
628 {
629
630 if ( buffer[0] == '#' )
631 continue; // simple comment
632
633 CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
634 if ( checksum.empty() )
635 continue; // empty line | unknown cheksum format
636
637 if ( buffer.empty() )
638 {
639 WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
640 continue;
641 }
642
643 _checksums[(basedir/buffer).asString()] = checksum;
644 }
645 }
646 else
647 ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
648 }
649
650 void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
651 {
652 MIL << "downloading index " << resource << endl;
653 // create a new fetcher with a different state to transfer the
654 // file containing checksums and its signature
655 Fetcher fetcher;
656 // signature checker for index. We havent got the signature from
657 // the nextwork yet.
658 SignatureFileChecker sigchecker;
659
660 // build the name of the index and the signature
661 const OnMediaLocation& idxloc(resource);
662 OnMediaLocation sigloc(resource);
663 OnMediaLocation keyloc(resource);
664
665 // we should not fail the download if those don't exists
666 // the checking will warn later
667 sigloc.setOptional(true);
668 keyloc.setOptional(true);
669
670 // calculate signature and key name
671 sigloc.changeFilename( sigloc.filename().extend(".asc") );
672 keyloc.changeFilename( keyloc.filename().extend(".key") );
673
674 //assert_dir(dest_dir + idxloc.filename().dirname());
675
676 // transfer the signature
677 fetcher.enqueue(sigloc);
678 fetcher.start( dest_dir, media );
679 // if we get the signature, update the checker
680 if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
681 sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
682
683 fetcher.reset();
684
685 // now the key
686 fetcher.enqueue(keyloc);
687 fetcher.start( dest_dir, media );
688 fetcher.reset();
689
690 // try to import the key
691 if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
692 getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
693 else
694 WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
695
696 // now the index itself
697 fetcher.enqueue( idxloc, FileChecker(sigchecker) );
698 fetcher.start( dest_dir, media );
699 fetcher.reset();
700 }
701
702 // this method takes all the user pointed indexes, gets them and also tries to
703 // download their signature, and verify them. After that, it parses each one
704 // to fill the checksum cache.
706 {
707 // if there is no indexes, then just return to avoid
708 // the directory listing
709 if ( _indexes.empty() )
710 {
711 MIL << "No indexes to read." << endl;
712 return;
713 }
714
715 for_( it_idx, _indexes.begin(), _indexes.end() )
716 {
717 if ( (*it_idx)->read )
718 {
719 DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
720 }
721 else
722 {
723 // base::LogControl::TmpLineWriter shutUp;
724 downloadIndex( media, (*it_idx)->location, dest_dir );
725 // now we have the indexes in dest_dir
726 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
727 // Take care we don't process it again
728 MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
729 (*it_idx)->read = true;
730 }
731 }
732 MIL << "done reading indexes" << endl;
733 }
734
735 void Fetcher::Impl::start( const Pathname &dest_dir,
736 const ProgressData::ReceiverFnc & progress )
737 {
738 if ( !_mediaSetAccess )
739 ZYPP_THROW( zypp::Exception("Called Fetcher::start without setting MediaSetAccess before.") );
740 start( dest_dir, *_mediaSetAccess, progress );
741 }
742
743 // start processing all fetcher jobs.
744 // it processes any user pointed index first
745 void Fetcher::Impl::start( const Pathname &dest_dir,
746 MediaSetAccess &media,
747 const ProgressData::ReceiverFnc & progress_receiver )
748 {
749 _mediaSetAccess = nullptr; //reset the internally stored MediaSetAccess
750
751 ProgressData progress(_resources.size());
752 progress.sendTo(progress_receiver);
753
754 downloadAndReadIndexList(media, dest_dir);
755
756 for ( const FetcherJob_Ptr & jobp : _resources )
757 {
758 if ( jobp->flags & FetcherJob::Directory )
759 {
760 const OnMediaLocation location(jobp->location);
761 addDirJobs(media, location, dest_dir, jobp->flags);
762 continue;
763 }
764
765 // may be this code can be factored out
766 // together with the autodiscovery of indexes
767 // of addDirJobs
770 {
771 // if auto indexing is enabled, then we need to read the
772 // index for each file. We look only in the directory
773 // where the file is. this is expensive of course.
775 getDirectoryContent(media, jobp->location.filename().dirname(), content);
776 // this method test for the option flags so indexes are added
777 // only if the options are enabled
778 MIL << "Autodiscovering signed indexes on '"
779 << jobp->location.filename().dirname() << "' for '"
780 << jobp->location.filename() << "'" << endl;
781
782 autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
783
784 // also look in the root of the media
785 content.clear();
786 getDirectoryContent(media, Pathname("/"), content);
787 // this method test for the option flags so indexes are added
788 // only if the options are enabled
789 MIL << "Autodiscovering signed indexes on '"
790 << "/" << "' for '"
791 << jobp->location.filename() << "'" << endl;
792
793 autoaddIndexes(content, media, Pathname("/"), dest_dir);
794 }
795
796 // if the checksum is empty, but the checksum is in one of the
797 // indexes checksum, then add a checker
798 if ( jobp->location.checksum().empty() )
799 {
800 if ( _checksums.find(jobp->location.filename().asString())
801 != _checksums.end() )
802 {
803 CheckSum chksm = _checksums[jobp->location.filename().asString()];
804 ChecksumFileChecker digest_check(chksm);
805 jobp->checkers.push_back(digest_check);
806 }
807 else
808 {
809 // if the index checksum is empty too, we only add the checker
810 // if the AlwaysVerifyChecksum option is set on
811 if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum )
812 {
813 // add the checker with the empty checksum
814 ChecksumFileChecker digest_check(jobp->location.checksum());
815 jobp->checkers.push_back(digest_check);
816 }
817 }
818 }
819 else
820 {
821 // checksum is not empty, so add a checksum checker
822 ChecksumFileChecker digest_check(jobp->location.checksum());
823 jobp->checkers.push_back(digest_check);
824 }
825
826 // Provide and validate the file. If the file was not transferred
827 // and no exception was thrown, it was an optional file.
828 provideToDest( media, dest_dir, jobp );
829
830 if ( ! progress.incr() )
831 ZYPP_THROW(AbortRequestException());
832 } // for each job
833 }
834
836 inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
837 {
838 for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
839 {
840 str << *it_res;
841 }
842 return str;
843 }
844
846 : _pimpl( new Impl() )
847 {}
848
851
852 void Fetcher::setOptions( Fetcher::Options options )
853 {
855 }
856
857 Fetcher::Options Fetcher::options() const
858 {
859 return _pimpl->options();
860 }
861
862 void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
863 {
864 enqueueDigested( OnMediaLocation(resource).setDeltafile(deltafile), checker );
865 }
866
867 void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
868 {
869 _pimpl->enqueueDigested( resource, checker );
870 }
871
873 bool recursive,
874 const FileChecker &checker )
875 {
876 _pimpl->enqueueDir(resource, recursive, checker);
877 }
878
880 bool recursive,
881 const FileChecker &checker )
882 {
883 _pimpl->enqueueDigestedDir(resource, recursive, checker);
884 }
885
886
887 void Fetcher::addIndex( const OnMediaLocation &resource )
888 {
889 _pimpl->addIndex(resource);
890 }
891
892
893 void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
894 {
895 _pimpl->enqueue(resource, checker);
896 }
897
898
899 void Fetcher::addCachePath( const Pathname &cache_dir )
900 {
901 _pimpl->addCachePath(cache_dir);
902 }
903
905 {
906 _pimpl->reset();
907 }
908
910 {
911 _pimpl->setMediaSetAccess( media );
912 }
913
915 {
916 _pimpl->start( dest_dir, progress );
917 }
918
919 void Fetcher::start( const Pathname &dest_dir,
920 MediaSetAccess &media,
921 const ProgressData::ReceiverFnc & progress_receiver )
922 {
923 _pimpl->start(dest_dir, media, progress_receiver);
924 }
925
926 std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
927 {
928 return str << *obj._pimpl;
929 }
930
932} // namespace zypp
934
Pathname deltafile
void resetDispose()
Set no dispose function.
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
bool empty() const
Definition CheckSum.cc:173
Built in file checkers.
Definition FileChecker.h:48
Integral type with defined initial value when default constructed.
Base class for Exception.
Definition Exception.h:147
void remember(const Exception &old_r)
Store an other Exception as history.
Definition Exception.cc:141
Fetcher implementation.
Definition Fetcher.cc:127
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition Fetcher.cc:244
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Definition Fetcher.cc:412
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Definition Fetcher.cc:650
Impl & operator=(const Impl &)=delete
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Definition Fetcher.cc:608
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Definition Fetcher.cc:597
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
Definition Fetcher.cc:231
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
std::map< std::string, CheckSum > _checksums
Definition Fetcher.cc:234
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Definition Fetcher.cc:705
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Definition Fetcher.cc:463
Impl * clone() const
clone for RWCOW_pointer
Definition Fetcher.cc:227
void addCachePath(const Pathname &cache_dir)
Definition Fetcher.cc:330
std::list< FetcherJob_Ptr > _resources
Definition Fetcher.cc:230
friend Impl * rwcowClone(const Impl *rhs)
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
Definition Fetcher.cc:621
Impl(Impl &&)=delete
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition Fetcher.cc:266
Impl & operator=(Impl &&)=delete
void addIndex(const OnMediaLocation &resource)
Definition Fetcher.cc:310
std::map< std::string, filesystem::DirContent > _dircontent
Definition Fetcher.cc:236
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
Definition Fetcher.cc:527
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition Fetcher.cc:298
MediaSetAccess * _mediaSetAccess
Definition Fetcher.cc:238
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory).
Definition Fetcher.cc:353
std::set< Pathname > _caches
Definition Fetcher.cc:232
Impl(const Impl &)=default
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition Fetcher.cc:281
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Definition Fetcher.cc:383
Fetcher::Options options() const
Definition Fetcher.cc:263
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
Definition Fetcher.cc:440
static shared_ptr< Impl > nullimpl()
Offer default Impl.
Definition Fetcher.cc:160
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress)
Definition Fetcher.cc:735
void setOptions(Fetcher::Options options)
Definition Fetcher.cc:260
Fetcher::Options _options
Definition Fetcher.cc:240
void setMediaSetAccess(MediaSetAccess &media)
Definition Fetcher.cc:325
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Definition Fetcher.h:104
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition Fetcher.cc:914
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
Definition Fetcher.h:344
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Definition Fetcher.cc:887
Options options() const
Get current options.
Definition Fetcher.cc:857
void reset()
Reset the transfer (jobs) list.
Definition Fetcher.cc:904
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition Fetcher.cc:893
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
Definition Fetcher.cc:872
void setOptions(Options options)
Set the Fetcher options.
Definition Fetcher.cc:852
@ AutoAddChecksumsIndexes
If a CHECKSUMS file is found, it is downloaded and read.
Definition Fetcher.h:126
@ AutoAddContentFileIndexes
If a content file is found, it is downloaded and read.
Definition Fetcher.h:121
Fetcher()
Default ctor.
Definition Fetcher.cc:845
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Definition Fetcher.cc:879
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition Fetcher.cc:867
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
Definition Fetcher.cc:899
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Definition Fetcher.cc:909
virtual ~Fetcher()
Dtor.
Definition Fetcher.cc:849
Media access layer responsible for handling files distributed on a set of media with media change and...
void precacheFiles(const std::vector< OnMediaLocation > &files)
Tries to fetch the given files and precaches them.
@ PROVIDE_DEFAULT
The user is not asked anything, and the error exception is just propagated.
Pathname provideFile(const OnMediaLocation &resource, ProvideFileOptions options=PROVIDE_DEFAULT)
Provides a file from a media location.
void dirInfo(filesystem::DirContent &retlist, const Pathname &dirname, bool dots=true, unsigned media_nr=1)
Fills retlist with directory information.
Describes a resource file located on a medium.
bool optional() const
Whether this is an optional resource.
const Pathname & filename() const
The path to the resource on the medium.
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
unsigned medianr() const
The media number the resource is located on.
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
const CheckSum & checksum() const
The checksum of the resource on the server.
OnMediaLocation & setOptional(bool val)
Set whether the resource is optional.
Maintain [min,max] and counter (value) for progress counting.
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
bool incr(value_type val_r=1)
Increment counter value (default by 1).
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Definition PublicKey.h:365
Checks for the validity of a signature.
Definition FileChecker.h:71
Wrapper class for stat/lstat.
Definition PathInfo.h:222
bool isExist() const
Return whether valid stat info exists.
Definition PathInfo.h:282
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition Pathname.h:175
Pathname dirname() const
Return all but the last component od this path.
Definition Pathname.h:126
const char * c_str() const
String representation.
Definition Pathname.h:112
const std::string & asString() const
String representation.
Definition Pathname.h:93
std::string basename() const
Return the last component of this path.
Definition Pathname.h:130
bool empty() const
Test for an empty path.
Definition Pathname.h:116
Parse repoindex part from a content file.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
String related utilities and Regular expression matching.
std::list< DirEntry > DirContent
Returned by readdir.
Definition PathInfo.h:519
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition PathInfo.cc:888
int unlink(const Pathname &path)
Like 'unlink'.
Definition PathInfo.cc:705
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Definition String.cc:264
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition String.h:1026
Easy-to use interface to the ZYPP dependency resolver.
std::ostream & operator<<(std::ostream &str, const SerialNumber &obj)
shared_ptr< FetcherIndex > FetcherIndex_Ptr
Definition Fetcher.cc:54
std::string asString(const Patch::Category &obj)
Definition Patch.cc:122
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition FileChecker.h:38
shared_ptr< FetcherJob > FetcherJob_Ptr
Definition Fetcher.cc:114
zypp::RepoManager::RefreshServiceOptions _options
parser::susetags::RepoIndex_Ptr _repoindex
Definition Fetcher.cc:593
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Definition Fetcher.cc:590
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
Definition Fetcher.cc:44
OnMediaLocation location
Index localtion.
Definition Fetcher.cc:49
FetcherIndex(const OnMediaLocation &loc)
Definition Fetcher.cc:45
DefaultIntegral< bool, false > read
Whether we read this index.
Definition Fetcher.cc:51
Class to encapsulate the OnMediaLocation object and the FileChecker together.
Definition Fetcher.cc:80
std::list< FileChecker > checkers
Definition Fetcher.cc:109
FetcherJob(const FetcherJob &)=default
FetcherJob(FetcherJob &&)=default
OnMediaLocation location
Definition Fetcher.cc:107
FetcherJob & operator=(const FetcherJob &)=default
ZYPP_DECLARE_FLAGS(Flags, Flag)
FetcherJob & operator=(FetcherJob &&)=default
FetcherJob(const OnMediaLocation &loc)
Definition Fetcher.cc:98
std::set ordering (less semantic)
Definition Fetcher.cc:58
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Definition Fetcher.cc:59
Listentry returned by readdir.
Definition PathInfo.h:502
zypp::parser::susetags::RepoIndex_Ptr _repoindex
Definition susetags.cc:324
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition Easy.h:28
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition Exception.h:444
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition Exception.h:440
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition Exception.h:424
#define ZYPP_DECLARE_OPERATORS_FOR_FLAGS(Name)
Definition Flags.h:177
#define DBG
Definition Logger.h:97
#define MIL
Definition Logger.h:98
#define ERR
Definition Logger.h:100
#define WAR
Definition Logger.h:99