libzypp 17.37.0
RepoMirrorList.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
13#include <iostream>
14#include <utility>
15#include <vector>
16#include <time.h>
18#include <zypp-curl/parser/MetaLinkParser>
19#include <zypp/MediaSetAccess.h>
20#include <zypp/base/LogTools.h>
21#include <zypp/ZConfig.h>
22#include <zypp/PathInfo.h>
24
31
33#include <zypp/media/MediaNetworkCommonHandler.h> // for the authentication workflow
34
36
37
39namespace zypp
40{
42 namespace repo
43 {
44
46 namespace
47 {
55 struct RepoMirrorListTempProvider
56 {
57 RepoMirrorListTempProvider()
58 {}
59
60 RepoMirrorListTempProvider( Pathname localfile_r )
61 : _localfile(std::move( localfile_r ))
62 {}
63
64 RepoMirrorListTempProvider( const Url & url_r )
65 {
66 if ( url_r.schemeIsDownloading()
68 && url_r.getQueryStringMap().count("mirrorlist") > 0 ) {
69
70 // Auth will probably never be triggered, but add it for completeness
71 const auto &authCb = [&]( const zypp::Url &, media::TransferSettings &settings, const std::string & availAuthTypes, bool firstTry, bool &canContinue ) {
72 media::CredentialManager cm(media::CredManagerOptions(ZConfig::instance().repoManagerRoot()));
73 if ( media::MediaNetworkCommonHandler::authenticate( url_r, cm, settings, availAuthTypes, firstTry ) ) {
74 canContinue = true;
75 return;
76 }
77 canContinue = false;
78 };
79
80 internal::MediaNetworkRequestExecutor executor;
81 executor.sigAuthRequired ().connect(authCb);
82
83 _tmpfile = filesystem::TmpFile();
84 _localfile = _tmpfile->path();
85
86 // prepare Url and Settings
87 auto url = url_r;
88 auto tSettings = media::TransferSettings();
89 ::internal::prepareSettingsAndUrl( url, tSettings );
90
91 auto req = std::make_shared<zyppng::NetworkRequest>( url_r, _localfile );
92 req->transferSettings () = tSettings;
93 executor.executeRequest ( req, nullptr );
94
95 // apply umask
96 if ( ::chmod( _localfile.c_str(), filesystem::applyUmaskTo( 0644 ) ) )
97 {
98 ERR << "Failed to chmod file " << _localfile << endl;
99 }
100
101 return;
102 }
103
104 // this will handle traditional media including URL resolver plugins
105 Url abs_url( url_r );
106 abs_url.setPathName( "/" );
107 _access.reset( new MediaSetAccess( std::vector<zypp::media::MediaUrl>{abs_url} ) );
108 _localfile = _access->provideFile( url_r.getPathName() );
109
110 }
111
112 const Pathname & localfile() const
113 { return _localfile; }
114 private:
115 shared_ptr<MediaSetAccess> _access;
116 Pathname _localfile;
117 std::optional<filesystem::TmpFile> _tmpfile;
118 };
119
120 enum class RepoMirrorListFormat {
121 Error,
122 Empty,
123 MirrorListTxt,
124 MirrorListJson,
125 MetaLink
126 };
127
128 static RepoMirrorListFormat detectRepoMirrorListFormat( const Pathname &localfile ) {
129 // a file starting with < is most likely a metalink file,
130 // a file starting with [ is most likely a json file,
131 // else we go for txt
132 MIL << "Detecting RepoMirrorlist Format based on file content" << std::endl;
133
134 if ( localfile.empty () )
135 return RepoMirrorListFormat::Empty;
136
137 InputStream tmpfstream (localfile);
138 auto &str = tmpfstream.stream();
139 auto c = str.get ();
140
141 // skip preceding whitespaces
142 while ( !str.eof () && !str.bad() && ( c == ' ' || c == '\t' || c == '\n' || c == '\r') )
143 c = str.get ();
144
145 if ( str.eof() ) {
146 ERR << "Failed to read RepoMirrorList file, stream hit EOF early." << std::endl;
147 return RepoMirrorListFormat::Empty;
148 }
149
150 if ( str.bad() ) {
151 ERR << "Failed to read RepoMirrorList file, stream became bad." << std::endl;
152 return RepoMirrorListFormat::Error;
153 }
154
155 switch ( c ) {
156 case '<': {
157 MIL << "Detected Metalink, file starts with <" << std::endl;
158 return RepoMirrorListFormat::MetaLink;
159 }
160 case '[': {
161 MIL << "Detected JSON, file starts with [" << std::endl;
162 return RepoMirrorListFormat::MirrorListJson;
163 }
164 default: {
165 MIL << "Detected TXT, file starts with " << c << std::endl;
166 return RepoMirrorListFormat::MirrorListTxt;
167 }
168 }
169 }
170
171 inline std::vector<Url> RepoMirrorListParseXML( const Pathname &tmpfile )
172 {
173 try {
174 media::MetaLinkParser metalink;
175 metalink.parse(tmpfile);
176 return metalink.getUrls();
177 } catch (...) {
178 ZYPP_CAUGHT( std::current_exception() );
179 zypp::parser::ParseException ex("Invalid repo metalink format.");
180 ex.remember ( std::current_exception () );
181 ZYPP_THROW(ex);
182 }
183 }
184
185 inline std::vector<Url> RepoMirrorListParseJSON( const Pathname &tmpfile )
186 {
187 InputStream tmpfstream (tmpfile);
188
189 try {
190 using namespace zyppng::operators;
191 using zyppng::operators::operator|;
192
193 json::Parser parser;
194 auto res = parser.parse ( tmpfstream )
195 | and_then([&]( json::Value data ) {
196
197 std::vector<Url> urls;
198 if ( data.isNull () ) {
199 MIL << "Empty mirrorlist received, no mirrors available." << std::endl;
201 }
202
203 if ( data.type() != json::Value::ArrayType ) {
204 MIL << "Unexpected JSON format, top level element must be an array." << std::endl;
205 return zyppng::expected<std::vector<Url>>::error( ZYPP_EXCPT_PTR( zypp::Exception("Unexpected JSON format, top level element must be an array.") ));
206 }
207 const auto &topArray = data.asArray ();
208 for ( const auto &val : topArray ) {
209 if ( val.type () != json::Value::ObjectType ) {
210 MIL << "Unexpected JSON element, array must contain only objects. Ignoring current element" << std::endl;
211 continue;
212 }
213
214 const auto &obj = val.asObject();
215 for ( const auto &key : obj ) {
216 if ( key.first == "url" ) {
217 const auto &elemValue = key.second;
218 if ( elemValue.type() != json::Value::StringType ) {
219 MIL << "Unexpected JSON element, element \"url\" must contain a string. Ignoring current element" << std::endl;
220 break;
221 }
222 try {
223 MIL << "Trying to parse URL: " << std::string(elemValue.asString()) << std::endl;
224 urls.push_back ( Url( elemValue.asString() ) );
225 } catch ( const url::UrlException &e ) {
226 ZYPP_CAUGHT(e);
227 MIL << "Invalid URL in mirrors file: "<< elemValue.asString() << ", ignoring" << std::endl;
228 }
229 }
230 }
231 }
233 });
234
235 if ( !res ) {
236 using zypp::operator<<;
237 MIL << "Error while parsing mirrorlist: (" << res.error() << "), no mirrors available" << std::endl;
238 ZYPP_RETHROW( res.error () );
239 }
240
241 return *res;
242
243 } catch (...) {
244 ZYPP_CAUGHT( std::current_exception() );
245 MIL << "Caught exception while parsing json" << std::endl;
246
247 zypp::parser::ParseException ex("Invalid repo mirror list format, valid JSON was expected.");
248 ex.remember ( std::current_exception () );
249 ZYPP_THROW(ex);
250 }
251 return {};
252 }
253
254 inline std::vector<Url> RepoMirrorListParseTXT( const Pathname &tmpfile )
255 {
256 InputStream tmpfstream (tmpfile);
257 std::vector<Url> my_urls;
258 std::string tmpurl;
259 while (getline(tmpfstream.stream(), tmpurl))
260 {
261 if ( tmpurl[0] == '#' )
262 continue;
263 try {
264 Url mirrUrl( tmpurl );
265 if ( !mirrUrl.schemeIsDownloading( ) ) {
266 MIL << "Ignoring non downloading URL " << tmpurl << std::endl;
267 }
268 my_urls.push_back(Url(tmpurl));
269 }
270 catch (...)
271 {
272 ZYPP_CAUGHT( std::current_exception() );
273
274 // fail on invalid URLs
275 ERR << "Invalid URL in mirrorlist file." << std::endl;
276
277 zypp::parser::ParseException ex("Invalid repo mirror list format, all Urls must be valid in a mirrorlist txt file.");
278 ex.remember ( std::current_exception () );
279 ZYPP_THROW(ex);
280 }
281 }
282 return my_urls;
283 }
284
286 inline std::vector<Url> RepoMirrorListParse( const Url & url_r, const Pathname & listfile_r )
287 {
288 USR << url_r << " " << listfile_r << endl;
289
290 std::vector<Url> mirrorurls;
291 switch( detectRepoMirrorListFormat (listfile_r) ) {
292 case RepoMirrorListFormat::Error:
293 // should not happen, except when the instr goes bad
294 ZYPP_THROW( zypp::parser::ParseException( str::Format("Unable to detect metalink file format for: %1%") % listfile_r ));
295 case RepoMirrorListFormat::Empty:
296 mirrorurls = {};
297 break;
298 case RepoMirrorListFormat::MetaLink:
299 mirrorurls = RepoMirrorListParseXML( listfile_r );
300 break;
301 case RepoMirrorListFormat::MirrorListJson:
302 mirrorurls = RepoMirrorListParseJSON( listfile_r );
303 break;
304 case RepoMirrorListFormat::MirrorListTxt:
305 mirrorurls = RepoMirrorListParseTXT( listfile_r );
306 break;
307 }
308
309 std::vector<Url> ret;
310 for ( auto & murl : mirrorurls )
311 {
312 if ( murl.getScheme() != "rsync" )
313 {
314 std::string pName = murl.getPathName();
315 size_t delpos = pName.find("repodata/repomd.xml");
316 if( delpos != std::string::npos )
317 {
318 murl.setPathName( pName.erase(delpos) );
319 }
320 ret.push_back( murl );
321 }
322 }
323 return ret;
324 }
325
326 } // namespace
328
329 RepoMirrorList::RepoMirrorList( const Url & url_r, const Pathname & metadatapath_r )
330 {
331 PathInfo metaPathInfo( metadatapath_r);
332 std::exception_ptr errors; // we collect errors here
333 try {
334 if ( url_r.getScheme() == "file" )
335 {
336 // never cache for local mirrorlist
337 _urls = RepoMirrorListParse( url_r, url_r.getPathName() );
338 }
339 else if ( !metaPathInfo.isDir() )
340 {
341 // no cachedir or no access
342 RepoMirrorListTempProvider provider( url_r ); // RAII: lifetime of any downloaded files
343 _urls = RepoMirrorListParse( url_r, provider.localfile() );
344 }
345 else
346 {
347 // have cachedir
348 Pathname cachefile = metadatapath_r / "mirrorlist";
349 zypp::filesystem::PathInfo cacheinfo( cachefile );
350 bool needRefresh = ( !cacheinfo.isFile()
351 // force a update on a old cache ONLY if the user can write the cache, otherwise we use an already existing cachefile
352 // it makes no sense to continously download the mirrors file if we can't store it
353 || ( cacheinfo.mtime() < time(NULL) - (long) ZConfig::instance().repo_refresh_delay() * 60 && metaPathInfo.userMayRWX () ) );
354
355 // up to date: try to parse and use the URLs if sucessful
356 // otherwise fetch the URL again
357 if ( !needRefresh ) {
358 try {
359 _urls = RepoMirrorListParse( url_r, cachefile );
360 if( _urls.empty() ) {
361 DBG << "Removing Cachefile as it contains no URLs" << endl;
362 zypp::filesystem::unlink( cachefile );
363 }
364 return;
365
366 } catch ( const zypp::Exception & e ) {
367 ZYPP_CAUGHT(e);
368 auto ex = e;
369 if ( errors )
370 ex.remember(errors);
371 errors = std::make_exception_ptr(ex);
372 MIL << "Invalid mirrorlist cachefile, deleting it and trying to fetch a new one" << std::endl;
373 }
374 }
375
376 if( cacheinfo.isFile() ) {
377 // remove the old one, it's either broken, empty or outdated
378 filesystem::unlink(cachefile);
379 }
380
381 DBG << "Getting MirrorList from URL: " << url_r << endl;
382 RepoMirrorListTempProvider provider( url_r ); // RAII: lifetime of downloaded file
383 _urls = RepoMirrorListParse( url_r, provider.localfile() );
384
385 if ( metaPathInfo.userMayRWX()
386 && !_urls.empty() ) {
387 // Create directory, if not existing
388 DBG << "Copy MirrorList file to " << cachefile << endl;
389 zypp::filesystem::assert_dir( metadatapath_r );
390 zypp::filesystem::hardlinkCopy( provider.localfile(), cachefile );
391 }
392 }
393 } catch ( const zypp::Exception &e ) {
394 // Make a more user readable exception
395 ZYPP_CAUGHT(e);
396 parser::ParseException ex( str::Format("Failed to parse/receive mirror information for URL: %1%") % url_r );
397 ex.remember(e);
398 if ( errors ) ex.remember(errors);
399 ZYPP_THROW(ex);
400 }
401 }
402
404 {
405 static const std::vector<std::string> hosts{
406 "download.opensuse.org",
407 "cdn.opensuse.org"
408 };
409 return ( std::find( hosts.begin(), hosts.end(), str::toLower( url.getHost() )) != hosts.end() );
410 }
411
413 } // namespace repo
416} // namespace zypp
Base class for Exception.
Definition Exception.h:153
void remember(const Exception &old_r)
Store an other Exception as history.
Definition Exception.cc:154
Url manipulation class.
Definition Url.h:93
std::string getScheme() const
Returns the scheme name of the URL.
Definition Url.cc:551
std::string getPathName(EEncoding eflag=zypp::url::E_DECODED) const
Returns the path name from the URL.
Definition Url.cc:622
static ZConfig & instance()
Singleton ctor.
Definition ZConfig.cc:940
Wrapper class for stat/lstat.
Definition PathInfo.h:226
zyppng::expected< Value > parse(const InputStream &input_r)
Parse the stream.
Definition json.cc:37
bool isNull() const
Definition JsonValue.h:277
Type type() const
Definition JsonValue.h:281
const Array & asArray() const
Definition JsonValue.h:269
bool authenticate(const Url &url, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
RepoMirrorList(const Url &url_r, const Pathname &metadatapath_r)
static bool urlSupportsMirrorLink(const zypp::Url &url)
std::vector< Url > _urls
void prepareSettingsAndUrl(zypp::Url &url_r, zypp::media::TransferSettings &s)
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition PathInfo.h:805
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition PathInfo.cc:888
int unlink(const Pathname &path)
Like 'unlink'.
Definition PathInfo.cc:705
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition PathInfo.cc:324
@ Error
Definition IOTools.h:74
std::string getline(std::istream &str)
Read one line from stream.
Definition IOStream.cc:33
std::string toLower(const std::string &s)
Return lowercase version of s.
Definition String.cc:180
Url details namespace.
Definition UrlBase.cc:58
Easy-to use interface to the ZYPP dependency resolver.
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
Definition expected.h:397
ResultType and_then(const expected< T, E > &exp, Function &&f)
Definition expected.h:423
zypp::Url Url
Definition url.h:15
Convenient building of std::string with boost::format.
Definition String.h:254
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition Exception.h:479
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition Exception.h:475
#define ZYPP_EXCPT_PTR(EXCPT)
Drops a logline and returns Exception as a std::exception_ptr.
Definition Exception.h:463
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition Exception.h:459
#define DBG
Definition Logger.h:99
#define MIL
Definition Logger.h:100
#define USR
Definition Logger.h:105
#define ERR
Definition Logger.h:102