18#include <zypp-curl/parser/MetaLinkParser>
55 struct RepoMirrorListTempProvider
57 RepoMirrorListTempProvider()
60 RepoMirrorListTempProvider( Pathname localfile_r )
61 : _localfile(std::move( localfile_r ))
64 RepoMirrorListTempProvider(
const Url & url_r )
66 if ( url_r.schemeIsDownloading()
68 && url_r.getQueryStringMap().count(
"mirrorlist") > 0 ) {
71 const auto &authCb = [&](
const zypp::Url &, media::TransferSettings &settings,
const std::string & availAuthTypes,
bool firstTry,
bool &canContinue ) {
72 media::CredentialManager cm(media::CredManagerOptions(
ZConfig::instance().repoManagerRoot()));
80 internal::MediaNetworkRequestExecutor executor;
81 executor.sigAuthRequired ().connect(authCb);
83 _tmpfile = filesystem::TmpFile();
84 _localfile = _tmpfile->path();
88 auto tSettings = media::TransferSettings();
91 auto req = std::make_shared<zyppng::NetworkRequest>( url_r, _localfile );
92 req->transferSettings () = tSettings;
93 executor.executeRequest ( req,
nullptr );
98 ERR <<
"Failed to chmod file " << _localfile << endl;
105 Url abs_url( url_r );
106 abs_url.setPathName(
"/" );
107 _access.reset(
new MediaSetAccess( std::vector<zypp::media::MediaUrl>{abs_url} ) );
108 _localfile = _access->provideFile( url_r.getPathName() );
112 const Pathname & localfile()
const
113 {
return _localfile; }
115 shared_ptr<MediaSetAccess> _access;
117 std::optional<filesystem::TmpFile> _tmpfile;
120 enum class RepoMirrorListFormat {
128 static RepoMirrorListFormat detectRepoMirrorListFormat(
const Pathname &localfile ) {
132 MIL <<
"Detecting RepoMirrorlist Format based on file content" << std::endl;
134 if ( localfile.empty () )
135 return RepoMirrorListFormat::Empty;
137 InputStream tmpfstream (localfile);
138 auto &str = tmpfstream.stream();
142 while ( !str.eof () && !str.bad() && ( c ==
' ' || c ==
'\t' || c ==
'\n' || c ==
'\r') )
146 ERR <<
"Failed to read RepoMirrorList file, stream hit EOF early." << std::endl;
147 return RepoMirrorListFormat::Empty;
151 ERR <<
"Failed to read RepoMirrorList file, stream became bad." << std::endl;
152 return RepoMirrorListFormat::Error;
157 MIL <<
"Detected Metalink, file starts with <" << std::endl;
158 return RepoMirrorListFormat::MetaLink;
161 MIL <<
"Detected JSON, file starts with [" << std::endl;
162 return RepoMirrorListFormat::MirrorListJson;
165 MIL <<
"Detected TXT, file starts with " << c << std::endl;
166 return RepoMirrorListFormat::MirrorListTxt;
171 inline std::vector<Url> RepoMirrorListParseXML(
const Pathname &tmpfile )
174 media::MetaLinkParser metalink;
175 metalink.parse(tmpfile);
176 return metalink.getUrls();
179 zypp::parser::ParseException ex(
"Invalid repo metalink format.");
180 ex.remember ( std::current_exception () );
185 inline std::vector<Url> RepoMirrorListParseJSON(
const Pathname &tmpfile )
187 InputStream tmpfstream (tmpfile);
190 using namespace zyppng::operators;
191 using zyppng::operators::operator|;
194 auto res = parser.
parse ( tmpfstream )
195 |
and_then([&]( json::Value data ) {
197 std::vector<Url> urls;
199 MIL <<
"Empty mirrorlist received, no mirrors available." << std::endl;
204 MIL <<
"Unexpected JSON format, top level element must be an array." << std::endl;
205 return zyppng::expected<std::vector<Url>>::error(
ZYPP_EXCPT_PTR( zypp::Exception(
"Unexpected JSON format, top level element must be an array.") ));
207 const auto &topArray = data.
asArray ();
208 for (
const auto &val : topArray ) {
210 MIL <<
"Unexpected JSON element, array must contain only objects. Ignoring current element" << std::endl;
214 const auto &obj = val.asObject();
215 for (
const auto &key : obj ) {
216 if ( key.first ==
"url" ) {
217 const auto &elemValue = key.second;
219 MIL <<
"Unexpected JSON element, element \"url\" must contain a string. Ignoring current element" << std::endl;
223 MIL <<
"Trying to parse URL: " << std::string(elemValue.asString()) << std::endl;
224 urls.push_back (
Url( elemValue.asString() ) );
225 }
catch (
const url::UrlException &e ) {
227 MIL <<
"Invalid URL in mirrors file: "<< elemValue.asString() <<
", ignoring" << std::endl;
236 using zypp::operator<<;
237 MIL <<
"Error while parsing mirrorlist: (" << res.error() <<
"), no mirrors available" << std::endl;
245 MIL <<
"Caught exception while parsing json" << std::endl;
247 zypp::parser::ParseException ex(
"Invalid repo mirror list format, valid JSON was expected.");
248 ex.remember ( std::current_exception () );
254 inline std::vector<Url> RepoMirrorListParseTXT(
const Pathname &tmpfile )
256 InputStream tmpfstream (tmpfile);
257 std::vector<Url> my_urls;
259 while (
getline(tmpfstream.stream(), tmpurl))
261 if ( tmpurl[0] ==
'#' )
264 Url mirrUrl( tmpurl );
265 if ( !mirrUrl.schemeIsDownloading( ) ) {
266 MIL <<
"Ignoring non downloading URL " << tmpurl << std::endl;
268 my_urls.push_back(
Url(tmpurl));
275 ERR <<
"Invalid URL in mirrorlist file." << std::endl;
277 zypp::parser::ParseException ex(
"Invalid repo mirror list format, all Urls must be valid in a mirrorlist txt file.");
278 ex.remember ( std::current_exception () );
286 inline std::vector<Url> RepoMirrorListParse(
const Url & url_r,
const Pathname & listfile_r )
288 USR << url_r <<
" " << listfile_r << endl;
290 std::vector<Url> mirrorurls;
291 switch( detectRepoMirrorListFormat (listfile_r) ) {
292 case RepoMirrorListFormat::Error:
294 ZYPP_THROW( zypp::parser::ParseException( str::Format(
"Unable to detect metalink file format for: %1%") % listfile_r ));
295 case RepoMirrorListFormat::Empty:
298 case RepoMirrorListFormat::MetaLink:
299 mirrorurls = RepoMirrorListParseXML( listfile_r );
301 case RepoMirrorListFormat::MirrorListJson:
302 mirrorurls = RepoMirrorListParseJSON( listfile_r );
304 case RepoMirrorListFormat::MirrorListTxt:
305 mirrorurls = RepoMirrorListParseTXT( listfile_r );
309 std::vector<Url> ret;
310 for (
auto & murl : mirrorurls )
312 if ( murl.getScheme() !=
"rsync" )
314 std::string pName = murl.getPathName();
315 size_t delpos = pName.find(
"repodata/repomd.xml");
316 if( delpos != std::string::npos )
318 murl.setPathName( pName.erase(delpos) );
320 ret.push_back( murl );
331 PathInfo metaPathInfo( metadatapath_r);
332 std::exception_ptr errors;
339 else if ( !metaPathInfo.
isDir() )
342 RepoMirrorListTempProvider provider( url_r );
343 _urls = RepoMirrorListParse( url_r, provider.localfile() );
348 Pathname cachefile = metadatapath_r /
"mirrorlist";
350 bool needRefresh = ( !cacheinfo.
isFile()
357 if ( !needRefresh ) {
359 _urls = RepoMirrorListParse( url_r, cachefile );
360 if(
_urls.empty() ) {
361 DBG <<
"Removing Cachefile as it contains no URLs" << endl;
371 errors = std::make_exception_ptr(ex);
372 MIL <<
"Invalid mirrorlist cachefile, deleting it and trying to fetch a new one" << std::endl;
376 if( cacheinfo.
isFile() ) {
381 DBG <<
"Getting MirrorList from URL: " << url_r << endl;
382 RepoMirrorListTempProvider provider( url_r );
383 _urls = RepoMirrorListParse( url_r, provider.localfile() );
386 && !
_urls.empty() ) {
388 DBG <<
"Copy MirrorList file to " << cachefile << endl;
405 static const std::vector<std::string> hosts{
406 "download.opensuse.org",
409 return ( std::find( hosts.begin(), hosts.end(),
str::toLower(
url.getHost() )) != hosts.end() );
Base class for Exception.
void remember(const Exception &old_r)
Store an other Exception as history.
std::string getScheme() const
Returns the scheme name of the URL.
std::string getPathName(EEncoding eflag=zypp::url::E_DECODED) const
Returns the path name from the URL.
static ZConfig & instance()
Singleton ctor.
Wrapper class for stat/lstat.
zyppng::expected< Value > parse(const InputStream &input_r)
Parse the stream.
const Array & asArray() const
RepoMirrorList(const Url &url_r, const Pathname &metadatapath_r)
static bool urlSupportsMirrorLink(const zypp::Url &url)
void prepareSettingsAndUrl(zypp::Url &url_r, zypp::media::TransferSettings &s)
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
int unlink(const Pathname &path)
Like 'unlink'.
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
std::string getline(std::istream &str)
Read one line from stream.
std::string toLower(const std::string &s)
Return lowercase version of s.
Easy-to use interface to the ZYPP dependency resolver.
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
ResultType and_then(const expected< T, E > &exp, Function &&f)
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
#define ZYPP_EXCPT_PTR(EXCPT)
Drops a logline and returns Exception as a std::exception_ptr.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.