From a777ff01c79d5e0e2cf3ae7b0652795577253bc3 Mon Sep 17 00:00:00 2001

From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>

Date: Thu, 14 Jan 2021 09:58:30 +0100

Subject: [PATCH 1/3] Fix recreate script

---

 tests/repos/rpm/recreate | 2 +-

 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/repos/rpm/recreate b/tests/repos/rpm/recreate

index da348d9799..0fbb9396bd 100755

--- a/tests/repos/rpm/recreate

+++ b/tests/repos/rpm/recreate

@@ -1,6 +1,6 @@

 #!/bin/bash

-THISDIR="$( readlink -f "$( dirname "$0 )" )"

+THISDIR="$( readlink -f "$( dirname "$0" )" )"

 cd "$THISDIR"

 git rm -rf repodata/

 createrepo --no-database -o . ..

From 5d4c0266f6967c7cd5f0e675b13fa3e9b395e4dd Mon Sep 17 00:00:00 2001

From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>

Date: Thu, 14 Jan 2021 10:28:53 +0100

Subject: [PATCH 2/3] Add unit test for fill_sack_from_repos_in_cache

 (RhBug:1865803)

https://bugzilla.redhat.com/show_bug.cgi?id=1865803

---

 tests/test_fill_sack_from_repos_in_cache.py | 262 ++++++++++++++++++++

 1 file changed, 262 insertions(+)

 create mode 100644 tests/test_fill_sack_from_repos_in_cache.py

diff --git a/tests/test_fill_sack_from_repos_in_cache.py b/tests/test_fill_sack_from_repos_in_cache.py

new file mode 100644

index 0000000000..24b0d4598d

--- /dev/null

+++ b/tests/test_fill_sack_from_repos_in_cache.py

@@ -0,0 +1,262 @@

+# -*- coding: utf-8 -*-

+

+# Copyright (C) 2012-2021 Red Hat, Inc.

+#

+# This copyrighted material is made available to anyone wishing to use,

+# modify, copy, or redistribute it subject to the terms and conditions of

+# the GNU General Public License v.2, or (at your option) any later version.

+# This program is distributed in the hope that it will be useful, but WITHOUT

+# ANY WARRANTY expressed or implied, including the implied warranties of

+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General

+# Public License for more details.  You should have received a copy of the

+# GNU General Public License along with this program; if not, write to the

+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

+# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the

+# source code or documentation are not subject to the GNU General Public

+# License and may only be used or replicated with the express permission of

+# Red Hat, Inc.

+#

+

+from __future__ import absolute_import

+from __future__ import unicode_literals

+

+import os

+import tempfile

+import glob

+import shutil

+import unittest

+

+import dnf.exceptions

+import dnf.repo

+import dnf.sack

+

+import hawkey

+

+import tests.support

+from tests.support import mock

+

+TEST_REPO_NAME = "test-repo"

+

+

+class FillSackFromReposInCacheTest(unittest.TestCase):

+    def _create_cache_for_repo(self, repopath, tmpdir):

+        conf = dnf.conf.MainConf()

+        conf.cachedir = os.path.join(tmpdir, "cache")

+

+        base = dnf.Base(conf=conf)

+

+        repoconf = dnf.repo.Repo(TEST_REPO_NAME, base.conf)

+        repoconf.baseurl = repopath

+        repoconf.enable()

+

+        base.repos.add(repoconf)

+

+        base.fill_sack(load_system_repo=False)

+        base.close()

+

+    def _setUp_from_repo_path(self, original_repo_path):

+        self.tmpdir = tempfile.mkdtemp(prefix="dnf_test_")

+

+        self.repo_copy_path = os.path.join(self.tmpdir, "repo")

+        shutil.copytree(original_repo_path, self.repo_copy_path)

+

+        self._create_cache_for_repo(self.repo_copy_path, self.tmpdir)

+

+        # Just to be sure remove repo (it shouldn't be used)

+        shutil.rmtree(self.repo_copy_path)

+

+        # Prepare base for the actual test

+        conf = dnf.conf.MainConf()

+        conf.cachedir = os.path.join(self.tmpdir, "cache")

+        self.test_base = dnf.Base(conf=conf)

+        repoconf = dnf.repo.Repo(TEST_REPO_NAME, conf)

+        repoconf.baseurl = self.repo_copy_path

+        repoconf.enable()

+        self.test_base.repos.add(repoconf)

+

+    def tearDown(self):

+        self.test_base.close()

+        shutil.rmtree(self.tmpdir)

+

+    def test_with_solv_solvx_repomd(self):

+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))

+

+        # Remove xml metadata except repomd

+        # repomd.xml is not compressed and doesn't end with .gz

+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))

+        for f in repodata_without_repomd:

+            os.remove(f)

+

+        # Now we only have cache with just solv, solvx files and repomd.xml

+

+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)

+

+        q = self.test_base.sack.query()

+        packages = q.run()

+        self.assertEqual(len(packages), 9)

+        self.assertEqual(packages[0].evr, "4-4")

+

+        # Use *-updateinfo.solvx

+        adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT)

+        adv_titles = set()

+        for pkg in adv_pkgs:

+            adv_titles.add(pkg.get_advisory(self.test_base.sack).title)

+        self.assertEqual(len(adv_titles), 3)

+

+    def test_with_just_solv_repomd(self):

+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))

+

+        # Remove xml metadata except repomd

+        # repomd.xml is not compressed and doesn't end with .gz

+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))

+        for f in repodata_without_repomd:

+            os.remove(f)

+

+        # Remove solvx files

+        solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solvx"))

+        for f in solvx:

+            os.remove(f)

+

+        # Now we only have cache with just solv files and repomd.xml

+

+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)

+

+        q = self.test_base.sack.query()

+        packages = q.run()

+        self.assertEqual(len(packages), 9)

+        self.assertEqual(packages[0].evr, "4-4")

+

+        # No *-updateinfo.solvx -> we get no advisory packages

+        adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT)

+        self.assertEqual(len(adv_pkgs), 0)

+

+    def test_with_xml_metadata(self):

+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))

+

+        # Remove all solv and solvx files

+        solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*"))

+        for f in solvx:

+            os.remove(f)

+

+        # Now we only have cache with just xml metadata

+

+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)

+

+        q = self.test_base.sack.query()

+        packages = q.run()

+        self.assertEqual(len(packages), 9)

+        self.assertEqual(packages[0].evr, "4-4")

+

+    def test_exception_without_repomd(self):

+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))

+

+        # Remove xml metadata

+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*"))

+        for f in repodata_without_repomd:

+            os.remove(f)

+

+        # Now we only have cache with just solv and solvx files

+        # Since we don't have repomd we cannot verify checksums -> fail (exception)

+

+        self.assertRaises(dnf.exceptions.RepoError,

+                          self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)

+

+    def test_exception_with_just_repomd(self):

+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))

+

+        # Remove xml metadata except repomd

+        # repomd.xml is not compressed and doesn't end with .gz

+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))

+        for f in repodata_without_repomd:

+            os.remove(f)

+

+        # Remove all solv and solvx files

+        solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*"))

+        for f in solvx:

+            os.remove(f)

+

+        # Now we only have cache with just repomd

+        # repomd is not enough, it doesn't contain the metadata it self -> fail (exception)

+

+        self.assertRaises(dnf.exceptions.RepoError,

+                          self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)

+

+    def test_exception_with_checksum_mismatch_and_only_repomd(self):

+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))

+

+        # Remove xml metadata except repomd

+        # repomd.xml is not compressed and doesn't end with .gz

+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))

+        for f in repodata_without_repomd:

+            os.remove(f)

+

+        # Modify checksum of solv file so it doesn't match with repomd

+        solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0]

+        with open(solv, "a") as opensolv:

+            opensolv.write("appended text to change checksum")

+

+        # Now we only have cache with solvx, modified solv file and just repomd

+        # Since we don't have original xml metadata we cannot regenerate solv -> fail (exception)

+

+        self.assertRaises(dnf.exceptions.RepoError,

+                          self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)

+

+    def test_checksum_mistmatch_regenerates_solv(self):

+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))

+

+        # Modify checksum of solv file so it doesn't match with repomd

+        solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0]

+        with open(solv, "a") as opensolv:

+            opensolv.write("appended text to change checksum")

+

+        # Now we only have cache with solvx, modified solv file and xml metadata.

+        # Checksum mistmatch causes regeneration of solv file and repo works.

+

+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)

+

+        q = self.test_base.sack.query()

+        packages = q.run()

+        self.assertEqual(len(packages), 9)

+        self.assertEqual(packages[0].evr, "4-4")

+

+    def test_with_modules_yaml(self):

+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)),

+                                                "modules/modules/_all/x86_64"))

+

+        # Now we have full cache (also with modules.yaml)

+

+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)

+

+        q = self.test_base.sack.query()

+        packages = q.run()

+        self.assertEqual(len(packages), 8)

+        self.assertEqual(packages[0].evr, "2.02-0.40")

+

+        self.module_base = dnf.module.module_base.ModuleBase(self.test_base)

+        modules, _ = self.module_base._get_modules("base-runtime*")

+        self.assertEqual(len(modules), 3)

+        self.assertEqual(modules[0].getFullIdentifier(), "base-runtime:f26:1::")

+

+    def test_with_modular_repo_without_modules_yaml(self):

+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)),

+                                                "modules/modules/_all/x86_64"))

+

+        # Remove xml and yaml metadata except repomd

+        # repomd.xml is not compressed and doesn't end with .gz

+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))

+        for f in repodata_without_repomd:

+            os.remove(f)

+

+        # Now we have just solv, *-filenames.solvx and repomd.xml (modules.yaml are not processed into *-modules.solvx)

+

+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)

+

+        q = self.test_base.sack.query()

+        packages = q.run()

+        # We have many more packages because they are not hidden by modules

+        self.assertEqual(len(packages), 44)

+        self.assertEqual(packages[0].evr, "10.0-7")

+

+        self.module_base = dnf.module.module_base.ModuleBase(self.test_base)

+        modules, _ = self.module_base._get_modules("base-runtime*")

+        self.assertEqual(len(modules), 0)

From de6177dba3dc20191e275eec14672570a0c4f4a8 Mon Sep 17 00:00:00 2001

From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>

Date: Thu, 14 Jan 2021 12:29:06 +0100

Subject: [PATCH 3/3] Add docs and examples for fill_sack_from_repos_in_cache

 (RhBug:1865803)

https://bugzilla.redhat.com/show_bug.cgi?id=1865803

---

 doc/api_base.rst | 41 +++++++++++++++++++++++++++++++++++++++++

 1 file changed, 41 insertions(+)

diff --git a/doc/api_base.rst b/doc/api_base.rst

index 24ecb50e43..f0b1992e88 100644

--- a/doc/api_base.rst

+++ b/doc/api_base.rst

@@ -111,6 +111,47 @@

             print("id: {}".format(repo.id))

             print("baseurl: {}".format(repo.baseurl))

+  .. method:: fill_sack_from_repos_in_cache(load_system_repo=True)

+

+    Prepare Sack and Goal objects and load all enabled repositories from cache only, it doesn't download anything and it doesn't check if metadata are expired.

+    To successfully load a repository cache it requires repond.xml plus metadata (xml, yaml) or repond.xml plus generated cache files (solv, solvx).

+    If there is not enough metadata given repo is either skipped or it throws a :exc:`dnf.exceptions.RepoError` exception depending on :attr:`dnf.conf.Conf.skip_if_unavailable` configuration.

+

+    All additional metadata are loaded if present but are not generally required. Note that some metadata like updateinfo.xml get processed into a solvx cache file and its sufficient to have either xml or solvx. Module metadata represented by modules.yaml are not processed therefore they are needed when they are defined in repomd.xml.

+

+    Example of loading all configured repositories from cache and printing available packages' names::

+

+        #!/usr/bin/python3

+        import dnf

+

+        with dnf.Base() as base:

+            base.read_all_repos()

+

+            base.fill_sack_from_repos_in_cache(load_system_repo=False)

+

+            query = base.sack.query().available()

+            for pkg in query.run():

+                print(pkg.name)

+

+    Example of loading a single repository and printing available packages' names without reading repository configuration::

+

+        #!/usr/bin/python3

+        import dnf

+

+        with dnf.Base() as base:

+            repo = dnf.repo.Repo("rawhide", base.conf)

+

+            # Repository cache is also identified by its source therefore to find it you need to

+            # set metalink, mirrorlist or baseurl to the same value from which it was created.

+            repo.metalink = "https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=x86_64"

+

+            base.repos.add(repo)

+

+            base.fill_sack_from_repos_in_cache(load_system_repo=False)

+

+            query = base.sack.query().available()

+            for pkg in query.run():

+                print(pkg.name)

   .. method:: do_transaction([display])