From 465a6a59279bd7fa2680c626ca0f10c059276668 Mon Sep 17 00:00:00 2001

From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>

Date: Wed, 9 Feb 2022 13:18:41 +0100

Subject: [PATCH 29/34] Use dnf solv userdata to check versions and checksum

 (RhBug:2027445)

Remove unused functions for checksums

= changelog =

msg: Write and check versions and checksums for solvfile cache by using new dnf solvfile userdata (RhBug:2027445)

     It is not possible to use old cache files, therefore cache regeneration is triggered automatically.

type: bugfix

resolves: https://bugzilla.redhat.com/show_bug.cgi?id=2027445

---

 libdnf/dnf-sack.cpp         | 254 ++++++++++++++++++++++--------------

 libdnf/hy-iutil-private.hpp |   2 -

 libdnf/hy-iutil.cpp         |  20 ---

 3 files changed, 156 insertions(+), 120 deletions(-)

diff --git a/libdnf/dnf-sack.cpp b/libdnf/dnf-sack.cpp

index b9baeaef..61f4807c 100644

--- a/libdnf/dnf-sack.cpp

+++ b/libdnf/dnf-sack.cpp

@@ -225,17 +225,39 @@ dnf_sack_new(void)

     return DNF_SACK(g_object_new(DNF_TYPE_SACK, NULL));

 }

-static int

-can_use_repomd_cache(FILE *fp_solv, unsigned char cs_repomd[CHKSUM_BYTES])

-{

-    unsigned char cs_cache[CHKSUM_BYTES];

-

-    if (fp_solv &&

-        !checksum_read(cs_cache, fp_solv) &&

-        !checksum_cmp(cs_cache, cs_repomd))

-        return 1;

+// Try to load cached solv file into repo otherwise return FALSE

+static gboolean

+try_to_use_cached_solvfile(const char *path, Repo *repo, int flags, const unsigned char *checksum, GError **err){

+    FILE *fp_cache = fopen(path, "r");

+    if (!fp_cache) {

+        // Missing cache files (ENOENT) are not an error and can even be expected in some cases

+        // (such as when repo doesn't have updateinfo/prestodelta metadata).

+        // Use g_debug in order not to pollute the log by default with such entries.

+        if (errno == ENOENT) {

+            g_debug("Failed to open solvfile cache: %s: %s", path, strerror(errno));

+        } else {

+            g_warning("Failed to open solvfile cache: %s: %s", path, strerror(errno));

+        }

+        return FALSE;

+    }

+    std::unique_ptr<SolvUserdata> solv_userdata = solv_userdata_read(fp_cache);

+    gboolean ret = TRUE;

+    if (solv_userdata && solv_userdata_verify(solv_userdata.get(), checksum)) {

+        // after reading the header rewind to the begining

+        fseek(fp_cache, 0, SEEK_SET);

+        if (repo_add_solv(repo, fp_cache, flags)) {

+            g_set_error (err,

+                         DNF_ERROR,

+                         DNF_ERROR_INTERNAL_ERROR,

+                         _("repo_add_solv() has failed."));

+            ret = FALSE;

+        }

+    } else {

+        ret = FALSE;

+    }

-    return 0;

+    fclose(fp_cache);

+    return ret;

 }

 void

@@ -375,33 +397,27 @@ load_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata,

     gboolean done = FALSE;

     char *fn_cache =  dnf_sack_give_cache_fn(sack, name, suffix);

-    fp = fopen(fn_cache, "r");

     assert(libdnf::repoGetImpl(hrepo)->checksum);

-    if (can_use_repomd_cache(fp, libdnf::repoGetImpl(hrepo)->checksum)) {

-        int flags = 0;

-        /* the updateinfo is not a real extension */

-        if (which_repodata != _HY_REPODATA_UPDATEINFO)

-            flags |= REPO_EXTEND_SOLVABLES;

-        /* do not pollute the main pool with directory component ids */

-        if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)

-            flags |= REPO_LOCALPOOL;

-        done = TRUE;

+

+    int flags = 0;

+    /* the updateinfo is not a real extension */

+    if (which_repodata != _HY_REPODATA_UPDATEINFO)

+        flags |= REPO_EXTEND_SOLVABLES;

+    /* do not pollute the main pool with directory component ids */

+    if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)

+        flags |= REPO_LOCALPOOL;

+    if (try_to_use_cached_solvfile(fn_cache, repo, flags, libdnf::repoGetImpl(hrepo)->checksum, error)) {

         g_debug("%s: using cache file: %s", __func__, fn_cache);

-        ret = repo_add_solv(repo, fp, flags);

-        if (ret) {

-            g_set_error_literal (error,

-                                 DNF_ERROR,

-                                 DNF_ERROR_INTERNAL_ERROR,

-                                 _("failed to add solv"));

-            return FALSE;

-        } else {

-            repo_update_state(hrepo, which_repodata, _HY_LOADED_CACHE);

-            repo_set_repodata(hrepo, which_repodata, repo->nrepodata - 1);

-        }

+        done = TRUE;

+        repo_update_state(hrepo, which_repodata, _HY_LOADED_CACHE);

+        repo_set_repodata(hrepo, which_repodata, repo->nrepodata - 1);

     }

+    if (error && *error) {

+        g_prefix_error(error, _("Loading extension cache %s (%d) failed: "), fn_cache, which_repodata);

+        return FALSE;

+    }

+

     g_free(fn_cache);

-    if (fp)

-        fclose(fp);

     if (done)

         return TRUE;

@@ -514,35 +530,53 @@ write_main(DnfSack *sack, HyRepo hrepo, int switchtosolv, GError **error)

                         strerror(errno));

             goto done;

         }

-        rc = repo_write(repo, fp);

-        rc |= checksum_write(repoImpl->checksum, fp);

-        rc |= fclose(fp);

+

+        SolvUserdata solv_userdata;

+        if (solv_userdata_fill(&solv_userdata, repoImpl->checksum, error)) {

+            ret = FALSE;

+            fclose(fp);

+            goto done;

+        }

+

+        Repowriter *writer = repowriter_create(repo);

+        repowriter_set_userdata(writer, &solv_userdata, solv_userdata_size);

+        rc = repowriter_write(writer, fp);

+        repowriter_free(writer);

         if (rc) {

+            ret = FALSE;

+            fclose(fp);

+            g_set_error(error,

+                       DNF_ERROR,

+                       DNF_ERROR_INTERNAL_ERROR,

+                       _("While writing primary cache %s repowriter write failed: %i, error: %s"),

+                       tmp_fn_templ, rc, pool_errstr(repo->pool));

+            goto done;

+        }

+

+        if (fclose(fp)) {

             ret = FALSE;

             g_set_error (error,

                         DNF_ERROR,

                         DNF_ERROR_FILE_INVALID,

-                        _("write_main() failed writing data: %i"), rc);

+                        _("Failed closing tmp file %s: %s"),

+                        tmp_fn_templ, strerror(errno));

             goto done;

         }

     }

     if (switchtosolv && repo_is_one_piece(repo)) {

+        repo_empty(repo, 1);

         /* switch over to written solv file activate paging */

-        FILE *fp = fopen(tmp_fn_templ, "r");

-        if (fp) {

-            repo_empty(repo, 1);

-            rc = repo_add_solv(repo, fp, 0);

-            fclose(fp);

-            if (rc) {

-                /* this is pretty fatal */

-                ret = FALSE;

-                g_set_error_literal (error,

-                                     DNF_ERROR,

-                                     DNF_ERROR_FILE_INVALID,

-                                     _("write_main() failed to re-load "

-                                       "written solv file"));

-                goto done;

-            }

+        gboolean loaded = try_to_use_cached_solvfile(tmp_fn_templ, repo, 0, repoImpl->checksum, error);

+        if (error && *error) {

+            g_prefix_error(error, _("Failed to use newly written primary cache: %s: "), tmp_fn_templ);

+            ret = FALSE;

+            goto done;

+        }

+        if (!loaded) {

+            g_set_error(error, DNF_ERROR, DNF_ERROR_INTERNAL_ERROR,

+                        _("Failed to use newly written primary cache: %s"), tmp_fn_templ);

+            ret = FALSE;

+            goto done;

         }

     }

@@ -569,20 +603,6 @@ write_ext_updateinfo_filter(Repo *repo, Repokey *key, void *kfdata)

     return repo_write_stdkeyfilter(repo, key, 0);

 }

-static int

-write_ext_updateinfo(HyRepo hrepo, Repodata *data, FILE *fp)

-{

-    auto repoImpl = libdnf::repoGetImpl(hrepo);

-    Repo *repo = repoImpl->libsolvRepo;

-    int oldstart = repo->start;

-    repo->start = repoImpl->main_end;

-    repo->nsolvables -= repoImpl->main_nsolvables;

-    int res = repo_write_filtered(repo, fp, write_ext_updateinfo_filter, data, 0);

-    repo->start = oldstart;

-    repo->nsolvables += repoImpl->main_nsolvables;

-    return res;

-}

-

 static gboolean

 write_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata,

           const char *suffix, GError **error)

@@ -611,37 +631,78 @@ write_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata,

         FILE *fp = fdopen(tmp_fd, "w+");

         g_debug("%s: storing %s to: %s", __func__, repo->name, tmp_fn_templ);

-        if (which_repodata != _HY_REPODATA_UPDATEINFO)

-            ret |= repodata_write(data, fp);

-        else

-            ret |= write_ext_updateinfo(hrepo, data, fp);

-        ret |= checksum_write(repoImpl->checksum, fp);

-        ret |= fclose(fp);

+

+        SolvUserdata solv_userdata;

+        if (solv_userdata_fill(&solv_userdata, repoImpl->checksum, error)) {

+            fclose(fp);

+            success = FALSE;

+            goto done;

+        }

+

+        Repowriter *writer = repowriter_create(repo);

+        repowriter_set_userdata(writer, &solv_userdata, solv_userdata_size);

+        if (which_repodata != _HY_REPODATA_UPDATEINFO) {

+            repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);

+            repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE);

+            ret = repowriter_write(writer, fp);

+        } else {

+            // write only updateinfo repodata

+            int oldstart = repo->start;

+            repo->start = repoImpl->main_end;

+            repo->nsolvables -= repoImpl->main_nsolvables;

+            repowriter_set_flags(writer, REPOWRITER_LEGACY);

+            repowriter_set_keyfilter(writer, write_ext_updateinfo_filter, data);

+            repowriter_set_keyqueue(writer, 0);

+            ret = repowriter_write(writer, fp);

+            repo->start = oldstart;

+            repo->nsolvables += repoImpl->main_nsolvables;

+        }

+        repowriter_free(writer);

         if (ret) {

+            success = FALSE;

+            fclose(fp);

+            g_set_error (error,

+                        DNF_ERROR,

+                        DNF_ERROR_INTERNAL_ERROR,

+                        _("While writing extension cache %s (%d): repowriter write failed: %i, error: %s"),

+                        tmp_fn_templ, which_repodata, ret, pool_errstr(repo->pool));

+            goto done;

+        }

+

+        if (fclose(fp)) {

             success = FALSE;

             g_set_error (error,

                         DNF_ERROR,

-                        DNF_ERROR_FAILED,

-                        _("write_ext(%1$d) has failed: %2$d"),

-                        which_repodata, ret);

+                        DNF_ERROR_FILE_INVALID,

+                        _("While writing extension cache (%d): cannot close temporary file: %s"),

+                        which_repodata, tmp_fn_templ);

             goto done;

         }

     }

     if (repo_is_one_piece(repo) && which_repodata != _HY_REPODATA_UPDATEINFO) {

         /* switch over to written solv file activate paging */

-        FILE *fp = fopen(tmp_fn_templ, "r");

-        if (fp) {

-            int flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES;

-            /* do not pollute the main pool with directory component ids */

-            if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)

-                flags |= REPO_LOCALPOOL;

-            repodata_extend_block(data, repo->start, repo->end - repo->start);

-            data->state = REPODATA_LOADING;

-            repo_add_solv(repo, fp, flags);

-            data->state = REPODATA_AVAILABLE;

-            fclose(fp);

+        int flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES;

+        /* do not pollute the main pool with directory component ids */

+        if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)

+            flags |= REPO_LOCALPOOL;

+        repodata_extend_block(data, repo->start, repo->end - repo->start);

+        data->state = REPODATA_LOADING;

+        int loaded = try_to_use_cached_solvfile(tmp_fn_templ, repo, flags, repoImpl->checksum, error);

+        if (error && *error) {

+            g_prefix_error(error, _("Failed to use newly written extension cache: %s (%d): "),

+                           tmp_fn_templ, which_repodata);

+            success = FALSE;

+            goto done;

+        }

+        if (!loaded) {

+            g_set_error(error, DNF_ERROR, DNF_ERROR_INTERNAL_ERROR,

+                        _("Failed to use newly written extension cache: %s (%d)"), tmp_fn_templ, which_repodata);

+            success = FALSE;

+            goto done;

         }

+

+        data->state = REPODATA_AVAILABLE;

     }

     if (!mv(tmp_fn_templ, fn, error)) {

@@ -672,7 +733,7 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error)

     FILE *fp_primary = NULL;

     FILE *fp_repomd = NULL;

-    FILE *fp_cache = fopen(fn_cache, "r");

+

     if (!fn_repomd) {

         g_set_error (error,

                      DNF_ERROR,

@@ -693,18 +754,17 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error)

     }

     checksum_fp(repoImpl->checksum, fp_repomd);

-    if (can_use_repomd_cache(fp_cache, repoImpl->checksum)) {

+    if (try_to_use_cached_solvfile(fn_cache, repo, 0, repoImpl->checksum, error)) {

         const char *chksum = pool_checksum_str(pool, repoImpl->checksum);

         g_debug("using cached %s (0x%s)", name, chksum);

-        if (repo_add_solv(repo, fp_cache, 0)) {

-            g_set_error (error,

-                         DNF_ERROR,

-                         DNF_ERROR_INTERNAL_ERROR,

-                         _("repo_add_solv() has failed."));

-            retval = FALSE;

-            goto out;

-        }

         repoImpl->state_main = _HY_LOADED_CACHE;

+        goto out;

+    }

+

+    if (error && *error) {

+        g_prefix_error(error, _("While loading repository failed to use %s: "), fn_cache);

+        retval = FALSE;

+        goto out;

     } else {

         auto primary = hrepo->getMetadataPath(MD_TYPE_PRIMARY);

         if (primary.empty()) {

@@ -733,8 +793,6 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error)

         repoImpl->state_main = _HY_LOADED_FETCH;

     }

 out:

-    if (fp_cache)

-        fclose(fp_cache);

     if (fp_repomd)

         fclose(fp_repomd);

     if (fp_primary)

diff --git a/libdnf/hy-iutil-private.hpp b/libdnf/hy-iutil-private.hpp

index d498c032..efc91c63 100644

--- a/libdnf/hy-iutil-private.hpp

+++ b/libdnf/hy-iutil-private.hpp

@@ -52,9 +52,7 @@ int solv_userdata_verify(const SolvUserdata *solv_userdata, const unsigned char

 /* crypto utils */

 int checksum_cmp(const unsigned char *cs1, const unsigned char *cs2);

 int checksum_fp(unsigned char *out, FILE *fp);

-int checksum_read(unsigned char *csout, FILE *fp);

 int checksum_stat(unsigned char *out, FILE *fp);

-int checksum_write(const unsigned char *cs, FILE *fp);

 int checksumt_l2h(int type);

 const char *pool_checksum_str(Pool *pool, const unsigned char *chksum);

diff --git a/libdnf/hy-iutil.cpp b/libdnf/hy-iutil.cpp

index f81ca52f..c409a10a 100644

--- a/libdnf/hy-iutil.cpp

+++ b/libdnf/hy-iutil.cpp

@@ -142,17 +142,6 @@ checksum_fp(unsigned char *out, FILE *fp)

     return 0;

 }

-/* calls rewind(fp) before returning */

-int

-checksum_read(unsigned char *csout, FILE *fp)

-{

-    if (fseek(fp, -32, SEEK_END) ||

-        fread(csout, CHKSUM_BYTES, 1, fp) != 1)

-        return 1;

-    rewind(fp);

-    return 0;

-}

-

 /* does not move the fp position */

 int

 checksum_stat(unsigned char *out, FILE *fp)

@@ -174,15 +163,6 @@ checksum_stat(unsigned char *out, FILE *fp)

     return 0;

 }

-/* moves fp to the end of file */

-int checksum_write(const unsigned char *cs, FILE *fp)

-{

-    if (fseek(fp, 0, SEEK_END) ||

-        fwrite(cs, CHKSUM_BYTES, 1, fp) != 1)

-        return 1;

-    return 0;

-}

-

 static std::array<char, solv_userdata_solv_toolversion_size>

 get_padded_solv_toolversion()

 {

-- 

2.31.1