From 3cbb20f7263ec6221d60073c7e85aef610c7318b Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: May 10 2022 07:05:30 +0000 Subject: import createrepo_c-0.17.7-5.el8 --- diff --git a/.createrepo_c.metadata b/.createrepo_c.metadata index eac90b8..60dfac0 100644 --- a/.createrepo_c.metadata +++ b/.createrepo_c.metadata @@ -1 +1 @@ -9feb9bf8e6a9812a08ef8156c3ccc1c4dac90b7a SOURCES/createrepo_c-0.17.2.tar.gz +5f3e45a79255654c7315c8667197fd562fd2ff5c SOURCES/createrepo_c-0.17.7.tar.gz diff --git a/.gitignore b/.gitignore index fe40994..0df04bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -SOURCES/createrepo_c-0.17.2.tar.gz +SOURCES/createrepo_c-0.17.7.tar.gz diff --git a/SOURCES/0001-Fix-cr_compress_file_with_stat-Memory-leak.patch b/SOURCES/0001-Fix-cr_compress_file_with_stat-Memory-leak.patch deleted file mode 100644 index 126a489..0000000 --- a/SOURCES/0001-Fix-cr_compress_file_with_stat-Memory-leak.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 1ec1e767e4a48bc2c1d09b52da9e749352f8d925 Mon Sep 17 00:00:00 2001 -From: Jaroslav Rohel -Date: Thu, 6 May 2021 19:09:19 +0200 -Subject: [PATCH] Fix: cr_compress_file_with_stat: Memory leak - ---- - src/misc.c | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/misc.c b/src/misc.c -index e5350ac..59c04e6 100644 ---- a/src/misc.c -+++ b/src/misc.c -@@ -503,6 +503,8 @@ cr_compress_file_with_stat(const char *src, - if (!orig) { - ret = tmp_err->code; - g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", src); -+ if (dst != in_dst) -+ g_free(dst); - return ret; - } - --- -libgit2 1.0.1 - diff --git a/SOURCES/0001-Preserve-changed-API-for-cr_compress_file_with_stat-RhBug1973588.patch b/SOURCES/0001-Preserve-changed-API-for-cr_compress_file_with_stat-RhBug1973588.patch new file mode 100644 index 0000000..a1b74ea --- /dev/null +++ b/SOURCES/0001-Preserve-changed-API-for-cr_compress_file_with_stat-RhBug1973588.patch @@ -0,0 +1,384 @@ +From 6bcfaac228236ac3c609d014cbd23c3bd645bf18 Mon Sep 17 00:00:00 2001 +From: Aleš Matěj +Date: Thu, 9 Sep 2021 08:31:03 +0200 +Subject: [PATCH] Preserve changed API for cr_compress_file_with_stat (RhBug:1973588) + +In order to be compatible in rhel8 we want to preserve the old API and +behavior. + +Keep the fixed version as cr_compress_file_with_stat_v2 only for rhel8 + +https://bugzilla.redhat.com/show_bug.cgi?id=1973588 + +With fixed memory leak of `tmp_err`, reported here: +https://bugzilla.redhat.com/show_bug.cgi?id=2005781 +--- + src/misc.c | 139 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- + src/misc.h | 42 ++++++++++++++++++++++++++++++++++++++++-- + src/modifyrepo_shared.c | 4 ++-- + src/python/misc-py.c | 2 +- + src/threads.c | 14 +++++++------- + tests/test_misc.c | 34 +++++++++++++++++----------------- + 6 files changed, 205 insertions(+), 30 deletions(-) + +diff --git a/src/misc.c b/src/misc.c +index 4bd9f4c..c4b2cb3 100644 +--- a/src/misc.c ++++ b/src/misc.c +@@ -446,7 +446,7 @@ cr_copy_file(const char *src, const char *in_dst, GError **err) + + int + cr_compress_file_with_stat(const char *src, +- const char *in_dst, ++ char **in_dst, + cr_CompressionType compression, + cr_ContentStat *stat, + const char *zck_dict_dir, +@@ -458,6 +458,143 @@ cr_compress_file_with_stat(const char *src, + char buf[BUFFER_SIZE]; + CR_FILE *orig = NULL; + CR_FILE *new = NULL; ++ gchar *dst = (gchar *) *in_dst; ++ GError *tmp_err = NULL; ++ ++ assert(src); ++ assert(!err || *err == NULL); ++ ++ const char *c_suffix = cr_compression_suffix(compression); ++ ++ // Src must be a file NOT a directory ++ if (!g_file_test(src, G_FILE_TEST_IS_REGULAR)) { ++ g_debug("%s: Source (%s) must be a regular file!", __func__, src); ++ g_set_error(err, ERR_DOMAIN, CRE_NOFILE, ++ "Not a regular file: %s", src); ++ return CRE_NOFILE; ++ } ++ ++ if (!dst) { ++ // If destination is NULL, use src + compression suffix ++ *in_dst = g_strconcat(src, ++ c_suffix, ++ NULL); ++ } else if (g_str_has_suffix(dst, "/")) { ++ // If destination is dir use filename from src + compression suffix ++ *in_dst = g_strconcat(dst, ++ cr_get_filename(src), ++ c_suffix, ++ NULL); ++ } else if (c_suffix && !g_str_has_suffix(dst, c_suffix)) { ++ cr_CompressionType old_type = cr_detect_compression(src, &tmp_err); ++ if (tmp_err) { ++ g_debug("%s: Unable to detect compression type of %s", __func__, src); ++ g_clear_error(&tmp_err); ++ } else if (old_type != CR_CW_NO_COMPRESSION) { ++ _cleanup_free_ gchar *tmp_file = g_strndup(dst, strlen(dst) - strlen(cr_compression_suffix(old_type))); ++ *in_dst = g_strconcat(tmp_file, ++ c_suffix, ++ NULL); ++ } ++ } ++ if (dst != *in_dst && dst) ++ g_free(dst); ++ dst = (gchar *) *in_dst; ++ ++ int mode = CR_CW_AUTO_DETECT_COMPRESSION; ++ ++ orig = cr_open(src, ++ CR_CW_MODE_READ, ++ mode, ++ &tmp_err); ++ if (!orig) { ++ ret = tmp_err->code; ++ g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", src); ++ return ret; ++ } ++ ++ _cleanup_free_ gchar *dict = NULL; ++ size_t dict_size = 0; ++ if (compression == CR_CW_ZCK_COMPRESSION && zck_dict_dir) { ++ /* Find zdict */ ++ _cleanup_free_ gchar *file_basename = NULL; ++ if (dst) { ++ _cleanup_free_ gchar *dict_base = NULL; ++ if (g_str_has_suffix(dst, ".zck")) ++ dict_base = g_strndup(dst, strlen(dst)-4); ++ else ++ dict_base = g_strdup(dst); ++ file_basename = g_path_get_basename(dict_base); ++ } else { ++ file_basename = g_path_get_basename(src); ++ } ++ _cleanup_free_ gchar *dict_file = cr_get_dict_file(zck_dict_dir, file_basename); ++ ++ /* Read dictionary from file */ ++ if (dict_file && !g_file_get_contents(dict_file, &dict, ++ &dict_size, &tmp_err)) { ++ g_set_error(err, ERR_DOMAIN, CRE_IO, ++ "Error reading zchunk dict %s: %s", ++ dict_file, tmp_err->message); ++ g_clear_error(&tmp_err); ++ ret = CRE_IO; ++ goto compress_file_cleanup; ++ } ++ } ++ ++ new = cr_sopen(dst, CR_CW_MODE_WRITE, compression, stat, &tmp_err); ++ if (tmp_err) { ++ g_debug("%s: Cannot open destination file %s", __func__, dst); ++ g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", dst); ++ ret = CRE_IO; ++ goto compress_file_cleanup; ++ } ++ if (compression == CR_CW_ZCK_COMPRESSION) { ++ if (dict && cr_set_dict(new, dict, dict_size, &tmp_err) != CRE_OK) { ++ ret = tmp_err->code; ++ g_propagate_prefixed_error(err, tmp_err, "Unable to set zdict for %s: ", dst); ++ goto compress_file_cleanup; ++ } ++ if (zck_auto_chunk && cr_set_autochunk(new, TRUE, &tmp_err) != CRE_OK) { ++ ret = tmp_err->code; ++ g_propagate_prefixed_error(err, tmp_err, "Unable to set auto-chunking for %s: ", dst); ++ goto compress_file_cleanup; ++ } ++ } ++ while ((readed = cr_read(orig, buf, BUFFER_SIZE, &tmp_err)) > 0) { ++ cr_write(new, buf, readed, &tmp_err); ++ if (tmp_err) { ++ ret = tmp_err->code; ++ g_propagate_prefixed_error(err, tmp_err, "Unable to write to %s: ", dst); ++ goto compress_file_cleanup; ++ } ++ } ++ ++compress_file_cleanup: ++ ++ if (orig) ++ cr_close(orig, NULL); ++ ++ if (new) ++ cr_close(new, NULL); ++ ++ return ret; ++} ++ ++int ++cr_compress_file_with_stat_v2(const char *src, ++ const char *in_dst, ++ cr_CompressionType compression, ++ cr_ContentStat *stat, ++ const char *zck_dict_dir, ++ gboolean zck_auto_chunk, ++ GError **err) ++{ ++ int ret = CRE_OK; ++ int readed; ++ char buf[BUFFER_SIZE]; ++ CR_FILE *orig = NULL; ++ CR_FILE *new = NULL; + gchar *dst = (gchar *) in_dst; + GError *tmp_err = NULL; + +diff --git a/src/misc.h b/src/misc.h +index 60f1a0f..528ccc3 100644 +--- a/src/misc.h ++++ b/src/misc.h +@@ -184,9 +184,24 @@ gboolean cr_copy_file(const char *src, + cr_compress_file_with_stat(SRC, DST, COMTYPE, NULL, ZCK_DICT_DIR, \ + ZCK_AUTO_CHUNK, ERR) + ++/** Compress file. This function is temporary and present ++ * only in rhel 8, it will be removed in future versions. ++ * @param SRC source filename ++ * @param DST destination (If dst is dir, filename of src + ++ * compression suffix is used. ++ * If dst is NULL, src + compression suffix is used) ++ * @param COMTYPE type of compression ++ * @param ZCK_DICT_DIR Location of zchunk zdicts (if zchunk is enabled) ++ * @param ZCK_AUTO_CHUNK Whether zchunk file should be auto-chunked ++ * @param ERR GError ** ++ * @return cr_Error return code ++ */ ++#define cr_compress_file_v2(SRC, DST, COMTYPE, ZCK_DICT_DIR, ZCK_AUTO_CHUNK, ERR) \ ++ cr_compress_file_with_stat_v2(SRC, DST, COMTYPE, NULL, ZCK_DICT_DIR, \ ++ ZCK_AUTO_CHUNK, ERR) + /** Compress file. + * @param src source filename +- * @param dst destination (If dst is dir, filename of src + ++ * @param dst pointer to destination (If dst is dir, filename of src + + * compression suffix is used. + * If dst is NULL, src + compression suffix is used) + * @param comtype type of compression +@@ -197,13 +212,36 @@ gboolean cr_copy_file(const char *src, + * @return cr_Error return code + */ + int cr_compress_file_with_stat(const char *src, +- const char *dst, ++ char **dst, + cr_CompressionType comtype, + cr_ContentStat *stat, + const char *zck_dict_dir, + gboolean zck_auto_chunk, + GError **err); + ++/** Compress file with stat versions 2. This function is temporary and present ++ * only in rhel 8, it will be removed in future versions. ++ * It is a compatibility function that preserves the API and behavior of ++ * cr_compress_file_with_stat from createrepo_c-0.12.0. ++ * @param src source filename ++ * @param dst destination (If dst is dir, filename of src + ++ * compression suffix is used. ++ * If dst is NULL, src + compression suffix is used) ++ * @param comtype type of compression ++ * @param stat pointer to cr_ContentStat or NULL ++ * @param zck_dict_dir Location of zchunk zdicts (if zchunk is enabled) ++ * @param zck_auto_chunk Whether zchunk file should be auto-chunked ++ * @param err GError ** ++ * @return cr_Error return code ++ */ ++int cr_compress_file_with_stat_v2(const char *src, ++ const char *dst, ++ cr_CompressionType comtype, ++ cr_ContentStat *stat, ++ const char *zck_dict_dir, ++ gboolean zck_auto_chunk, ++ GError **err); ++ + /** Decompress file. + * @param SRC source filename + * @param DST destination (If dst is dir, filename of src without +diff --git a/src/modifyrepo_shared.c b/src/modifyrepo_shared.c +index 4e59660..8cf246d 100644 +--- a/src/modifyrepo_shared.c ++++ b/src/modifyrepo_shared.c +@@ -120,8 +120,8 @@ cr_write_file(gchar *repopath, cr_ModifyRepoTask *task, + g_debug("%s: Copy & compress operation %s -> %s", + __func__, src_fn, dst_fn); + +- if (cr_compress_file(src_fn, dst_fn, compress_type, +- task->zck_dict_dir, TRUE, err) != CRE_OK) { ++ if (cr_compress_file_v2(src_fn, dst_fn, compress_type, ++ task->zck_dict_dir, TRUE, err) != CRE_OK) { + g_debug("%s: Copy & compress operation failed", __func__); + return NULL; + } +diff --git a/src/python/misc-py.c b/src/python/misc-py.c +index 6a7871e..cc28448 100644 +--- a/src/python/misc-py.c ++++ b/src/python/misc-py.c +@@ -49,7 +49,7 @@ py_compress_file_with_stat(G_GNUC_UNUSED PyObject *self, PyObject *args) + return NULL; + } + +- cr_compress_file_with_stat(src, dst, type, contentstat, NULL, FALSE, &tmp_err); ++ cr_compress_file_with_stat_v2(src, dst, type, contentstat, NULL, FALSE, &tmp_err); + if (tmp_err) { + nice_exception(&tmp_err, NULL); + return NULL; +diff --git a/src/threads.c b/src/threads.c +index f0c3f93..b529d55 100644 +--- a/src/threads.c ++++ b/src/threads.c +@@ -101,13 +101,13 @@ cr_compressing_thread(gpointer data, G_GNUC_UNUSED gpointer user_data) + cr_compression_suffix(task->type), + NULL); + +- cr_compress_file_with_stat(task->src, +- task->dst, +- task->type, +- task->stat, +- task->zck_dict_dir, +- task->zck_auto_chunk, +- &tmp_err); ++ cr_compress_file_with_stat_v2(task->src, ++ task->dst, ++ task->type, ++ task->stat, ++ task->zck_dict_dir, ++ task->zck_auto_chunk, ++ &tmp_err); + + if (tmp_err) { + // Error encountered +diff --git a/tests/test_misc.c b/tests/test_misc.c +index 6614809..1acccb7 100644 +--- a/tests/test_misc.c ++++ b/tests/test_misc.c +@@ -548,8 +548,8 @@ compressfile_test_text_file(Copyfiletest *copyfiletest, + GError *tmp_err = NULL; + + g_assert(!g_file_test(copyfiletest->dst_file, G_FILE_TEST_EXISTS)); +- ret = cr_compress_file(TEST_TEXT_FILE, copyfiletest->dst_file, +- CR_CW_GZ_COMPRESSION, NULL, FALSE, &tmp_err); ++ ret = cr_compress_file_v2(TEST_TEXT_FILE, copyfiletest->dst_file, ++ CR_CW_GZ_COMPRESSION, NULL, FALSE, &tmp_err); + g_assert(!tmp_err); + g_assert_cmpint(ret, ==, CRE_OK); + g_assert(g_file_test(copyfiletest->dst_file, G_FILE_TEST_IS_REGULAR)); +@@ -574,9 +574,9 @@ compressfile_with_stat_test_text_file(Copyfiletest *copyfiletest, + g_assert(!tmp_err); + + g_assert(!g_file_test(copyfiletest->dst_file, G_FILE_TEST_EXISTS)); +- ret = cr_compress_file_with_stat(TEST_TEXT_FILE, copyfiletest->dst_file, +- CR_CW_GZ_COMPRESSION, stat, NULL, FALSE, +- &tmp_err); ++ ret = cr_compress_file_with_stat_v2(TEST_TEXT_FILE, copyfiletest->dst_file, ++ CR_CW_GZ_COMPRESSION, stat, NULL, FALSE, ++ &tmp_err); + g_assert(!tmp_err); + g_assert_cmpint(ret, ==, CRE_OK); + g_assert(g_file_test(copyfiletest->dst_file, G_FILE_TEST_IS_REGULAR)); +@@ -603,9 +603,9 @@ compressfile_with_stat_test_gz_file_gz_output(Copyfiletest *copyfiletest, + char * dst_full_name = g_strconcat(copyfiletest->dst_file, ".gz", NULL); + + g_assert(!g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); +- ret = cr_compress_file_with_stat(TEST_TEXT_FILE_GZ, dst_full_name, +- CR_CW_GZ_COMPRESSION, stat, NULL, FALSE, +- &tmp_err); ++ ret = cr_compress_file_with_stat_v2(TEST_TEXT_FILE_GZ, dst_full_name, ++ CR_CW_GZ_COMPRESSION, stat, NULL, FALSE, ++ &tmp_err); + g_assert(!tmp_err); + g_assert_cmpint(ret, ==, CRE_OK); + g_assert(g_file_test(dst_full_name, G_FILE_TEST_IS_REGULAR)); +@@ -633,9 +633,9 @@ compressfile_test_gz_file_xz_output(Copyfiletest *copyfiletest, + char * dst_full_name = g_strconcat(copyfiletest->dst_file, ".xz", NULL); + + g_assert(!g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); +- ret = cr_compress_file(TEST_TEXT_FILE_GZ, dst_full_name, +- CR_CW_XZ_COMPRESSION, NULL, FALSE, +- &tmp_err); ++ ret = cr_compress_file_v2(TEST_TEXT_FILE_GZ, dst_full_name, ++ CR_CW_XZ_COMPRESSION, NULL, FALSE, ++ &tmp_err); + g_assert(!tmp_err); + g_assert_cmpint(ret, ==, CRE_OK); + g_assert(g_file_test(dst_full_name, G_FILE_TEST_IS_REGULAR)); +@@ -660,9 +660,9 @@ compressfile_test_xz_file_gz_output(Copyfiletest *copyfiletest, + char * dst_full_name = g_strconcat(copyfiletest->dst_file, ".gz", NULL); + + g_assert(!g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); +- ret = cr_compress_file(TEST_TEXT_FILE_XZ, dst_full_name, +- CR_CW_GZ_COMPRESSION, NULL, FALSE, +- &tmp_err); ++ ret = cr_compress_file_v2(TEST_TEXT_FILE_XZ, dst_full_name, ++ CR_CW_GZ_COMPRESSION, NULL, FALSE, ++ &tmp_err); + g_assert(!tmp_err); + g_assert_cmpint(ret, ==, CRE_OK); + g_assert(g_file_test(dst_full_name, G_FILE_TEST_IS_REGULAR)); +@@ -687,9 +687,9 @@ compressfile_test_sqlite_file_gz_output(Copyfiletest *copyfiletest, + char * dst_full_name = g_strconcat(copyfiletest->dst_file, ".gz", NULL); + + g_assert(!g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); +- ret = cr_compress_file(TEST_SQLITE_FILE, dst_full_name, +- CR_CW_GZ_COMPRESSION, NULL, FALSE, +- &tmp_err); ++ ret = cr_compress_file_v2(TEST_SQLITE_FILE, dst_full_name, ++ CR_CW_GZ_COMPRESSION, NULL, FALSE, ++ &tmp_err); + g_assert(!tmp_err); + g_assert_cmpint(ret, ==, CRE_OK); + g_assert(g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); +-- +libgit2 1.1.0 + diff --git a/SOURCES/0002-Default---keep-all-metadata-to-TRUE-and-add---discard-additional-metadata.patch b/SOURCES/0002-Default---keep-all-metadata-to-TRUE-and-add---discard-additional-metadata.patch new file mode 100644 index 0000000..b45969c --- /dev/null +++ b/SOURCES/0002-Default---keep-all-metadata-to-TRUE-and-add---discard-additional-metadata.patch @@ -0,0 +1,94 @@ +From b5f425fec738c1de344f4f917d3614e9efb98e2b Mon Sep 17 00:00:00 2001 +From: Aleš Matěj +Date: Thu, 23 Sep 2021 08:30:47 +0200 +Subject: [PATCH] Default --keep-all-metadata to TRUE and add --discard-additional-metadata + += changelog = +msg: Switch default of --keep-all-metadata to TRUE and add --discard-additional-metadata +type: enhancement +resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1992209 +--- + doc/createrepo_c.8 | 7 +++++-- + src/cmd_parser.c | 15 ++++++++++++--- + src/cmd_parser.h | 1 + + 3 files changed, 18 insertions(+), 5 deletions(-) + +diff --git a/doc/createrepo_c.8 b/doc/createrepo_c.8 +index c9017c5..1a3e101 100644 +--- a/doc/createrepo_c.8 ++++ b/doc/createrepo_c.8 +@@ -1,6 +1,6 @@ + .\" Man page generated from reStructuredText. + . +-.TH CREATEREPO_C 8 "2020-07-02" "" "" ++.TH CREATEREPO_C 8 "2021-09-23" "" "" + .SH NAME + createrepo_c \- Create rpm-md format (xml-rpm-metadata) repository + . +@@ -173,7 +173,10 @@ Generate zchunk files as well as the standard repodata. + Directory containing compression dictionaries for use by zchunk + .SS \-\-keep\-all\-metadata + .sp +-Keep all additional metadata (not primary, filelists and other xml or sqlite files, nor their compressed variants) from source repository during update. ++Keep all additional metadata (not primary, filelists and other xml or sqlite files, nor their compressed variants) from source repository during update (default). ++.SS \-\-discard\-additional\-metadata ++.sp ++Discard all additional metadata (not primary, filelists and other xml or sqlite files, nor their compressed variants) from source repository during update. + .SS \-\-compatibility + .sp + Enforce maximal compatibility with classical createrepo (Affects only: \-\-retain\-old\-md). +diff --git a/src/cmd_parser.c b/src/cmd_parser.c +index bbefa08..639d7e9 100644 +--- a/src/cmd_parser.c ++++ b/src/cmd_parser.c +@@ -65,6 +65,8 @@ struct CmdOptions _cmd_options = { + .zck_compression = FALSE, + .zck_dict_dir = NULL, + .recycle_pkglist = FALSE, ++ ++ .keep_all_metadata = TRUE, + }; + + +@@ -168,6 +170,9 @@ static GOptionEntry cmd_entries[] = + #endif + { "keep-all-metadata", 0, 0, G_OPTION_ARG_NONE, &(_cmd_options.keep_all_metadata), + "Keep all additional metadata (not primary, filelists and other xml or sqlite files, " ++ "nor their compressed variants) from source repository during update (default).", NULL }, ++ { "discard-additional-metadata", 0, 0, G_OPTION_ARG_NONE, &(_cmd_options.discard_additional_metadata), ++ "Discard all additional metadata (not primary, filelists and other xml or sqlite files, " + "nor their compressed variants) from source repository during update.", NULL }, + { "compatibility", 0, 0, G_OPTION_ARG_NONE, &(_cmd_options.compatibility), + "Enforce maximal compatibility with classical createrepo (Affects only: --retain-old-md).", NULL }, +@@ -510,9 +515,13 @@ check_arguments(struct CmdOptions *options, + x++; + } + +- // Check keep-all-metadata +- if (options->keep_all_metadata && !options->update) { +- g_warning("--keep-all-metadata has no effect (--update is not used)"); ++ if (options->discard_additional_metadata) { ++ options->keep_all_metadata = FALSE; ++ } ++ ++ // Check discard-additional-metadata ++ if (options->discard_additional_metadata && !options->update) { ++ g_warning("--discard-additional-metadata has no effect (--update is not used)"); + } + + // Process --distro tags +diff --git a/src/cmd_parser.h b/src/cmd_parser.h +index 32bcf99..03cfcf0 100644 +--- a/src/cmd_parser.h ++++ b/src/cmd_parser.h +@@ -77,6 +77,7 @@ struct CmdOptions { + char *zck_dict_dir; /*!< directory with zchunk dictionaries */ + gboolean keep_all_metadata; /*!< keep groupfile and updateinfo from source + repo during update */ ++ gboolean discard_additional_metadata; /*!< Inverse option to keep_all_metadata */ + gboolean ignore_lock; /*!< Ignore existing .repodata/ - remove it, + create the new one (empty) to serve as + a lock and use a .repodata.date.pid for +-- +libgit2 1.1.0 + diff --git a/SOURCES/0002-Preserve-changed-API-for-cr_compress_file_with_stat-RhBug1973588.patch b/SOURCES/0002-Preserve-changed-API-for-cr_compress_file_with_stat-RhBug1973588.patch deleted file mode 100644 index d14a0ae..0000000 --- a/SOURCES/0002-Preserve-changed-API-for-cr_compress_file_with_stat-RhBug1973588.patch +++ /dev/null @@ -1,380 +0,0 @@ -From a611bb0c3729713af28031cdf1d056823d663db1 Mon Sep 17 00:00:00 2001 -From: Aleš Matěj -Date: Thu, 9 Sep 2021 08:31:03 +0200 -Subject: [PATCH] Preserve changed API for cr_compress_file_with_stat (RhBug:1973588) - -In order to be compatible in rhel8 we want to preserve the old API and -behavior. - -Keep the fixed version as cr_compress_file_with_stat_v2 only for rhel8 - -https://bugzilla.redhat.com/show_bug.cgi?id=1973588 ---- - src/misc.c | 138 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- - src/misc.h | 42 ++++++++++++++++++++++++++++++++++++++++-- - src/modifyrepo_shared.c | 4 ++-- - src/python/misc-py.c | 2 +- - src/threads.c | 14 +++++++------- - tests/test_misc.c | 34 +++++++++++++++++----------------- - 6 files changed, 204 insertions(+), 30 deletions(-) - -diff --git a/src/misc.c b/src/misc.c -index 59c04e6..b3a218c 100644 ---- a/src/misc.c -+++ b/src/misc.c -@@ -446,7 +446,7 @@ cr_copy_file(const char *src, const char *in_dst, GError **err) - - int - cr_compress_file_with_stat(const char *src, -- const char *in_dst, -+ char **in_dst, - cr_CompressionType compression, - cr_ContentStat *stat, - const char *zck_dict_dir, -@@ -458,6 +458,142 @@ cr_compress_file_with_stat(const char *src, - char buf[BUFFER_SIZE]; - CR_FILE *orig = NULL; - CR_FILE *new = NULL; -+ gchar *dst = (gchar *) *in_dst; -+ GError *tmp_err = NULL; -+ -+ assert(src); -+ assert(!err || *err == NULL); -+ -+ const char *c_suffix = cr_compression_suffix(compression); -+ -+ // Src must be a file NOT a directory -+ if (!g_file_test(src, G_FILE_TEST_IS_REGULAR)) { -+ g_debug("%s: Source (%s) must be a regular file!", __func__, src); -+ g_set_error(err, ERR_DOMAIN, CRE_NOFILE, -+ "Not a regular file: %s", src); -+ return CRE_NOFILE; -+ } -+ -+ if (!dst) { -+ // If destination is NULL, use src + compression suffix -+ *in_dst = g_strconcat(src, -+ c_suffix, -+ NULL); -+ } else if (g_str_has_suffix(dst, "/")) { -+ // If destination is dir use filename from src + compression suffix -+ *in_dst = g_strconcat(dst, -+ cr_get_filename(src), -+ c_suffix, -+ NULL); -+ } else if (c_suffix && !g_str_has_suffix(dst, c_suffix)) { -+ cr_CompressionType old_type = cr_detect_compression(src, &tmp_err); -+ if (tmp_err) { -+ g_debug("%s: Unable to detect compression type of %s", __func__, src); -+ g_clear_error(&tmp_err); -+ } else if (old_type != CR_CW_NO_COMPRESSION) { -+ _cleanup_free_ gchar *tmp_file = g_strndup(dst, strlen(dst) - strlen(cr_compression_suffix(old_type))); -+ *in_dst = g_strconcat(tmp_file, -+ c_suffix, -+ NULL); -+ } -+ } -+ if (dst != *in_dst && dst) -+ g_free(dst); -+ dst = (gchar *) *in_dst; -+ -+ int mode = CR_CW_AUTO_DETECT_COMPRESSION; -+ -+ orig = cr_open(src, -+ CR_CW_MODE_READ, -+ mode, -+ &tmp_err); -+ if (!orig) { -+ ret = tmp_err->code; -+ g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", src); -+ return ret; -+ } -+ -+ _cleanup_free_ gchar *dict = NULL; -+ size_t dict_size = 0; -+ if (compression == CR_CW_ZCK_COMPRESSION && zck_dict_dir) { -+ /* Find zdict */ -+ _cleanup_free_ gchar *file_basename = NULL; -+ if (dst) { -+ _cleanup_free_ gchar *dict_base = NULL; -+ if (g_str_has_suffix(dst, ".zck")) -+ dict_base = g_strndup(dst, strlen(dst)-4); -+ else -+ dict_base = g_strdup(dst); -+ file_basename = g_path_get_basename(dict_base); -+ } else { -+ file_basename = g_path_get_basename(src); -+ } -+ _cleanup_free_ gchar *dict_file = cr_get_dict_file(zck_dict_dir, file_basename); -+ -+ /* Read dictionary from file */ -+ if (dict_file && !g_file_get_contents(dict_file, &dict, -+ &dict_size, &tmp_err)) { -+ g_set_error(err, ERR_DOMAIN, CRE_IO, -+ "Error reading zchunk dict %s: %s", -+ dict_file, tmp_err->message); -+ ret = CRE_IO; -+ goto compress_file_cleanup; -+ } -+ } -+ -+ new = cr_sopen(dst, CR_CW_MODE_WRITE, compression, stat, &tmp_err); -+ if (tmp_err) { -+ g_debug("%s: Cannot open destination file %s", __func__, dst); -+ g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", dst); -+ ret = CRE_IO; -+ goto compress_file_cleanup; -+ } -+ if (compression == CR_CW_ZCK_COMPRESSION) { -+ if (dict && cr_set_dict(new, dict, dict_size, &tmp_err) != CRE_OK) { -+ ret = tmp_err->code; -+ g_propagate_prefixed_error(err, tmp_err, "Unable to set zdict for %s: ", dst); -+ goto compress_file_cleanup; -+ } -+ if (zck_auto_chunk && cr_set_autochunk(new, TRUE, &tmp_err) != CRE_OK) { -+ ret = tmp_err->code; -+ g_propagate_prefixed_error(err, tmp_err, "Unable to set auto-chunking for %s: ", dst); -+ goto compress_file_cleanup; -+ } -+ } -+ while ((readed = cr_read(orig, buf, BUFFER_SIZE, &tmp_err)) > 0) { -+ cr_write(new, buf, readed, &tmp_err); -+ if (tmp_err) { -+ ret = tmp_err->code; -+ g_propagate_prefixed_error(err, tmp_err, "Unable to write to %s: ", dst); -+ goto compress_file_cleanup; -+ } -+ } -+ -+compress_file_cleanup: -+ -+ if (orig) -+ cr_close(orig, NULL); -+ -+ if (new) -+ cr_close(new, NULL); -+ -+ return ret; -+} -+ -+int -+cr_compress_file_with_stat_v2(const char *src, -+ const char *in_dst, -+ cr_CompressionType compression, -+ cr_ContentStat *stat, -+ const char *zck_dict_dir, -+ gboolean zck_auto_chunk, -+ GError **err) -+{ -+ int ret = CRE_OK; -+ int readed; -+ char buf[BUFFER_SIZE]; -+ CR_FILE *orig = NULL; -+ CR_FILE *new = NULL; - gchar *dst = (gchar *) in_dst; - GError *tmp_err = NULL; - -diff --git a/src/misc.h b/src/misc.h -index 60f1a0f..528ccc3 100644 ---- a/src/misc.h -+++ b/src/misc.h -@@ -184,9 +184,24 @@ gboolean cr_copy_file(const char *src, - cr_compress_file_with_stat(SRC, DST, COMTYPE, NULL, ZCK_DICT_DIR, \ - ZCK_AUTO_CHUNK, ERR) - -+/** Compress file. This function is temporary and present -+ * only in rhel 8, it will be removed in future versions. -+ * @param SRC source filename -+ * @param DST destination (If dst is dir, filename of src + -+ * compression suffix is used. -+ * If dst is NULL, src + compression suffix is used) -+ * @param COMTYPE type of compression -+ * @param ZCK_DICT_DIR Location of zchunk zdicts (if zchunk is enabled) -+ * @param ZCK_AUTO_CHUNK Whether zchunk file should be auto-chunked -+ * @param ERR GError ** -+ * @return cr_Error return code -+ */ -+#define cr_compress_file_v2(SRC, DST, COMTYPE, ZCK_DICT_DIR, ZCK_AUTO_CHUNK, ERR) \ -+ cr_compress_file_with_stat_v2(SRC, DST, COMTYPE, NULL, ZCK_DICT_DIR, \ -+ ZCK_AUTO_CHUNK, ERR) - /** Compress file. - * @param src source filename -- * @param dst destination (If dst is dir, filename of src + -+ * @param dst pointer to destination (If dst is dir, filename of src + - * compression suffix is used. - * If dst is NULL, src + compression suffix is used) - * @param comtype type of compression -@@ -197,13 +212,36 @@ gboolean cr_copy_file(const char *src, - * @return cr_Error return code - */ - int cr_compress_file_with_stat(const char *src, -- const char *dst, -+ char **dst, - cr_CompressionType comtype, - cr_ContentStat *stat, - const char *zck_dict_dir, - gboolean zck_auto_chunk, - GError **err); - -+/** Compress file with stat versions 2. This function is temporary and present -+ * only in rhel 8, it will be removed in future versions. -+ * It is a compatibility function that preserves the API and behavior of -+ * cr_compress_file_with_stat from createrepo_c-0.12.0. -+ * @param src source filename -+ * @param dst destination (If dst is dir, filename of src + -+ * compression suffix is used. -+ * If dst is NULL, src + compression suffix is used) -+ * @param comtype type of compression -+ * @param stat pointer to cr_ContentStat or NULL -+ * @param zck_dict_dir Location of zchunk zdicts (if zchunk is enabled) -+ * @param zck_auto_chunk Whether zchunk file should be auto-chunked -+ * @param err GError ** -+ * @return cr_Error return code -+ */ -+int cr_compress_file_with_stat_v2(const char *src, -+ const char *dst, -+ cr_CompressionType comtype, -+ cr_ContentStat *stat, -+ const char *zck_dict_dir, -+ gboolean zck_auto_chunk, -+ GError **err); -+ - /** Decompress file. - * @param SRC source filename - * @param DST destination (If dst is dir, filename of src without -diff --git a/src/modifyrepo_shared.c b/src/modifyrepo_shared.c -index 4e59660..8cf246d 100644 ---- a/src/modifyrepo_shared.c -+++ b/src/modifyrepo_shared.c -@@ -120,8 +120,8 @@ cr_write_file(gchar *repopath, cr_ModifyRepoTask *task, - g_debug("%s: Copy & compress operation %s -> %s", - __func__, src_fn, dst_fn); - -- if (cr_compress_file(src_fn, dst_fn, compress_type, -- task->zck_dict_dir, TRUE, err) != CRE_OK) { -+ if (cr_compress_file_v2(src_fn, dst_fn, compress_type, -+ task->zck_dict_dir, TRUE, err) != CRE_OK) { - g_debug("%s: Copy & compress operation failed", __func__); - return NULL; - } -diff --git a/src/python/misc-py.c b/src/python/misc-py.c -index 6a7871e..cc28448 100644 ---- a/src/python/misc-py.c -+++ b/src/python/misc-py.c -@@ -49,7 +49,7 @@ py_compress_file_with_stat(G_GNUC_UNUSED PyObject *self, PyObject *args) - return NULL; - } - -- cr_compress_file_with_stat(src, dst, type, contentstat, NULL, FALSE, &tmp_err); -+ cr_compress_file_with_stat_v2(src, dst, type, contentstat, NULL, FALSE, &tmp_err); - if (tmp_err) { - nice_exception(&tmp_err, NULL); - return NULL; -diff --git a/src/threads.c b/src/threads.c -index 9ef839d..91b5430 100644 ---- a/src/threads.c -+++ b/src/threads.c -@@ -100,13 +100,13 @@ cr_compressing_thread(gpointer data, G_GNUC_UNUSED gpointer user_data) - cr_compression_suffix(task->type), - NULL); - -- cr_compress_file_with_stat(task->src, -- task->dst, -- task->type, -- task->stat, -- task->zck_dict_dir, -- task->zck_auto_chunk, -- &tmp_err); -+ cr_compress_file_with_stat_v2(task->src, -+ task->dst, -+ task->type, -+ task->stat, -+ task->zck_dict_dir, -+ task->zck_auto_chunk, -+ &tmp_err); - - if (tmp_err) { - // Error encountered -diff --git a/tests/test_misc.c b/tests/test_misc.c -index 6614809..1acccb7 100644 ---- a/tests/test_misc.c -+++ b/tests/test_misc.c -@@ -548,8 +548,8 @@ compressfile_test_text_file(Copyfiletest *copyfiletest, - GError *tmp_err = NULL; - - g_assert(!g_file_test(copyfiletest->dst_file, G_FILE_TEST_EXISTS)); -- ret = cr_compress_file(TEST_TEXT_FILE, copyfiletest->dst_file, -- CR_CW_GZ_COMPRESSION, NULL, FALSE, &tmp_err); -+ ret = cr_compress_file_v2(TEST_TEXT_FILE, copyfiletest->dst_file, -+ CR_CW_GZ_COMPRESSION, NULL, FALSE, &tmp_err); - g_assert(!tmp_err); - g_assert_cmpint(ret, ==, CRE_OK); - g_assert(g_file_test(copyfiletest->dst_file, G_FILE_TEST_IS_REGULAR)); -@@ -574,9 +574,9 @@ compressfile_with_stat_test_text_file(Copyfiletest *copyfiletest, - g_assert(!tmp_err); - - g_assert(!g_file_test(copyfiletest->dst_file, G_FILE_TEST_EXISTS)); -- ret = cr_compress_file_with_stat(TEST_TEXT_FILE, copyfiletest->dst_file, -- CR_CW_GZ_COMPRESSION, stat, NULL, FALSE, -- &tmp_err); -+ ret = cr_compress_file_with_stat_v2(TEST_TEXT_FILE, copyfiletest->dst_file, -+ CR_CW_GZ_COMPRESSION, stat, NULL, FALSE, -+ &tmp_err); - g_assert(!tmp_err); - g_assert_cmpint(ret, ==, CRE_OK); - g_assert(g_file_test(copyfiletest->dst_file, G_FILE_TEST_IS_REGULAR)); -@@ -603,9 +603,9 @@ compressfile_with_stat_test_gz_file_gz_output(Copyfiletest *copyfiletest, - char * dst_full_name = g_strconcat(copyfiletest->dst_file, ".gz", NULL); - - g_assert(!g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); -- ret = cr_compress_file_with_stat(TEST_TEXT_FILE_GZ, dst_full_name, -- CR_CW_GZ_COMPRESSION, stat, NULL, FALSE, -- &tmp_err); -+ ret = cr_compress_file_with_stat_v2(TEST_TEXT_FILE_GZ, dst_full_name, -+ CR_CW_GZ_COMPRESSION, stat, NULL, FALSE, -+ &tmp_err); - g_assert(!tmp_err); - g_assert_cmpint(ret, ==, CRE_OK); - g_assert(g_file_test(dst_full_name, G_FILE_TEST_IS_REGULAR)); -@@ -633,9 +633,9 @@ compressfile_test_gz_file_xz_output(Copyfiletest *copyfiletest, - char * dst_full_name = g_strconcat(copyfiletest->dst_file, ".xz", NULL); - - g_assert(!g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); -- ret = cr_compress_file(TEST_TEXT_FILE_GZ, dst_full_name, -- CR_CW_XZ_COMPRESSION, NULL, FALSE, -- &tmp_err); -+ ret = cr_compress_file_v2(TEST_TEXT_FILE_GZ, dst_full_name, -+ CR_CW_XZ_COMPRESSION, NULL, FALSE, -+ &tmp_err); - g_assert(!tmp_err); - g_assert_cmpint(ret, ==, CRE_OK); - g_assert(g_file_test(dst_full_name, G_FILE_TEST_IS_REGULAR)); -@@ -660,9 +660,9 @@ compressfile_test_xz_file_gz_output(Copyfiletest *copyfiletest, - char * dst_full_name = g_strconcat(copyfiletest->dst_file, ".gz", NULL); - - g_assert(!g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); -- ret = cr_compress_file(TEST_TEXT_FILE_XZ, dst_full_name, -- CR_CW_GZ_COMPRESSION, NULL, FALSE, -- &tmp_err); -+ ret = cr_compress_file_v2(TEST_TEXT_FILE_XZ, dst_full_name, -+ CR_CW_GZ_COMPRESSION, NULL, FALSE, -+ &tmp_err); - g_assert(!tmp_err); - g_assert_cmpint(ret, ==, CRE_OK); - g_assert(g_file_test(dst_full_name, G_FILE_TEST_IS_REGULAR)); -@@ -687,9 +687,9 @@ compressfile_test_sqlite_file_gz_output(Copyfiletest *copyfiletest, - char * dst_full_name = g_strconcat(copyfiletest->dst_file, ".gz", NULL); - - g_assert(!g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); -- ret = cr_compress_file(TEST_SQLITE_FILE, dst_full_name, -- CR_CW_GZ_COMPRESSION, NULL, FALSE, -- &tmp_err); -+ ret = cr_compress_file_v2(TEST_SQLITE_FILE, dst_full_name, -+ CR_CW_GZ_COMPRESSION, NULL, FALSE, -+ &tmp_err); - g_assert(!tmp_err); - g_assert_cmpint(ret, ==, CRE_OK); - g_assert(g_file_test(dst_full_name, G_FILE_TEST_EXISTS)); --- -libgit2 1.0.1 - diff --git a/SOURCES/0003-Revert-added-API-for-parsing-main-metadata-together-RhBug2062299.patch b/SOURCES/0003-Revert-added-API-for-parsing-main-metadata-together-RhBug2062299.patch new file mode 100644 index 0000000..35858b5 --- /dev/null +++ b/SOURCES/0003-Revert-added-API-for-parsing-main-metadata-together-RhBug2062299.patch @@ -0,0 +1,2640 @@ +From 9f0762b90cc1d85900064c24fad8f7d5a8b73afe Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:23:29 +0100 +Subject: [PATCH 1/5] Revert "Add xml_parse_main_metadata_together to python + parser examples" + +This reverts commit 0633e31e491179f0153ef8267874812b4691957b. +--- + examples/python/repodata_parsing.py | 46 ----------------------------- + 1 file changed, 46 deletions(-) + +diff --git a/examples/python/repodata_parsing.py b/examples/python/repodata_parsing.py +index 4519518..ea517a1 100755 +--- a/examples/python/repodata_parsing.py ++++ b/examples/python/repodata_parsing.py +@@ -211,48 +211,6 @@ def second_method(): + for pkg in packages.values(): + print_package_info(pkg) + +-def third_method(): +- """Parsing main metadata types (primary, filelists, other) at the same time. +- This approach significantly reduces memory footprint because we don't need +- to keep all the packages in memory, user can handle them one by one. +- +- The API reflects xml_parse_primary/filelists/other except that it handles +- all of them at the same time. +- +- """ +- def warningcb(warning_type, message): +- print("PARSER WARNING: %s" % message) +- return True +- +- repomd = cr.Repomd() +- cr.xml_parse_repomd(os.path.join(REPO_PATH, "repodata/repomd.xml"), repomd, warningcb) +- +- primary_xml_path = None +- filelists_xml_path = None +- other_xml_path = None +- for record in repomd.records: +- if record.type == "primary": +- primary_xml_path = os.path.join(REPO_PATH, record.location_href) +- elif record.type == "filelists": +- filelists_xml_path = os.path.join(REPO_PATH, record.location_href) +- elif record.type == "other": +- other_xml_path = os.path.join(REPO_PATH, record.location_href) +- +- # +- # Main XML metadata parsing (primary, filelists, other) +- # +- +- def pkgcb(pkg): +- # Called when whole package entry from all 3 metadata xml files is parsed +- print_package_info(pkg) +- +- cr.xml_parse_main_metadata_together(primary_xml_path, +- filelists_xml_path, +- other_xml_path, +- None, +- pkgcb, +- warningcb, +- False) + + if __name__ == "__main__": + print('"All in one shot" method:') +@@ -263,7 +221,3 @@ if __name__ == "__main__": + print("Callback based method:") + second_method() + +- print() +- +- print("Streaming callback based method:") +- third_method() +-- +2.35.1 + + +From 01493aea9bc53e46a0e1020656e60ecb6c543bca Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:23:40 +0100 +Subject: [PATCH 2/5] Revert "Add python tests for parsing metadata together" + +This reverts commit 0c050128f7e24222011cdfcc67ba58da744affa9. +--- + tests/python/tests/test_xml_parser.py | 206 -------------------------- + 1 file changed, 206 deletions(-) + +diff --git a/tests/python/tests/test_xml_parser.py b/tests/python/tests/test_xml_parser.py +index ed13cac..ddc089f 100644 +--- a/tests/python/tests/test_xml_parser.py ++++ b/tests/python/tests/test_xml_parser.py +@@ -950,209 +950,3 @@ class TestCaseXmlParserRepomd(unittest.TestCase): + self.assertEqual(repomd.content_tags, []) + self.assertEqual(len(repomd.records), 3) + +-class TestCaseXmlParserMainMetadataTogether(unittest.TestCase): +- def test_xml_parser_main_metadata_together_repo01(self): +- userdata = { +- "pkgs": [], +- "pkgcb_calls": 0, +- "warnings": [] +- } +- +- def newpkgcb(pkgId, name, arch): +- pkg = cr.Package() +- userdata["pkgs"].append(pkg) +- return pkg +- +- def pkgcb(pkg): +- userdata["pkgcb_calls"] += 1 +- +- def warningcb(warn_type, msg): +- userdata["warnings"].append((warn_type, msg)) +- +- cr.xml_parse_main_metadata_together(primary=REPO_01_PRIXML, filelists=REPO_01_FILXML, other=REPO_01_OTHXML, +- newpkgcb=newpkgcb, pkgcb=pkgcb, warningcb=warningcb) +- +- self.assertEqual([pkg.name for pkg in userdata["pkgs"]], ['super_kernel']) +- self.assertEqual(userdata["pkgcb_calls"], 1) +- self.assertEqual(userdata["warnings"], []) +- +- pkg = userdata["pkgs"][0] +- self.assertEqual(pkg.pkgId, "152824bff2aa6d54f429d43e87a3ff3a0286505c6d93ec87692b5e3a9e3b97bf") +- self.assertEqual(pkg.name, "super_kernel") +- self.assertEqual(pkg.arch, "x86_64") +- self.assertEqual(pkg.version, "6.0.1") +- self.assertEqual(pkg.epoch, "0") +- self.assertEqual(pkg.release, "2") +- self.assertEqual(pkg.summary, "Test package") +- self.assertEqual(pkg.description, "This package has provides, requires, obsoletes, conflicts options.") +- self.assertEqual(pkg.url, "http://so_super_kernel.com/it_is_awesome/yep_it_really_is") +- self.assertEqual(pkg.time_file, 1334667003) +- self.assertEqual(pkg.time_build, 1334667003) +- self.assertEqual(pkg.rpm_license, "LGPLv2") +- self.assertEqual(pkg.rpm_vendor, None) +- self.assertEqual(pkg.rpm_group, "Applications/System") +- self.assertEqual(pkg.rpm_buildhost, "localhost.localdomain") +- self.assertEqual(pkg.rpm_sourcerpm, "super_kernel-6.0.1-2.src.rpm") +- self.assertEqual(pkg.rpm_header_start, 280) +- self.assertEqual(pkg.rpm_header_end, 2637) +- self.assertEqual(pkg.rpm_packager, None) +- self.assertEqual(pkg.size_package, 2845) +- self.assertEqual(pkg.size_installed, 0) +- self.assertEqual(pkg.size_archive, 404) +- self.assertEqual(pkg.location_href, "super_kernel-6.0.1-2.x86_64.rpm") +- self.assertEqual(pkg.location_base, None) +- self.assertEqual(pkg.checksum_type, "sha256") +- self.assertEqual(pkg.requires, +- [('bzip2', 'GE', '0', '1.0.0', None, True), +- ('expat', None, None, None, None, True), +- ('glib', 'GE', '0', '2.26.0', None, False), +- ('zlib', None, None, None, None, False)]) +- self.assertEqual(pkg.provides, +- [('not_so_super_kernel', 'LT', '0', '5.8.0', None, False), +- ('super_kernel', 'EQ', '0', '6.0.0', None, False), +- ('super_kernel', 'EQ', '0', '6.0.1', '2', False), +- ('super_kernel(x86-64)', 'EQ', '0', '6.0.1', '2', False)]) +- self.assertEqual(pkg.conflicts, +- [('kernel', None, None, None, None, False), +- ('super_kernel', 'EQ', '0', '5.0.0', None, False), +- ('super_kernel', 'LT', '0', '4.0.0', None, False)]) +- self.assertEqual(pkg.obsoletes, +- [('kernel', None, None, None, None, False), +- ('super_kernel', 'EQ', '0', '5.9.0', None, False)]) +- self.assertEqual(pkg.files, +- [(None, '/usr/bin/', 'super_kernel'), +- (None, '/usr/share/man/', 'super_kernel.8.gz')]) +- self.assertEqual(pkg.changelogs, +- [('Tomas Mlcoch - 6.0.1-1', +- 1334664000, +- '- First release'), +- ('Tomas Mlcoch - 6.0.1-2', +- 1334664001, +- '- Second release')]) +- +- def test_xml_parser_main_metadata_together_repo02(self): +- +- userdata = { +- "pkgs": [], +- "pkgcb_calls": 0, +- "warnings": [] +- } +- +- def newpkgcb(pkgId, name, arch): +- pkg = cr.Package() +- userdata["pkgs"].append(pkg) +- return pkg +- +- def pkgcb(pkg): +- userdata["pkgcb_calls"] += 1 +- +- def warningcb(warn_type, msg): +- userdata["warnings"].append((warn_type, msg)) +- +- cr.xml_parse_main_metadata_together(REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, newpkgcb, pkgcb, warningcb, +- allow_out_of_order=True) +- +- self.assertEqual([pkg.name for pkg in userdata["pkgs"]], +- ['fake_bash', 'super_kernel']) +- self.assertEqual(userdata["pkgcb_calls"], 2) +- self.assertEqual(userdata["warnings"], []) +- +- def test_xml_parser_main_metadata_together_repo02_only_pkgcb(self): +- +- pkgs = [] +- +- def pkgcb(pkg): +- pkgs.append(pkg) +- +- cr.xml_parse_main_metadata_together(REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, +- None, pkgcb, None) +- +- self.assertEqual([pkg.name for pkg in pkgs], +- ['fake_bash', 'super_kernel']) +- +- def test_xml_parser_main_metadata_together_repo02_no_cbs(self): +- self.assertRaises(ValueError, +- cr.xml_parse_main_metadata_together, +- REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, +- None, None, None) +- +- def test_xml_parser_main_metadata_together_warnings(self): +- userdata = { +- "pkgs": [], +- "warnings": [] +- } +- +- def newpkgcb(pkgId, name, arch): +- pkg = cr.Package() +- userdata["pkgs"].append(pkg) +- return pkg +- +- def warningcb(warn_type, msg): +- userdata["warnings"].append((warn_type, msg)) +- +- cr.xml_parse_main_metadata_together(PRIMARY_MULTI_WARN_00_PATH, +- FILELISTS_MULTI_WARN_00_PATH, +- OTHER_MULTI_WARN_00_PATH, +- newpkgcb, None, warningcb) +- +- self.assertEqual([pkg.name for pkg in userdata["pkgs"]], +- ['fake_bash', 'super_kernel']) +- self.assertEqual(userdata["warnings"], +- [(0, 'Unknown element "fooelement"'), +- (1, 'Missing attribute "type" of a package element'), +- (0, 'Unknown element "foo"'), +- (3, 'Conversion of "foobar" to integer failed'), +- (0, 'Unknown element "bar"'), +- (1, 'Missing attribute "arch" of a package element'), +- (2, 'Unknown file type "xxx"'), +- (0, 'Unknown element "bar"'), +- (1, 'Missing attribute "name" of a package element'), +- (0, 'Unknown element "bar"'), +- (3, 'Conversion of "xxx" to integer failed')]) +- +- +- def test_xml_parser_main_metadata_together_error(self): +- userdata = { "pkgs": [] } +- +- def newpkgcb(pkgId, name, arch): +- pkg = cr.Package() +- userdata["pkgs"].append(pkg) +- return pkg +- +- self.assertRaises(cr.CreaterepoCError, cr.xml_parse_main_metadata_together, +- PRIMARY_ERROR_00_PATH, FILELISTS_ERROR_00_PATH, +- OTHER_ERROR_00_PATH, newpkgcb, None, None) +- +- # unlike parsing just primary, filelists or other separately when parsing together primary is parsed first fully +- # before newpkgcb is called so the error is detected before any user package is created +- self.assertEqual(len(userdata["pkgs"]), 0) +- +- def test_xml_parser_main_metadata_together_newpkgcb_abort(self): +- def newpkgcb(pkgId, name, arch): +- raise Error("Foo error") +- self.assertRaises(cr.CreaterepoCError, +- cr.xml_parse_main_metadata_together, +- REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, +- newpkgcb, None, None) +- +- def test_xml_parser_main_metadata_together_pkgcb_abort(self): +- def newpkgcb(pkgId, name, arch): +- return cr.Package() +- def pkgcb(pkg): +- raise Error("Foo error") +- self.assertRaises(cr.CreaterepoCError, +- cr.xml_parse_main_metadata_together, +- REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, +- newpkgcb, pkgcb, None) +- +- def test_xml_parser_main_metadata_together_warningcb_abort(self): +- def newpkgcb(pkgId, name, arch): +- return cr.Package() +- def warningcb(type, msg): +- raise Error("Foo error") +- self.assertRaises(cr.CreaterepoCError, +- cr.xml_parse_main_metadata_together, +- PRIMARY_MULTI_WARN_00_PATH, +- FILELISTS_MULTI_WARN_00_PATH, +- OTHER_MULTI_WARN_00_PATH, +- newpkgcb, None, warningcb) +-- +2.35.1 + + +From 9210f5d725a71f130bf5fb880f386c2e8e5f66fa Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:23:47 +0100 +Subject: [PATCH 3/5] Revert "Wrap c api to python for parsing metadata + together" + +This reverts commit 670006b9f37d8298fe2ec1d6cfc5cdf175a3128b. +--- + src/python/createrepo_cmodule.c | 2 - + src/python/xml_parser-py.c | 156 +++++++------------------------- + src/python/xml_parser-py.h | 8 -- + 3 files changed, 32 insertions(+), 134 deletions(-) + +diff --git a/src/python/createrepo_cmodule.c b/src/python/createrepo_cmodule.c +index c0b9200..6937246 100644 +--- a/src/python/createrepo_cmodule.c ++++ b/src/python/createrepo_cmodule.c +@@ -79,8 +79,6 @@ static struct PyMethodDef createrepo_c_methods[] = { + METH_VARARGS, xml_parse_repomd__doc__}, + {"xml_parse_updateinfo", (PyCFunction)py_xml_parse_updateinfo, + METH_VARARGS, xml_parse_updateinfo__doc__}, +- {"xml_parse_main_metadata_together",(PyCFunctionWithKeywords)py_xml_parse_main_metadata_together, +- METH_VARARGS | METH_KEYWORDS, xml_parse_main_metadata_together__doc__}, + {"checksum_name_str", (PyCFunction)py_checksum_name_str, + METH_VARARGS, checksum_name_str__doc__}, + {"checksum_type", (PyCFunction)py_checksum_type, +diff --git a/src/python/xml_parser-py.c b/src/python/xml_parser-py.c +index a1f92f8..0879128 100644 +--- a/src/python/xml_parser-py.c ++++ b/src/python/xml_parser-py.c +@@ -34,7 +34,7 @@ typedef struct { + PyObject *py_newpkgcb; + PyObject *py_pkgcb; + PyObject *py_warningcb; +- PyObject *py_pkgs; /*!< Current processed package */ ++ PyObject *py_pkg; /*!< Current processed package */ + } CbData; + + static int +@@ -48,6 +48,12 @@ c_newpkgcb(cr_Package **pkg, + PyObject *arglist, *result; + CbData *data = cbdata; + ++ if (data->py_pkg) { ++ // Decref ref count on previous processed package ++ Py_DECREF(data->py_pkg); ++ data->py_pkg = NULL; ++ } ++ + arglist = Py_BuildValue("(sss)", pkgId, name, arch); + result = PyObject_CallObject(data->py_newpkgcb, arglist); + Py_DECREF(arglist); +@@ -67,18 +73,13 @@ c_newpkgcb(cr_Package **pkg, + + if (result == Py_None) { + *pkg = NULL; ++ data->py_pkg = NULL; ++ Py_DECREF(result); + } else { + *pkg = Package_FromPyObject(result); +- if (data->py_pkgcb != Py_None) { +- // Store reference to the python pkg (result) only if we will need it later +- PyObject *keyFromPtr = PyLong_FromVoidPtr(*pkg); +- PyDict_SetItem(data->py_pkgs, keyFromPtr, result); +- Py_DECREF(keyFromPtr); +- } ++ data->py_pkg = result; // Store reference to current package + } + +- Py_DECREF(result); +- + return CR_CB_RET_OK; + } + +@@ -90,23 +91,16 @@ c_pkgcb(cr_Package *pkg, + PyObject *arglist, *result, *py_pkg; + CbData *data = cbdata; + +- PyObject *keyFromPtr = PyLong_FromVoidPtr(pkg); +- py_pkg = PyDict_GetItem(data->py_pkgs, keyFromPtr); +- if (py_pkg) { +- arglist = Py_BuildValue("(O)", py_pkg); +- result = PyObject_CallObject(data->py_pkgcb, arglist); +- PyDict_DelItem(data->py_pkgs, keyFromPtr); +- } else { +- // The package was not provided by user in c_newpkgcb, +- // create new python package object +- PyObject *new_py_pkg = Object_FromPackage(pkg, 1); +- arglist = Py_BuildValue("(O)", new_py_pkg); +- result = PyObject_CallObject(data->py_pkgcb, arglist); +- Py_DECREF(new_py_pkg); +- } ++ if (data->py_pkg) ++ py_pkg = data->py_pkg; ++ else ++ py_pkg = Object_FromPackage(pkg, 1); + ++ arglist = Py_BuildValue("(O)", py_pkg); ++ result = PyObject_CallObject(data->py_pkgcb, arglist); + Py_DECREF(arglist); +- Py_DECREF(keyFromPtr); ++ Py_DECREF(py_pkg); ++ data->py_pkg = NULL; + + if (result == NULL) { + // Exception raised +@@ -197,7 +191,7 @@ py_xml_parse_primary(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_primary(filename, + ptr_c_newpkgcb, +@@ -212,8 +206,7 @@ py_xml_parse_primary(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -279,7 +272,7 @@ py_xml_parse_primary_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_primary_snippet(target, ptr_c_newpkgcb, &cbdata, ptr_c_pkgcb, &cbdata, + ptr_c_warningcb, &cbdata, do_files, &tmp_err); +@@ -287,7 +280,7 @@ py_xml_parse_primary_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -351,7 +344,7 @@ py_xml_parse_filelists(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_filelists(filename, + ptr_c_newpkgcb, +@@ -365,7 +358,7 @@ py_xml_parse_filelists(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -429,7 +422,7 @@ py_xml_parse_filelists_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_filelists_snippet(target, ptr_c_newpkgcb, &cbdata, ptr_c_pkgcb, + &cbdata, ptr_c_warningcb, &cbdata, &tmp_err); +@@ -437,7 +430,7 @@ py_xml_parse_filelists_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -501,7 +494,7 @@ py_xml_parse_other(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_other(filename, + ptr_c_newpkgcb, +@@ -515,7 +508,7 @@ py_xml_parse_other(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -579,7 +572,7 @@ py_xml_parse_other_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_other_snippet(target, ptr_c_newpkgcb, &cbdata, ptr_c_pkgcb, &cbdata, + ptr_c_warningcb, &cbdata, &tmp_err); +@@ -587,7 +580,7 @@ py_xml_parse_other_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -630,7 +623,7 @@ py_xml_parse_repomd(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = NULL; + cbdata.py_pkgcb = NULL; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = NULL; ++ cbdata.py_pkg = NULL; + + repomd = Repomd_FromPyObject(py_repomd); + +@@ -684,7 +677,7 @@ py_xml_parse_updateinfo(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = NULL; + cbdata.py_pkgcb = NULL; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = NULL; ++ cbdata.py_pkg = NULL; + + updateinfo = UpdateInfo_FromPyObject(py_updateinfo); + +@@ -704,88 +697,3 @@ py_xml_parse_updateinfo(G_GNUC_UNUSED PyObject *self, PyObject *args) + + Py_RETURN_NONE; + } +- +- +-PyObject * +-py_xml_parse_main_metadata_together(G_GNUC_UNUSED PyObject *self, PyObject *args, PyObject *kwargs) +-{ +- char *primary_filename; +- char *filelists_filename; +- char *other_filename; +- int allow_out_of_order = 1; +- PyObject *py_newpkgcb, *py_pkgcb, *py_warningcb; +- CbData cbdata; +- GError *tmp_err = NULL; +- static char *kwlist[] = { "primary", "filelists", "other", "newpkgcb", "pkgcb", +- "warningcb", "allow_out_of_order", NULL }; +- +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "sssOOO|p:py_xml_parse_main_metadata_together", kwlist, +- &primary_filename, &filelists_filename, &other_filename, &py_newpkgcb, +- &py_pkgcb, &py_warningcb, &allow_out_of_order)) { +- return NULL; +- } +- +- if (!PyCallable_Check(py_newpkgcb) && py_newpkgcb != Py_None) { +- PyErr_SetString(PyExc_TypeError, "newpkgcb must be callable or None"); +- return NULL; +- } +- +- if (!PyCallable_Check(py_pkgcb) && py_pkgcb != Py_None) { +- PyErr_SetString(PyExc_TypeError, "pkgcb must be callable or None"); +- return NULL; +- } +- +- if (!PyCallable_Check(py_warningcb) && py_warningcb != Py_None) { +- PyErr_SetString(PyExc_TypeError, "warningcb must be callable or None"); +- return NULL; +- } +- +- if (py_newpkgcb == Py_None && py_pkgcb == Py_None) { +- PyErr_SetString(PyExc_ValueError, "both pkgcb and newpkgcb cannot be None"); +- return NULL; +- } +- +- Py_XINCREF(py_newpkgcb); +- Py_XINCREF(py_pkgcb); +- Py_XINCREF(py_warningcb); +- +- cr_XmlParserNewPkgCb ptr_c_newpkgcb = NULL; +- cr_XmlParserPkgCb ptr_c_pkgcb = NULL; +- cr_XmlParserWarningCb ptr_c_warningcb = NULL; +- +- if (py_newpkgcb != Py_None) +- ptr_c_newpkgcb = c_newpkgcb; +- if (py_pkgcb != Py_None) +- ptr_c_pkgcb = c_pkgcb; +- if (py_warningcb != Py_None) +- ptr_c_warningcb = c_warningcb; +- +- cbdata.py_newpkgcb = py_newpkgcb; +- cbdata.py_pkgcb = py_pkgcb; +- cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); +- +- cr_xml_parse_main_metadata_together(primary_filename, +- filelists_filename, +- other_filename, +- ptr_c_newpkgcb, +- &cbdata, +- ptr_c_pkgcb, +- &cbdata, +- ptr_c_warningcb, +- &cbdata, +- allow_out_of_order, +- &tmp_err); +- +- Py_XDECREF(py_newpkgcb); +- Py_XDECREF(py_pkgcb); +- Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); +- +- if (tmp_err) { +- nice_exception(&tmp_err, NULL); +- return NULL; +- } +- +- Py_RETURN_NONE; +-} +diff --git a/src/python/xml_parser-py.h b/src/python/xml_parser-py.h +index 3260005..e5eea9f 100644 +--- a/src/python/xml_parser-py.h ++++ b/src/python/xml_parser-py.h +@@ -64,12 +64,4 @@ PyDoc_STRVAR(xml_parse_updateinfo__doc__, + + PyObject *py_xml_parse_updateinfo(PyObject *self, PyObject *args); + +-PyDoc_STRVAR(xml_parse_main_metadata_together__doc__, +-"xml_parse_main_metadata_together(primary_filename, filelists_filename, other_filename, newpkgcb, pkgcb, warningcb) -> None\n\n" +-"Parse primary.xml, filelists.xml and other.xml together at the same time." +-"- It can handle if packages are not in the same order in all 3 files but memory requirements grow." +-"- It is not guaranteed that newpkgcb is always followed by pkgcb for the given package, it is possible newpkgcb will be called several times for different packages and only after that pkgcbs will be called."); +- +-PyObject *py_xml_parse_main_metadata_together(PyObject *self, PyObject *args, PyObject *kwargs); +- + #endif +-- +2.35.1 + + +From ac6f56e87f8a4b1f0a35d9dedc02a8846c9a6390 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:23:58 +0100 +Subject: [PATCH 4/5] Revert "Add c tests for parsing metadata together" + +This reverts commit 3ef3d4d77e283ee916fc7977cdc3855b912463e0. +--- + tests/CMakeLists.txt | 4 - + tests/fixtures.h | 2 - + .../test_xml_parser_main_metadata_together.c | 298 ----- + .../modified_repo_files/long_primary.xml | 1060 ----------------- + .../repo_02_different_order_filelists.xml | 16 - + 5 files changed, 1380 deletions(-) + delete mode 100644 tests/test_xml_parser_main_metadata_together.c + delete mode 100644 tests/testdata/modified_repo_files/long_primary.xml + delete mode 100644 tests/testdata/modified_repo_files/repo_02_different_order_filelists.xml + +diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt +index 4ffe837..3936176 100644 +--- a/tests/CMakeLists.txt ++++ b/tests/CMakeLists.txt +@@ -38,10 +38,6 @@ ADD_EXECUTABLE(test_xml_parser_updateinfo test_xml_parser_updateinfo.c) + TARGET_LINK_LIBRARIES(test_xml_parser_updateinfo libcreaterepo_c ${GLIB2_LIBRARIES}) + ADD_DEPENDENCIES(tests test_xml_parser_updateinfo) + +-ADD_EXECUTABLE(test_xml_parser_main_metadata_together test_xml_parser_main_metadata_together.c) +-TARGET_LINK_LIBRARIES(test_xml_parser_main_metadata_together libcreaterepo_c ${GLIB2_LIBRARIES}) +-ADD_DEPENDENCIES(tests test_xml_parser_main_metadata_together) +- + ADD_EXECUTABLE(test_xml_dump test_xml_dump.c) + TARGET_LINK_LIBRARIES(test_xml_dump libcreaterepo_c ${GLIB2_LIBRARIES}) + ADD_DEPENDENCIES(tests test_xml_dump) +diff --git a/tests/fixtures.h b/tests/fixtures.h +index 676518c..a47dd66 100644 +--- a/tests/fixtures.h ++++ b/tests/fixtures.h +@@ -85,8 +85,6 @@ + #define TEST_MRF_UE_OTH_00 TEST_MODIFIED_REPO_FILES_PATH"unknown_element_00-other.xml" + #define TEST_MRF_UE_OTH_01 TEST_MODIFIED_REPO_FILES_PATH"unknown_element_01-other.xml" + #define TEST_MRF_UE_OTH_02 TEST_MODIFIED_REPO_FILES_PATH"unknown_element_02-other.xml" +-#define TEST_LONG_PRIMARY TEST_MODIFIED_REPO_FILES_PATH"long_primary.xml" +-#define TEST_DIFF_ORDER_FILELISTS TEST_MODIFIED_REPO_FILES_PATH"repo_02_different_order_filelists.xml" + + // Test files + +diff --git a/tests/test_xml_parser_main_metadata_together.c b/tests/test_xml_parser_main_metadata_together.c +deleted file mode 100644 +index a4e4ad7..0000000 +--- a/tests/test_xml_parser_main_metadata_together.c ++++ /dev/null +@@ -1,298 +0,0 @@ +-/* +- * Copyright (C) 2021 Red Hat, Inc. +- * +- * Licensed under the GNU Lesser General Public License Version 2.1 +- * +- * This library is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2.1 of the License, or (at your option) any later version. +- * +- * This library is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * Lesser General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +- */ +- +-#include +-#include +-#include +-#include "fixtures.h" +-#include "createrepo/error.h" +-#include "createrepo/package.h" +-#include "createrepo/misc.h" +-#include "createrepo/xml_parser.h" +-#include "createrepo/xml_parser_internal.h" +- +-// Callbacks +- +-static int +-pkgcb(cr_Package *pkg, void *cbdata, GError **err) +-{ +- g_assert(pkg); +- g_assert(!err || *err == NULL); +- if (cbdata) *((int *)cbdata) += 1; +- cr_package_free(pkg); +- return CR_CB_RET_OK; +-} +- +-static int +-pkgcb_interrupt(cr_Package *pkg, void *cbdata, GError **err) +-{ +- g_assert(pkg); +- g_assert(!err || *err == NULL); +- if (cbdata) *((int *)cbdata) += 1; +- cr_package_free(pkg); +- return CR_CB_RET_ERR; +-} +- +-static int +-newpkgcb_skip_fake_bash(cr_Package **pkg, +- G_GNUC_UNUSED const char *pkgId, +- const char *name, +- G_GNUC_UNUSED const char *arch, +- G_GNUC_UNUSED void *cbdata, +- GError **err) +-{ +- g_assert(pkg != NULL); +- g_assert(*pkg == NULL); +- g_assert(pkgId != NULL); +- g_assert(!err || *err == NULL); +- +- +- if (!g_strcmp0(name, "fake_bash")) { +- return CRE_OK; +- } +- +- *pkg = cr_package_new(); +- return CR_CB_RET_OK; +-} +- +-static int +-newpkgcb_interrupt(cr_Package **pkg, +- G_GNUC_UNUSED const char *pkgId, +- G_GNUC_UNUSED const char *name, +- G_GNUC_UNUSED const char *arch, +- G_GNUC_UNUSED void *cbdata, +- GError **err) +-{ +- g_assert(pkg != NULL); +- g_assert(*pkg == NULL); +- g_assert(pkgId != NULL); +- g_assert(!err || *err == NULL); +- +- if (cbdata) *((int *)cbdata) += 1; +- +- return CR_CB_RET_ERR; +-} +- +-static int +-warningcb(G_GNUC_UNUSED cr_XmlParserWarningType type, +- G_GNUC_UNUSED char *msg, +- void *cbdata, +- G_GNUC_UNUSED GError **err) +-{ +- g_assert(type < CR_XML_WARNING_SENTINEL); +- g_assert(!err || *err == NULL); +- +- g_string_append((GString *) cbdata, msg); +- g_string_append((GString *) cbdata, ";"); +- +- return CR_CB_RET_OK; +-} +- +-static int +-warningcb_interrupt(G_GNUC_UNUSED cr_XmlParserWarningType type, +- G_GNUC_UNUSED char *msg, +- G_GNUC_UNUSED void *cbdata, +- G_GNUC_UNUSED GError **err) +-{ +- g_assert(type < CR_XML_WARNING_SENTINEL); +- g_assert(!err || *err == NULL); +- +- if (cbdata) *((int *)cbdata) += 1; +- +- return CR_CB_RET_ERR; +-} +- +-// Tests +- +-static void +-test_cr_xml_parse_main_metadata_together_00(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 2); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_01_out_of_order_pkgs(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, +- TEST_DIFF_ORDER_FILELISTS, +- TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, FALSE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_assert_cmpint(ret, ==, CRE_XMLPARSER); +- +- g_clear_error(&tmp_err); +- ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_DIFF_ORDER_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 2); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_02_invalid_path(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together("/non/existent/file", TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_assert_cmpint(ret, ==, CRE_NOFILE); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_03_newpkgcb_returns_null(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- newpkgcb_skip_fake_bash, NULL, pkgcb, &parsed, NULL, NULL, TRUE, +- &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 1); +- +- parsed = 0; +- ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_DIFF_ORDER_FILELISTS, TEST_REPO_02_OTHER, +- newpkgcb_skip_fake_bash, NULL, pkgcb, &parsed, NULL, NULL, TRUE, +- &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 1); +- +- parsed = 0; +- ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_DIFF_ORDER_FILELISTS, TEST_REPO_02_OTHER, +- newpkgcb_skip_fake_bash, NULL, pkgcb, &parsed, NULL, NULL, FALSE, +- &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 1); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_04_newpkgcb_interrupt(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- newpkgcb_interrupt, &parsed, NULL, NULL, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_error_free(tmp_err); +- g_assert_cmpint(ret, ==, CRE_CBINTERRUPTED); +- g_assert_cmpint(parsed, ==, 1); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_05_pkgcb_interrupt(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb_interrupt, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_error_free(tmp_err); +- g_assert_cmpint(ret, ==, CRE_CBINTERRUPTED); +- g_assert_cmpint(parsed, ==, 1); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_06_warnings_bad_file_type(void) +-{ +- int parsed = 0; +- char *warnmsgs; +- GError *tmp_err = NULL; +- GString *warn_strings = g_string_new(0); +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_MRF_BAD_TYPE_FIL, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, warningcb, warn_strings, TRUE, &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 2); +- warnmsgs = g_string_free(warn_strings, FALSE); +- g_assert_cmpstr(warnmsgs, ==, "Unknown file type \"foo\";"); +- g_free(warnmsgs); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_07_warningcb_interrupt(void) +-{ +- int numofwarnings = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_MRF_BAD_TYPE_FIL, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, NULL, warningcb_interrupt, &numofwarnings, +- TRUE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_error_free(tmp_err); +- g_assert_cmpint(ret, ==, CRE_CBINTERRUPTED); +- g_assert_cmpint(numofwarnings, ==, 1); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_08_long_primary(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_LONG_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 2); +-} +- +-int +-main(int argc, char *argv[]) +-{ +- g_test_init(&argc, &argv, NULL); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_00", +- test_cr_xml_parse_main_metadata_together_00); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_01_out_of_order_pkgs", +- test_cr_xml_parse_main_metadata_together_01_out_of_order_pkgs); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_02_invalid_path", +- test_cr_xml_parse_main_metadata_together_02_invalid_path); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_03_newpkgcb_returns_null", +- test_cr_xml_parse_main_metadata_together_03_newpkgcb_returns_null); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_04_newpkgcb_interrupt", +- test_cr_xml_parse_main_metadata_together_04_newpkgcb_interrupt); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_05_pkgcb_interrupt", +- test_cr_xml_parse_main_metadata_together_05_pkgcb_interrupt); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_06_warnings_bad_file_type", +- test_cr_xml_parse_main_metadata_together_06_warnings_bad_file_type); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_07_warningcb_interrupt", +- test_cr_xml_parse_main_metadata_together_07_warningcb_interrupt); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_08_long_primary", +- test_cr_xml_parse_main_metadata_together_08_long_primary); +- +- return g_test_run(); +-} +diff --git a/tests/testdata/modified_repo_files/long_primary.xml b/tests/testdata/modified_repo_files/long_primary.xml +deleted file mode 100644 +index 41d8a89..0000000 +--- a/tests/testdata/modified_repo_files/long_primary.xml ++++ /dev/null +@@ -1,1060 +0,0 @@ +- +- +- +- fake_bash +- x86_64 +- +- 90f61e546938a11449b710160ad294618a5bd3062e46f8cf851fd0088af184b7 +- Fake bash +- Fake bash package +- +- http://fake_bash_shell.com/ +- +- +- super_kernel +- x86_64 +- +- 6d43a638af70ef899933b1fd86a866f18f65b0e0e17dcbf2e42bfd0cdd7c63c3 +- Test package +- This package has provides, requires, obsoletes, conflicts options. +- +- http://so_super_kernel.com/it_is_awesome/yep_it_really_is +- +- +diff --git a/tests/testdata/modified_repo_files/repo_02_different_order_filelists.xml b/tests/testdata/modified_repo_files/repo_02_different_order_filelists.xml +deleted file mode 100644 +index 4953c95..0000000 +--- a/tests/testdata/modified_repo_files/repo_02_different_order_filelists.xml ++++ /dev/null +@@ -1,16 +0,0 @@ +- +- +- +- +- +- /usr/bin/super_kernel +- /usr/share/man/super_kernel.8.gz +- +- +- +- +- +- /usr/bin/fake_bash +- +- +- +-- +2.35.1 + + +From 16f3c1f1abe995fc1c0023f9109b7f5d520ff4b4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:24:05 +0100 +Subject: [PATCH 5/5] Revert "Add c API for parsing metadata together" + +This reverts commit ff0912d2a89722700f001ef462abd730dacaa8e7. +--- + src/CMakeLists.txt | 1 - + src/xml_parser.h | 39 -- + src/xml_parser_main_metadata_together.c | 468 ------------------------ + 3 files changed, 508 deletions(-) + delete mode 100644 src/xml_parser_main_metadata_together.c + +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index b3ca9e6..64de052 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -31,7 +31,6 @@ SET (createrepo_c_SRCS + xml_parser_primary.c + xml_parser_repomd.c + xml_parser_updateinfo.c +- xml_parser_main_metadata_together.c + koji.c) + + SET(headers +diff --git a/src/xml_parser.h b/src/xml_parser.h +index 5ce7c0e..a31718d 100644 +--- a/src/xml_parser.h ++++ b/src/xml_parser.h +@@ -278,45 +278,6 @@ cr_xml_parse_updateinfo(const char *path, + void *warningcb_data, + GError **err); + +-/** Parse all 3 main metadata types (primary, filelists and other) at the same time. +- * Once a package is fully parsed pkgcb is called which transfers ownership of the package +- * to the user, cr_xml_parse_main_metadata_together no longer needs it and it can be freed. +- * This means we don't have store all the packages in memory at the same time, which +- * significantly reduces the memory footprint. +- * Input metadata files can be compressed. +- * @param primary_path Path to a primary xml file. +- * @param filelists_path Path to a filelists xml file. +- * @param other_path Path to an other xml file. +- * @param newpkgcb Callback for a new package. Called when the new package +- * xml chunk is found and a package object to store +- * the data is needed. +- * @param newpkgcb_data User data for the newpkgcb. +- * @param pkgcb Package callback. Called when a package is completely +- * parsed containing information from all 3 main metadata +- * files. Could be NULL if newpkgcb is not NULL. +- * @param pkgcb_data User data for the pkgcb. +- * @param warningcb Callback for warning messages. +- * @param warningcb_data User data for the warningcb. +- * @param allow_out_of_order Whether we should allow different order of packages +- * among the main metadata files. If allowed, the more +- * the order varies the more memory we will need to +- * store all the started but unfinished packages. +- * @param err GError ** +- * @return cr_Error code. +- */ +-int +-cr_xml_parse_main_metadata_together(const char *primary_path, +- const char *filelists_path, +- const char *other_path, +- cr_XmlParserNewPkgCb newpkgcb, +- void *newpkgcb_data, +- cr_XmlParserPkgCb pkgcb, +- void *pkgcb_data, +- cr_XmlParserWarningCb warningcb, +- void *warningcb_data, +- gboolean allow_out_of_order, +- GError **err); +- + /** @} */ + + #ifdef __cplusplus +diff --git a/src/xml_parser_main_metadata_together.c b/src/xml_parser_main_metadata_together.c +deleted file mode 100644 +index 9347c79..0000000 +--- a/src/xml_parser_main_metadata_together.c ++++ /dev/null +@@ -1,468 +0,0 @@ +-/* +- * Copyright (C) 2021 Red Hat, Inc. +- * +- * Licensed under the GNU Lesser General Public License Version 2.1 +- * +- * This library is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2.1 of the License, or (at your option) any later version. +- * +- * This library is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * Lesser General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +- */ +- +-#include +-#include +-#include +-#include +-#include "error.h" +-#include "xml_parser.h" +-#include "xml_parser_internal.h" +-#include "package_internal.h" +-#include "misc.h" +- +-#define ERR_DOMAIN CREATEREPO_C_ERROR +- +-typedef struct { +- GHashTable *in_progress_pkgs_hash; //used only when allowing out of order pkgs +- GSList *in_progress_pkgs_list; // used only when not allowing out of order pkgs +- int in_progress_count_primary; +- int in_progress_count_filelists; +- int in_progress_count_other; +- cr_XmlParserNewPkgCb newpkgcb; // newpkgcb passed in from user +- void *newpkgcb_data;// newpkgcb data passed in from user +- cr_XmlParserPkgCb pkgcb; // pkgcb passed in from user +- void *pkgcb_data; // pkgcb data passed in from user +-} cr_CbData; +- +-static int +-call_user_callback_if_package_finished(cr_Package *pkg, cr_CbData *cb_data, GError **err) +-{ +- if (pkg && (pkg->loadingflags & CR_PACKAGE_LOADED_PRI) && (pkg->loadingflags & CR_PACKAGE_LOADED_OTH) && +- (pkg->loadingflags & CR_PACKAGE_LOADED_FIL)) +- { +- if (cb_data->in_progress_pkgs_hash) { +- g_hash_table_remove(cb_data->in_progress_pkgs_hash, pkg->pkgId); +- } else { +- //remove first element in the list +- cb_data->in_progress_pkgs_list = cb_data->in_progress_pkgs_list->next; +- } +- +- // One package was fully finished +- cb_data->in_progress_count_primary--; +- cb_data->in_progress_count_filelists--; +- cb_data->in_progress_count_other--; +- +- // call user package callback +- GError *tmp_err = NULL; +- if (cb_data->pkgcb && cb_data->pkgcb(pkg, cb_data->pkgcb_data, &tmp_err)) { +- if (tmp_err) +- g_propagate_prefixed_error(err, tmp_err, "Parsing interrupted: "); +- else +- g_set_error(err, ERR_DOMAIN, CRE_CBINTERRUPTED, "Parsing interrupted"); +- return CR_CB_RET_ERR; +- } else { +- // If callback return CRE_OK but it simultaneously set +- // the tmp_err then it's a programming error. +- assert(tmp_err == NULL); +- }; +- } +- return CR_CB_RET_OK; +-} +- +-static cr_Package* +-find_in_progress_pkg(cr_CbData *cb_data, const char *pkgId, int in_progress_pkg_index, GError **err) +-{ +- gpointer pval = NULL; +- if (cb_data->in_progress_pkgs_hash) { +- if (!g_hash_table_lookup_extended(cb_data->in_progress_pkgs_hash, pkgId, NULL, &pval)) { +- pval = NULL; +- } +- } else { +- // This is checking out of order pkgs because if we don't have in_progress_pkgs_hash -> we enforce +- // order by using a list +- pval = g_slist_nth_data(cb_data->in_progress_pkgs_list, in_progress_pkg_index); +- if (pval && g_strcmp0(((cr_Package *) pval)->pkgId, pkgId)) { +- g_set_error(err, ERR_DOMAIN, CRE_XMLPARSER, +- "Out of order metadata: %s vs %s.", ((cr_Package *) pval)->pkgId, pkgId); +- pval = NULL; +- } +- } +- +- return pval; +-} +- +-static void +-store_in_progress_pkg(cr_CbData *cb_data, cr_Package *pkg, const char *pkgId) +-{ +- if (!pkg) { +- return; +- } +- if (cb_data->in_progress_pkgs_hash) { +- g_hash_table_insert(cb_data->in_progress_pkgs_hash, g_strdup(pkgId), pkg); +- } else { +- cb_data->in_progress_pkgs_list = g_slist_append(cb_data->in_progress_pkgs_list, pkg); +- } +-} +- +-static int +-newpkgcb_primary(cr_Package **pkg, +- G_GNUC_UNUSED const char *pkgId, +- G_GNUC_UNUSED const char *name, +- G_GNUC_UNUSED const char *arch, +- G_GNUC_UNUSED void *cbdata, +- GError **err) +-{ +- assert(pkg && *pkg == NULL); +- assert(!err || *err == NULL); +- +- // This callback is called when parsing of the opening element of a package +- // is done. However because the opening element doesn't contain pkgId +- // (instead it looks like: ) we cannot check if we +- // already have this package. +- // The only option is to create a new package and after its fully +- // parsed (in pkgcb_primary) either use this package or copy its data +- // into an already existing one. +- // Filelists and other have pkgId present in the opening element so we can +- // avoid this overhead. +- *pkg = cr_package_new(); +- +- return CR_CB_RET_OK; +-} +- +-static int +-newpkg_general(cr_Package **pkg, +- const char *pkgId, +- const char *name, +- const char *arch, +- void *cbdata, +- int in_progress_count, +- GError **err) +-{ +- assert(pkg && *pkg == NULL); +- assert(!err || *err == NULL); +- +- cr_CbData *cb_data = cbdata; +- +- GError *out_of_order_err = NULL; +- *pkg = find_in_progress_pkg(cb_data, pkgId, in_progress_count, &out_of_order_err); +- +- if (!*pkg) { +- // we are handling never before seen package +- +- if (cb_data->newpkgcb) { +- // user specified their own new function: call it +- if (cb_data->newpkgcb(pkg, pkgId, name, arch, cb_data->newpkgcb_data, err)) { +- return CR_CB_RET_ERR; +- } +- if (!*pkg) { +- // when the user callback doesn't return a pkg we should skip it, +- // this means out of order error doesn't apply +- g_clear_error(&out_of_order_err); +- } +- } else { +- *pkg = cr_package_new(); +- } +- +- store_in_progress_pkg(cb_data, *pkg, pkgId); +- } +- +- if (*err) { +- return CR_CB_RET_ERR; +- } +- +- if (out_of_order_err) { +- g_propagate_error(err, out_of_order_err); +- return CR_CB_RET_ERR; +- } +- +- return CR_CB_RET_OK; +-} +- +-static int +-newpkgcb_filelists(cr_Package **pkg, +- const char *pkgId, +- G_GNUC_UNUSED const char *name, +- G_GNUC_UNUSED const char *arch, +- void *cbdata, +- GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- return newpkg_general(pkg, pkgId, name, arch, cbdata, cb_data->in_progress_count_filelists, err); +-} +- +-static int +-newpkgcb_other(cr_Package **pkg, +- const char *pkgId, +- G_GNUC_UNUSED const char *name, +- G_GNUC_UNUSED const char *arch, +- void *cbdata, +- GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- return newpkg_general(pkg, pkgId, name, arch, cbdata, cb_data->in_progress_count_other, err); +-} +- +-static int +-pkgcb_filelists(cr_Package *pkg, void *cbdata, G_GNUC_UNUSED GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- cb_data->in_progress_count_filelists++; +- pkg->loadingflags |= CR_PACKAGE_LOADED_FIL; +- return call_user_callback_if_package_finished(pkg, cb_data, err); +-} +- +-static int +-pkgcb_other(cr_Package *pkg, void *cbdata, G_GNUC_UNUSED GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- cb_data->in_progress_count_other++; +- pkg->loadingflags |= CR_PACKAGE_LOADED_OTH; +- return call_user_callback_if_package_finished(pkg, cb_data, err); +-} +- +-static int +-pkgcb_primary(cr_Package *pkg, void *cbdata, G_GNUC_UNUSED GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- +- GError *out_of_order_err = NULL; +- cr_Package *in_progress_pkg = find_in_progress_pkg(cb_data, pkg->pkgId, cb_data->in_progress_count_primary, +- &out_of_order_err); +- if (in_progress_pkg) { +- // package was already encountered in some other metadata type +- +- cr_package_copy_into(pkg, in_progress_pkg); +- cr_package_free(pkg); +- pkg = in_progress_pkg; +- } else { +- // we are handling never before seen package +- +- if (cb_data->newpkgcb) { +- // user specified their own new function: call it and copy package data into user_created_pkg +- cr_Package *user_created_pkg = NULL; +- if (cb_data->newpkgcb(&user_created_pkg, pkg->pkgId, pkg->name, pkg->arch, cb_data->newpkgcb_data, err)) { +- return CR_CB_RET_ERR; +- } else { +- if (user_created_pkg) { +- cr_package_copy_into(pkg, user_created_pkg); +- } +- // user_created_pkg can be NULL if newpkgcb returns OK but +- // not an allocated pkg -> this means we should skip it +- store_in_progress_pkg(cb_data, user_created_pkg, pkg->pkgId); +- cr_package_free(pkg); +- pkg = user_created_pkg; +- } +- if (!pkg) { +- // when the user callback doesn't return a pkg we should skip it, +- // this means out of order error doesn't apply +- g_clear_error(&out_of_order_err); +- } +- } else { +- store_in_progress_pkg(cb_data, pkg, pkg->pkgId); +- } +- +- } +- +- if (*err) { +- return CR_CB_RET_ERR; +- } +- +- if (out_of_order_err) { +- g_propagate_error(err, out_of_order_err); +- return CR_CB_RET_ERR; +- } +- +- +- if (pkg) { +- cb_data->in_progress_count_primary++; +- pkg->loadingflags |= CR_PACKAGE_LOADED_PRI; +- pkg->loadingflags |= CR_PACKAGE_FROM_XML; +- } +- +- return call_user_callback_if_package_finished(pkg, cb_data, err); +-} +- +-static gboolean +-parse_next_section(CR_FILE *target_file, const char *path, cr_ParserData *pd, GError **err) +-{ +- char buf[XML_BUFFER_SIZE]; +- GError *tmp_err = NULL; +- int parsed_len = cr_read(target_file, buf, XML_BUFFER_SIZE, &tmp_err); +- if (tmp_err) { +- g_critical("%s: Error while reading xml '%s': %s", __func__, path, tmp_err->message); +- g_propagate_prefixed_error(err, tmp_err, "Read error: "); +- return FALSE; +- } +- int done = parsed_len == 0; +- if (xmlParseChunk(pd->parser, buf, parsed_len, done)) { +- xmlErrorPtr xml_err = xmlCtxtGetLastError(pd->parser); +- g_critical("%s: parsing error '%s': %s", __func__, path, +- (xml_err) ? xml_err->message : "UNKNOWN_ERROR"); +- g_set_error(err, ERR_DOMAIN, CRE_XMLPARSER, +- "Parse error '%s' at line: %d (%s)", +- path, +- (xml_err) ? (int) xml_err->line : 0, +- (xml_err) ? (char *) xml_err->message : "UNKNOWN_ERROR"); +- return FALSE; +- } +- +- if (pd->err) { +- g_propagate_error(err, pd->err); +- return FALSE; +- } +- +- return done; +-} +- +-//TODO(amatej): there is quite some overlap with this and cr_load_xml_files, +-// we could use this api and just wrap it in cr_loax_xml_files? +-int cr_xml_parse_main_metadata_together(const char *primary_path, +- const char *filelists_path, +- const char *other_path, +- cr_XmlParserNewPkgCb newpkgcb, +- void *newpkgcb_data, +- cr_XmlParserPkgCb pkgcb, +- void *pkgcb_data, +- cr_XmlParserWarningCb warningcb, +- void *warningcb_data, +- gboolean allow_out_of_order, +- GError **err) +-{ +- int ret = CRE_OK; +- CR_FILE *primary_f = NULL; +- CR_FILE *filelists_f = NULL; +- CR_FILE *other_f = NULL; +- GError *tmp_err = NULL; +- +- cr_CbData cbdata; +- cbdata.in_progress_pkgs_list = NULL; +- cbdata.in_progress_pkgs_hash = NULL; +- cbdata.newpkgcb = newpkgcb; +- cbdata.newpkgcb_data = newpkgcb_data; +- cbdata.pkgcb = pkgcb; +- cbdata.pkgcb_data = pkgcb_data; +- +- if (allow_out_of_order) { +- cbdata.in_progress_pkgs_hash = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); +- } +- +- assert(primary_path); +- assert(filelists_path); +- assert(other_path); +- assert(newpkgcb || pkgcb); +- assert(!err || *err == NULL); +- +- cr_ParserData *primary_pd = NULL; +- cr_ParserData *filelists_pd = NULL; +- cr_ParserData *other_pd = NULL; +- +- primary_f = cr_open(primary_path, CR_CW_MODE_READ, CR_CW_AUTO_DETECT_COMPRESSION, &tmp_err); +- if (tmp_err) { +- ret = tmp_err->code; +- g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", primary_path); +- g_clear_error(&tmp_err); +- goto out; +- } +- filelists_f = cr_open(filelists_path, CR_CW_MODE_READ, CR_CW_AUTO_DETECT_COMPRESSION, &tmp_err); +- if (tmp_err) { +- ret = tmp_err->code; +- g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", filelists_path); +- g_clear_error(&tmp_err); +- goto out; +- } +- other_f = cr_open(other_path, CR_CW_MODE_READ, CR_CW_AUTO_DETECT_COMPRESSION, &tmp_err); +- if (tmp_err) { +- ret = tmp_err->code; +- g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", other_path); +- g_clear_error(&tmp_err); +- goto out; +- } +- +- //TODO(amatej): In the future we could make filelists/other optional if there is a need for it. That would mean we +- // should replace the last 0 in primary_parser_data_new depending on whether we have filelists or not. +- primary_pd = primary_parser_data_new(newpkgcb_primary, &cbdata, pkgcb_primary, &cbdata, warningcb, warningcb_data, 0); +- filelists_pd = filelists_parser_data_new(newpkgcb_filelists, &cbdata, pkgcb_filelists, &cbdata, warningcb, warningcb_data); +- other_pd = other_parser_data_new(newpkgcb_other, &cbdata, pkgcb_other, &cbdata, warningcb, warningcb_data); +- +- gboolean primary_is_done = 0; +- gboolean filelists_is_done = 0; +- gboolean other_is_done = 0; +- cbdata.in_progress_count_primary = 0; +- cbdata.in_progress_count_filelists = 0; +- cbdata.in_progress_count_other = 0; +- while (!primary_is_done || !filelists_is_done || !other_is_done) { +- while ((cbdata.in_progress_count_primary <= cbdata.in_progress_count_filelists || +- cbdata.in_progress_count_primary <= cbdata.in_progress_count_other) && +- !primary_is_done) +- { +- primary_is_done = parse_next_section(primary_f, primary_path, primary_pd, err); +- if (*err) { +- ret = (*err)->code; +- goto out; +- } +- +- } +- +- while ((cbdata.in_progress_count_filelists <= cbdata.in_progress_count_primary || +- cbdata.in_progress_count_filelists <= cbdata.in_progress_count_other) && +- !filelists_is_done) +- { +- filelists_is_done = parse_next_section(filelists_f, filelists_path, filelists_pd, err); +- if (*err) { +- ret = (*err)->code; +- goto out; +- } +- } +- +- while ((cbdata.in_progress_count_other <= cbdata.in_progress_count_filelists || +- cbdata.in_progress_count_other <= cbdata.in_progress_count_primary) && +- !other_is_done) +- { +- other_is_done = parse_next_section(other_f, other_path, other_pd, err); +- if (*err) { +- ret = (*err)->code; +- goto out; +- } +- } +- } +- +-out: +- if (ret != CRE_OK) { +- // An error already encountered +- // just close the file without error checking +- cr_close(primary_f, NULL); +- cr_close(filelists_f, NULL); +- cr_close(other_f, NULL); +- } else { +- // No error encountered yet +- cr_close(primary_f, &tmp_err); +- if (!tmp_err) +- cr_close(filelists_f, &tmp_err); +- if (!tmp_err) +- cr_close(other_f, &tmp_err); +- if (tmp_err) { +- ret = tmp_err->code; +- g_propagate_prefixed_error(err, tmp_err, "Error while closing: "); +- } +- } +- +- cr_xml_parser_data_free(primary_pd); +- cr_xml_parser_data_free(filelists_pd); +- cr_xml_parser_data_free(other_pd); +- +- if (allow_out_of_order) { +- g_hash_table_destroy(cbdata.in_progress_pkgs_hash); +- } else { +- cr_slist_free_full(cbdata.in_progress_pkgs_list, (GDestroyNotify) cr_package_free); +- } +- +- return ret; +-} +-- +2.35.1 + diff --git a/SPECS/createrepo_c.spec b/SPECS/createrepo_c.spec index cb35469..eb637ba 100644 --- a/SPECS/createrepo_c.spec +++ b/SPECS/createrepo_c.spec @@ -10,27 +10,34 @@ %bcond_without drpm %endif -%if 0%{?rhel} || (0%{?fedora} && 0%{?fedora} < 29) +%if 0%{?rhel} %bcond_with zchunk %else %bcond_without zchunk %endif -%if (0%{?rhel} && 0%{?rhel} < 8) || (0%{?fedora} && 0%{?fedora} < 29) +%if 0%{?rhel} && 0%{?rhel} < 8 %bcond_with libmodulemd %else %bcond_without libmodulemd %endif +%if 0%{?rhel} && 0%{?rhel} <= 8 +%bcond_without legacy_hashes +%else +%bcond_with legacy_hashes +%endif + Summary: Creates a common metadata repository Name: createrepo_c -Version: 0.17.2 -Release: 3%{?dist} +Version: 0.17.7 +Release: 5%{?dist} License: GPLv2+ URL: https://github.com/rpm-software-management/createrepo_c Source0: %{url}/archive/%{version}/%{name}-%{version}.tar.gz -Patch1: 0001-Fix-cr_compress_file_with_stat-Memory-leak.patch -Patch2: 0002-Preserve-changed-API-for-cr_compress_file_with_stat-RhBug1973588.patch +Patch1: 0001-Preserve-changed-API-for-cr_compress_file_with_stat-RhBug1973588.patch +Patch2: 0002-Default---keep-all-metadata-to-TRUE-and-add---discard-additional-metadata.patch +Patch3: 0003-Revert-added-API-for-parsing-main-metadata-together-RhBug2062299.patch BuildRequires: cmake BuildRequires: gcc @@ -109,6 +116,7 @@ pushd build-py3 %cmake .. \ -DWITH_ZCHUNK=%{?with_zchunk:ON}%{!?with_zchunk:OFF} \ -DWITH_LIBMODULEMD=%{?with_libmodulemd:ON}%{!?with_libmodulemd:OFF} \ + -DWITH_LEGACY_HASHES=%{?with_legacy_hashes:ON}%{!?with_legacy_hashes:OFF} \ -DENABLE_DRPM=%{?with_drpm:ON}%{!?with_drpm:OFF} make %{?_smp_mflags} RPM_OPT_FLAGS="%{optflags}" # Build C documentation @@ -177,6 +185,26 @@ ln -sr %{buildroot}%{_bindir}/modifyrepo_c %{buildroot}%{_bindir}/modifyrepo %{python3_sitearch}/%{name}-%{version}-py%{python3_version}.egg-info %changelog +* Fri Mar 11 2022 Ales Matej - 0.17.7-5 +- Revert addition of new API for parsing main metadata together (RhBug:2062299) + +* Wed Feb 16 2022 Pavla Kratochvilova - 0.17.7-4 +- Switch default of --keep-all-metadata to TRUE and add --discard-additional-metadata (RhBug:1992209) + +* Mon Nov 29 2021 Pavla Kratochvilova - 0.17.7-3 +- Fix memory leak of `tmp_err` (RhBug:2005781) + +* Mon Nov 15 2021 Pavla Kratochvilova - 0.17.7-2 +- Build with legacy hashes (RhBug:2022271) + +* Tue Nov 09 2021 Pavla Kratochvilova - 0.17.7-1 +- Update to 0.17.7 +- Remove insecure hashes SHA-1 and MD5 from the default build (RhBug:1935486) +- Fix error when updating repo with removed modules metadata +- Exit with status code 1 when loading of repo's metadata fails +- Fix memory leaks (RhBug:1998426) +- Fix valgrind warnings caused by subprocess calls + * Wed Sep 15 2021 Pavla Kratochvilova - 0.17.2-3 - Preserve changed API for cr_compress_file_with_stat (RhBug:1973588)