From 55de6f0be10d96b56de214b677ed185bf12d15ee Mon Sep 17 00:00:00 2001

From: Matthew Almond <malmond@fb.com>

Date: Fri, 8 Nov 2019 09:29:43 -0800

Subject: [PATCH] RPM with Copy on Write

This is part of https://fedoraproject.org/wiki/Changes/RPMCoW

The majority of changes are in two new programs:

= rpm2extents

Modeled as a 'stream processor'. It reads a regular .rpm file on stdin,

and produces a modified .rpm file on stdout. The lead, signature and

headers are preserved 1:1 to allow all the normal metadata inspection,

signature verification to work as expected. Only the 'payload' is

modified.

The primary motivation for this tool is to re-organize the payload as a

sequence of raw file extents (hence the name). The files are organized

by their digest identity instead of path/filename. If any digest is

repeated, then the file is skipped/de-duped. Only regular files are

represented. All other entries like directories, symlinks, devices are

fully described in the headers and are omitted.

The files are padded so they start on `sysconf(_SC_PAGESIZE)` boundries

to permit 'reflink' syscalls to work in the `reflink` plugin.

At the end of the file is a footer with 3 sections:

1. List of calculated digests of the input stream. This is used in

   `librepo` because the file *written* is a derivative, and not the

   same as the repo metadata describes. `rpm2extents` takes one or more

   positional arguments that described which digest algorithms are

   desired. This is often just `SHA256`. This program is only measuring

   and recording the digest - it does not express an opinion on whether

   the file is correct. Due to the API on most compression libraries

   directly reading the source file, the whole file digest is measured

   using a subprocess and pipes. I don't love it, but it works.

2. Sorted List of file content digests + offset pairs. This is used in

   the plugin with a trivial binary search to locate the start of file

   content. The size is not needed because it's part of normal headers.

3. (offset of 1., offset of 2., 8 byte MAGIC value) triple

= reflink plugin

Looks for the 8 byte magic value at the end of the rpm file. If present

it alters the `RPMTAG_PAYLOADFORMAT` in memory to `clon`, and reads in

the digest-> offset table.

`rpmPackageFilesInstall()` in `fsm.c` is

modified to alter the enumeration strategy from

`rpmfiNewArchiveReader()` to `rpmfilesIter()` if not `cpio`. This is

needed because there is no cpio to enumerate. In the same function, if

`rpmpluginsCallFsmFilePre()` returns `RPMRC_PLUGIN_CONTENTS` then

`fsmMkfile()` is skipped as it is assumed the plugin did the work.

The majority of the work is in `reflink_fsm_file_pre()` - the per file

hook for RPM plugins. If the file enumerated in

`rpmPackageFilesInstall()` is a regular file, this function will look up

the offset in the digest->offset table and will try to reflink it, then

fall back to a regular copy. If reflinking does work: we will have

reflinked a whole number of pages, so we truncate the file to the

expected size. Therefore installing most files does involve two writes:

the reflink of the full size, then a fork/copy on write for the last

page worth.

If the file passed to `reflink_fsm_file_pre()` is anything other than a

regular file, it return `RPMRC_OK` so the normal mechanics of

`rpmPackageFilesInstall()` are used. That handles directories, symlinks

and other non file types.

= New API for internal use

1. `rpmReadPackageRaw()` is used within `rpm2extents` to read all the

   headers without trying to validate signatures. This eliminates the

   runtime dependency on rpmdb.

2. `rpmteFd()` exposes the Fd behind the rpmte, so plugins can interact

   with the rpm itself.

3. `RPMRC_PLUGIN_CONTENTS` in `rpmRC_e` for use in

   `rpmpluginsCallFsmFilePre()` specifically.

4. `pgpStringVal()` is used to help parse the command line in

   `rpm2extents` - the positional arguments are strings, and this

   converts the values back to the values in the table.

Nothing has been removed, and none of the changes are intended to be

used externally, so I don't think a soname bump is warranted here.

---

 Makefile.am               |   8 +-

 build/pack.c              |   2 +-

 lib/Makefile.am           |   3 +-

 lib/Makefile.in           |  51 +--

 lib/fsm.c                 |  45 ++-

 lib/package.c             |  36 ++

 lib/rpmchecksig.c         | 117 +++++--

 lib/rpmcli.h              |  10 +

 lib/rpmextents.c          | 110 ++++++

 lib/rpmextents_internal.h |  58 ++++

 lib/rpmlead.c             |  43 ++-

 lib/rpmlead.h             |  37 +-

 lib/rpmlib.h              |   9 +

 lib/rpmplugin.h           |   9 +

 lib/rpmplugins.c          |  92 ++++-

 lib/rpmplugins.h          |  17 +

 lib/rpmte.c               |   5 +

 lib/rpmte.h               |   2 +

 lib/rpmtypes.h            |   3 +-

 lib/transaction.c         |  29 +-

 macros.in                 |   1 +

 plugins/Makefile.am       |   4 +

 plugins/reflink.c         | 401 +++++++++++++++++++++

 rpm2extents.c             | 708 ++++++++++++++++++++++++++++++++++++++

 rpmio/rpmpgp.c            |  10 +

 rpmio/rpmpgp.h            |   9 +

 scripts/rpm2extents_dump  |  94 +++++

 sign/rpmgensig.c          |   2 +-

 tests/Makefile.am         |   1 +

 tests/atlocal.in          |  22 ++

 tests/rpm2extents.at      | 151 ++++++++

 tests/rpmtests.at         |   1 +

 32 files changed, 1996 insertions(+), 94 deletions(-)

 create mode 100644 lib/rpmextents.c

 create mode 100644 lib/rpmextents_internal.h

 create mode 100644 plugins/reflink.c

 create mode 100644 rpm2extents.c

 create mode 100755 scripts/rpm2extents_dump

 create mode 100644 tests/rpm2extents.at

diff --git a/Makefile.am b/Makefile.am

index 78bc131..43fbd94 100644

--- a/Makefile.am

+++ b/Makefile.am

@@ -106,7 +106,7 @@ pkginclude_HEADERS += build/rpmfc.h

 pkginclude_HEADERS += build/rpmspec.h

-bin_PROGRAMS =		rpm rpm2cpio rpmbuild rpmdb rpmkeys rpmsign rpmspec

+bin_PROGRAMS =		rpm rpm2cpio rpmbuild rpmdb rpmkeys rpmsign rpmspec rpm2extents

 if WITH_ARCHIVE

 bin_PROGRAMS += 	rpm2archive 

 endif

@@ -160,6 +160,10 @@ rpm2cpio_SOURCES =	rpm2cpio.c debug.h system.h

 rpm2cpio_LDADD =	lib/librpm.la rpmio/librpmio.la

 rpm2cpio_LDADD +=	@WITH_POPT_LIB@

+rpm2extents_SOURCES =	rpm2extents.c debug.h system.h

+rpm2extents_LDADD =	lib/librpm.la rpmio/librpmio.la

+rpm2extents_LDADD +=	@WITH_POPT_LIB@

+

 rpm2archive_SOURCES =	rpm2archive.c debug.h system.h

 rpm2archive_LDADD =	lib/librpm.la rpmio/librpmio.la

 rpm2archive_LDADD +=	@WITH_POPT_LIB@ @WITH_ARCHIVE_LIB@

@@ -199,7 +203,7 @@ bin_PROGRAMS +=		rpmgraph

 rpmgraph_SOURCES =	tools/rpmgraph.c

 rpmgraph_LDADD =	lib/librpm.la rpmio/librpmio.la @WITH_POPT_LIB@

-dist_bin_SCRIPTS =	scripts/gendiff

+dist_bin_SCRIPTS =	scripts/gendiff scripts/rpm2extents_dump

 rpmconfig_DATA = rpmrc

 rpmrc: $(top_srcdir)/rpmrc.in

diff --git a/build/pack.c b/build/pack.c

index 8d6f749..e2f05b6 100644

--- a/build/pack.c

+++ b/build/pack.c

@@ -493,7 +493,7 @@ static rpmRC writeRPM(Package pkg, unsigned char ** pkgidp,

     }

     /* Write the lead section into the package. */

-    if (rpmLeadWrite(fd, pkg->header)) {

+    if (rpmLeadWriteFromHeader(fd, pkg->header)) {

 	rpmlog(RPMLOG_ERR, _("Unable to write package: %s\n"), Fstrerror(fd));

 	goto exit;

     }

diff --git a/lib/Makefile.am b/lib/Makefile.am

index c561ad5..4a83e9a 100644

--- a/lib/Makefile.am

+++ b/lib/Makefile.am

@@ -41,7 +41,8 @@ librpm_la_SOURCES = \

 	rpmscript.h rpmscript.c \

 	rpmchroot.c rpmchroot.h \

 	rpmplugins.c rpmplugins.h rpmplugin.h rpmug.c rpmug.h \

-	rpmtriggers.h rpmtriggers.c rpmvs.c rpmvs.h

+	rpmtriggers.h rpmtriggers.c rpmvs.c rpmvs.h \

+	rpmextents.c rpmextents_internal.h

 librpm_la_LDFLAGS = -version-info $(rpm_version_info)

diff --git a/lib/Makefile.in b/lib/Makefile.in

index 1aafaac..bf911fa 100644

--- a/lib/Makefile.in

+++ b/lib/Makefile.in

@@ -173,10 +173,11 @@ am__librpm_la_SOURCES_DIST = backend/dbi.c backend/dbi.h \

 	rpmlock.c rpmlock.h misc.h relocation.c rpmscript.h \

 	rpmscript.c rpmchroot.c rpmchroot.h rpmplugins.c rpmplugins.h \

 	rpmplugin.h rpmug.c rpmug.h rpmtriggers.h rpmtriggers.c \

-	rpmvs.c rpmvs.h backend/db3.c backend/bdb_ro.c \

-	backend/ndb/glue.c backend/ndb/rpmpkg.c backend/ndb/rpmpkg.h \

-	backend/ndb/rpmidx.c backend/ndb/rpmidx.h backend/ndb/rpmxdb.c \

-	backend/ndb/rpmxdb.h backend/sqlite.c

+	rpmvs.c rpmvs.h rpmextents.c rpmextents_internal.h \

+	backend/db3.c backend/bdb_ro.c backend/ndb/glue.c \

+	backend/ndb/rpmpkg.c backend/ndb/rpmpkg.h backend/ndb/rpmidx.c \

+	backend/ndb/rpmidx.h backend/ndb/rpmxdb.c backend/ndb/rpmxdb.h \

+	backend/sqlite.c

 am__dirstamp = $(am__leading_dot)dirstamp

 @BDB_TRUE@am__objects_1 = backend/db3.lo

 @BDB_RO_TRUE@am__objects_2 = backend/bdb_ro.lo

@@ -192,8 +193,8 @@ am_librpm_la_OBJECTS = backend/dbi.lo backend/dummydb.lo \

 	rpmlead.lo rpmps.lo rpmprob.lo rpmrc.lo rpmte.lo rpmts.lo \

 	rpmfs.lo signature.lo transaction.lo verify.lo rpmlock.lo \

 	relocation.lo rpmscript.lo rpmchroot.lo rpmplugins.lo rpmug.lo \

-	rpmtriggers.lo rpmvs.lo $(am__objects_1) $(am__objects_2) \

-	$(am__objects_3) $(am__objects_4)

+	rpmtriggers.lo rpmvs.lo rpmextents.lo $(am__objects_1) \

+	$(am__objects_2) $(am__objects_3) $(am__objects_4)

 librpm_la_OBJECTS = $(am_librpm_la_OBJECTS)

 AM_V_lt = $(am__v_lt_@AM_V@)

 am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)

@@ -228,21 +229,21 @@ am__depfiles_remade = ./$(DEPDIR)/cpio.Plo ./$(DEPDIR)/depends.Plo \

 	./$(DEPDIR)/relocation.Plo ./$(DEPDIR)/rpmal.Plo \

 	./$(DEPDIR)/rpmchecksig.Plo ./$(DEPDIR)/rpmchroot.Plo \

 	./$(DEPDIR)/rpmdb.Plo ./$(DEPDIR)/rpmds.Plo \

-	./$(DEPDIR)/rpmfi.Plo ./$(DEPDIR)/rpmfs.Plo \

-	./$(DEPDIR)/rpmgi.Plo ./$(DEPDIR)/rpminstall.Plo \

-	./$(DEPDIR)/rpmlead.Plo ./$(DEPDIR)/rpmlock.Plo \

-	./$(DEPDIR)/rpmplugins.Plo ./$(DEPDIR)/rpmprob.Plo \

-	./$(DEPDIR)/rpmps.Plo ./$(DEPDIR)/rpmrc.Plo \

-	./$(DEPDIR)/rpmscript.Plo ./$(DEPDIR)/rpmtd.Plo \

-	./$(DEPDIR)/rpmte.Plo ./$(DEPDIR)/rpmtriggers.Plo \

-	./$(DEPDIR)/rpmts.Plo ./$(DEPDIR)/rpmug.Plo \

-	./$(DEPDIR)/rpmvs.Plo ./$(DEPDIR)/signature.Plo \

-	./$(DEPDIR)/tagexts.Plo ./$(DEPDIR)/tagname.Plo \

-	./$(DEPDIR)/transaction.Plo ./$(DEPDIR)/verify.Plo \

-	backend/$(DEPDIR)/bdb_ro.Plo backend/$(DEPDIR)/db3.Plo \

-	backend/$(DEPDIR)/dbi.Plo backend/$(DEPDIR)/dbiset.Plo \

-	backend/$(DEPDIR)/dummydb.Plo backend/$(DEPDIR)/sqlite.Plo \

-	backend/ndb/$(DEPDIR)/glue.Plo \

+	./$(DEPDIR)/rpmextents.Plo ./$(DEPDIR)/rpmfi.Plo \

+	./$(DEPDIR)/rpmfs.Plo ./$(DEPDIR)/rpmgi.Plo \

+	./$(DEPDIR)/rpminstall.Plo ./$(DEPDIR)/rpmlead.Plo \

+	./$(DEPDIR)/rpmlock.Plo ./$(DEPDIR)/rpmplugins.Plo \

+	./$(DEPDIR)/rpmprob.Plo ./$(DEPDIR)/rpmps.Plo \

+	./$(DEPDIR)/rpmrc.Plo ./$(DEPDIR)/rpmscript.Plo \

+	./$(DEPDIR)/rpmtd.Plo ./$(DEPDIR)/rpmte.Plo \

+	./$(DEPDIR)/rpmtriggers.Plo ./$(DEPDIR)/rpmts.Plo \

+	./$(DEPDIR)/rpmug.Plo ./$(DEPDIR)/rpmvs.Plo \

+	./$(DEPDIR)/signature.Plo ./$(DEPDIR)/tagexts.Plo \

+	./$(DEPDIR)/tagname.Plo ./$(DEPDIR)/transaction.Plo \

+	./$(DEPDIR)/verify.Plo backend/$(DEPDIR)/bdb_ro.Plo \

+	backend/$(DEPDIR)/db3.Plo backend/$(DEPDIR)/dbi.Plo \

+	backend/$(DEPDIR)/dbiset.Plo backend/$(DEPDIR)/dummydb.Plo \

+	backend/$(DEPDIR)/sqlite.Plo backend/ndb/$(DEPDIR)/glue.Plo \

 	backend/ndb/$(DEPDIR)/rpmidx.Plo \

 	backend/ndb/$(DEPDIR)/rpmpkg.Plo \

 	backend/ndb/$(DEPDIR)/rpmxdb.Plo

@@ -586,8 +587,9 @@ librpm_la_SOURCES = backend/dbi.c backend/dbi.h backend/dummydb.c \

 	signature.h transaction.c verify.c rpmlock.c rpmlock.h misc.h \

 	relocation.c rpmscript.h rpmscript.c rpmchroot.c rpmchroot.h \

 	rpmplugins.c rpmplugins.h rpmplugin.h rpmug.c rpmug.h \

-	rpmtriggers.h rpmtriggers.c rpmvs.c rpmvs.h $(am__append_1) \

-	$(am__append_4) $(am__append_5) $(am__append_8)

+	rpmtriggers.h rpmtriggers.c rpmvs.c rpmvs.h rpmextents.c \

+	rpmextents_internal.h $(am__append_1) $(am__append_4) \

+	$(am__append_5) $(am__append_8)

 librpm_la_LDFLAGS = -version-info $(rpm_version_info)

 librpm_la_LIBADD = $(top_builddir)/rpmio/librpmio.la @WITH_POPT_LIB@ \

 	@WITH_CAP_LIB@ @WITH_ACL_LIB@ @LIBINTL@ $(am__append_2) \

@@ -748,6 +750,7 @@ distclean-compile:

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rpmchroot.Plo@am__quote@ # am--include-marker

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rpmdb.Plo@am__quote@ # am--include-marker

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rpmds.Plo@am__quote@ # am--include-marker

+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rpmextents.Plo@am__quote@ # am--include-marker

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rpmfi.Plo@am__quote@ # am--include-marker

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rpmfs.Plo@am__quote@ # am--include-marker

 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rpmgi.Plo@am__quote@ # am--include-marker

@@ -978,6 +981,7 @@ distclean: distclean-am

 	-rm -f ./$(DEPDIR)/rpmchroot.Plo

 	-rm -f ./$(DEPDIR)/rpmdb.Plo

 	-rm -f ./$(DEPDIR)/rpmds.Plo

+	-rm -f ./$(DEPDIR)/rpmextents.Plo

 	-rm -f ./$(DEPDIR)/rpmfi.Plo

 	-rm -f ./$(DEPDIR)/rpmfs.Plo

 	-rm -f ./$(DEPDIR)/rpmgi.Plo

@@ -1077,6 +1081,7 @@ maintainer-clean: maintainer-clean-am

 	-rm -f ./$(DEPDIR)/rpmchroot.Plo

 	-rm -f ./$(DEPDIR)/rpmdb.Plo

 	-rm -f ./$(DEPDIR)/rpmds.Plo

+	-rm -f ./$(DEPDIR)/rpmextents.Plo

 	-rm -f ./$(DEPDIR)/rpmfi.Plo

 	-rm -f ./$(DEPDIR)/rpmfs.Plo

 	-rm -f ./$(DEPDIR)/rpmgi.Plo

diff --git a/lib/fsm.c b/lib/fsm.c

index 9dd50b7..e562220 100644

--- a/lib/fsm.c

+++ b/lib/fsm.c

@@ -868,6 +868,24 @@ static rpmfi fsmIterFini(rpmfi fi, struct diriter_s *di)

     return rpmfiFree(fi);

 }

+static int fiIterator(rpmPlugins plugins, FD_t payload, rpmfiles files, rpmfi *fi)

+{

+    rpmRC plugin_rc = rpmpluginsCallFsmFileArchiveReader(plugins, payload, files, fi);

+    switch (plugin_rc) {

+	case RPMRC_PLUGIN_CONTENTS:

+	    if (*fi == NULL)

+                return RPMERR_BAD_MAGIC;

+            return RPMRC_OK;

+	case RPMRC_OK:

+	    *fi = rpmfiNewArchiveReader(payload, files, RPMFI_ITER_READ_ARCHIVE);

+	    if (*fi == NULL)

+                return RPMERR_BAD_MAGIC;

+            return RPMRC_OK;

+	default:

+            return RPMRC_FAIL;

+    }

+}

+

 int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files,

               rpmpsm psm, char ** failedFile)

 {

@@ -919,8 +937,9 @@ int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files,

     if (rc)

 	goto exit;

-    fi = fsmIter(payload, files,

-		 payload ? RPMFI_ITER_READ_ARCHIVE : RPMFI_ITER_FWD, &di);

+    rc = fiIterator(plugins, payload, files, &fi);

+    if (rc)

+        goto exit;

     if (fi == NULL) {

         rc = RPMERR_BAD_MAGIC;

@@ -943,6 +962,9 @@ int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files,

         if (!fp->skip) {

 	    int mayopen = 0;

 	    int fd = -1;

+

+	    if (di.dirfd >= 0)

+		fsmClose(&di.dirfd);

 	    rc = ensureDir(plugins, rpmfiDN(fi), 0,

 			    (fp->action == FA_CREATE), 0, &di.dirfd);

@@ -952,9 +974,10 @@ int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files,

 	    }

 	    /* Run fsm file pre hook for all plugins */

-	    if (!rc)

+	    if (!rc) {

 		rc = rpmpluginsCallFsmFilePre(plugins, fi, fp->fpath,

 					      fp->sb.st_mode, fp->action);

+	    }

 	    if (rc)

 		goto setmeta; /* for error notification */

@@ -982,11 +1005,18 @@ int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files,

 	    if (fp->action == FA_TOUCH)

 		goto setmeta;

-            if (S_ISREG(fp->sb.st_mode)) {

+	    rpmRC plugin_rc = rpmpluginsCallFsmFileInstall(plugins, fi, fp->fpath, fp->sb.st_mode, fp->action);

+	    if (!(plugin_rc == RPMRC_PLUGIN_CONTENTS || plugin_rc == RPMRC_OK)){

+		rc = plugin_rc;

+	    } else if(plugin_rc == RPMRC_PLUGIN_CONTENTS){

+		rc = RPMRC_OK;

+		/* The reflink plugins handles hardlink differently, metadata has to be set. */

+		fp->setmeta = 1;

+	    } else if (S_ISREG(fp->sb.st_mode)) {

 		if (rc == RPMERR_ENOENT) {

 		    rc = fsmMkfile(di.dirfd, fi, fp, files, psm, nodigest,

-				   &firstlink, &firstlinkfile, &di.firstdir,

-				   &fd;;

+				   &firstlink, &firstlinkfile,

+				   &di.firstdir, &fd;;

 		}

             } else if (S_ISDIR(fp->sb.st_mode)) {

                 if (rc == RPMERR_ENOENT) {

@@ -1055,10 +1085,13 @@ setmeta:

     /* If all went well, commit files to final destination */

     fi = fsmIter(NULL, files, RPMFI_ITER_FWD, &di);

+

     while (!rc && (fx = rpmfiNext(fi)) >= 0) {

 	struct filedata_s *fp = &fdata[fx];

 	if (!fp->skip) {

+	    if (di.dirfd >= 0)

+		fsmClose(&di.dirfd);

 	    if (!rc)

 		rc = ensureDir(NULL, rpmfiDN(fi), 0, 0, 0, &di.dirfd);

diff --git a/lib/package.c b/lib/package.c

index 8c2b66b..fb7ec7b 100644

--- a/lib/package.c

+++ b/lib/package.c

@@ -402,5 +402,41 @@ exit:

     return rc;

 }

+rpmRC rpmReadPackageRaw(FD_t fd, Header * sigp, Header * hdrp)

+{

+    char *msg = NULL;

+    hdrblob sigblob = hdrblobCreate();

+    hdrblob blob = hdrblobCreate();

+    Header h = NULL;

+    Header sigh = NULL;

+

+    rpmRC rc = hdrblobRead(fd, 1, 0, RPMTAG_HEADERSIGNATURES, sigblob, &msg;;

+    if (rc != RPMRC_OK)

+	goto exit;

+

+    rc = hdrblobRead(fd, 1, 1, RPMTAG_HEADERIMMUTABLE, blob, &msg;;

+    if (rc != RPMRC_OK)

+	goto exit;

+

+    rc = hdrblobImport(sigblob, 0, &sigh, &msg;;

+    if (rc)

+	goto exit;

+

+    rc = hdrblobImport(blob, 0, &h, &msg;;

+    if (rc)

+	goto exit;

+

+    *sigp = headerLink(sigh);

+    *hdrp = headerLink(h);

+exit:

+    if (rc != RPMRC_OK && msg)

+	rpmlog(RPMLOG_ERR, "%s: %s\n", Fdescr(fd), msg);

+    hdrblobFree(sigblob);

+    hdrblobFree(blob);

+    headerFree(sigh);

+    headerFree(h);

+    free(msg);

+    return rc;

+}

diff --git a/lib/rpmchecksig.c b/lib/rpmchecksig.c

index 40a3ab8..7f85615 100644

--- a/lib/rpmchecksig.c

+++ b/lib/rpmchecksig.c

@@ -20,6 +20,7 @@

 #include "rpmio/rpmio_internal.h" 	/* fdSetBundle() */

 #include "lib/rpmlead.h"

 #include "lib/header_internal.h"

+#include "lib/rpmextents_internal.h"

 #include "lib/rpmvs.h"

 #include "debug.h"

@@ -221,36 +222,24 @@ exit:

 }

 static int rpmpkgVerifySigs(rpmKeyring keyring, int vfylevel, rpmVSFlags flags,

-			   FD_t fd, const char *fn)

+			   FD_t fd, rpmsinfoCb cb, void *cbdata)

 {

     char *msg = NULL;

-    struct vfydata_s vd = { .seen = 0,

-			    .bad = 0,

-			    .verbose = rpmIsVerbose(),

-    };

     int rc;

-    struct rpmvs_s *vs = rpmvsCreate(vfylevel, flags, keyring);

-    rpmlog(RPMLOG_NOTICE, "%s:%s", fn, vd.verbose ? "\n" : "");

+

+    if(isTranscodedRpm(fd) == RPMRC_OK){

+	return extentsVerifySigs(fd, 1);

+    }

+

+    struct rpmvs_s *vs = rpmvsCreate(vfylevel, flags, keyring);

     rc = rpmpkgRead(vs, fd, NULL, NULL, &msg;;

     if (rc)

 	goto exit;

-    rc = rpmvsVerify(vs, RPMSIG_VERIFIABLE_TYPE, vfyCb, &vd);

-

-    if (!vd.verbose) {

-	if (vd.seen & RPMSIG_DIGEST_TYPE) {

-	    rpmlog(RPMLOG_NOTICE, " %s", (vd.bad & RPMSIG_DIGEST_TYPE) ?

-					_("DIGESTS") : _("digests"));

-	}

-	if (vd.seen & RPMSIG_SIGNATURE_TYPE) {

-	    rpmlog(RPMLOG_NOTICE, " %s", (vd.bad & RPMSIG_SIGNATURE_TYPE) ?

-					_("SIGNATURES") : _("signatures"));

-	}

-	rpmlog(RPMLOG_NOTICE, " %s\n", rc ? _("NOT OK") : _("OK"));

-    }

+    rc = rpmvsVerify(vs, RPMSIG_VERIFIABLE_TYPE, cb, cbdata);

 exit:

     if (rc && msg)

@@ -260,15 +249,39 @@ exit:

     return rc;

 }

+static void rpmkgVerifySigsPreLogging(struct vfydata_s *vd, const char *fn){

+    rpmlog(RPMLOG_NOTICE, "%s:%s", fn, vd->verbose ? "\n" : "");

+}

+

+static void rpmkgVerifySigsPostLogging(struct vfydata_s *vd, int rc){

+    if (!vd->verbose) {

+	if (vd->seen & RPMSIG_DIGEST_TYPE) {

+	    rpmlog(RPMLOG_NOTICE, " %s", (vd->bad & RPMSIG_DIGEST_TYPE) ?

+					_("DIGESTS") : _("digests"));

+	}

+	if (vd->seen & RPMSIG_SIGNATURE_TYPE) {

+	    rpmlog(RPMLOG_NOTICE, " %s", (vd->bad & RPMSIG_SIGNATURE_TYPE) ?

+					_("SIGNATURES") : _("signatures"));

+	}

+	rpmlog(RPMLOG_NOTICE, " %s\n", rc ? _("NOT OK") : _("OK"));

+    }

+}

+

 /* Wrapper around rpmkVerifySigs to preserve API */

 int rpmVerifySignatures(QVA_t qva, rpmts ts, FD_t fd, const char * fn)

 {

     int rc = 1; /* assume failure */

+    struct vfydata_s vd = { .seen = 0,

+			    .bad = 0,

+			    .verbose = rpmIsVerbose(),

+    };

     if (ts && qva && fd && fn) {

 	rpmKeyring keyring = rpmtsGetKeyring(ts, 1);

 	rpmVSFlags vsflags = rpmtsVfyFlags(ts);

 	int vfylevel = rpmtsVfyLevel(ts);

-	rc = rpmpkgVerifySigs(keyring, vfylevel, vsflags, fd, fn);

+	rpmkgVerifySigsPreLogging(&vd, fn);

+	rc = rpmpkgVerifySigs(keyring, vfylevel, vsflags, fd, vfyCb, &vd);

+	rpmkgVerifySigsPostLogging(&vd, rc);

     	rpmKeyringFree(keyring);

     }

     return rc;

@@ -290,12 +303,22 @@ int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv)

     while ((arg = *argv++) != NULL) {

 	FD_t fd = Fopen(arg, "r.ufdio");

+	struct vfydata_s vd = { .seen = 0,

+				.bad = 0,

+				.verbose = rpmIsVerbose(),

+	};

 	if (fd == NULL || Ferror(fd)) {

 	    rpmlog(RPMLOG_ERR, _("%s: open failed: %s\n"), 

 		     arg, Fstrerror(fd));

 	    res++;

-	} else if (rpmpkgVerifySigs(keyring, vfylevel, vsflags, fd, arg)) {

+	} else {

+	    rpmkgVerifySigsPreLogging(&vd, arg);

+	    int rc = rpmpkgVerifySigs(keyring, vfylevel, vsflags, fd,

+				      vfyCb, &vd);

+	    rpmkgVerifySigsPostLogging(&vd, rc);

+	    if (rc) {

 	    res++;

+	    }

 	}

 	Fclose(fd);

@@ -304,3 +327,53 @@ int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv)

     rpmKeyringFree(keyring);

     return res;

 }

+

+struct vfydatafd_s {

+    size_t len;

+    char msg[BUFSIZ];

+};

+

+

+static int vfyFDCb(struct rpmsinfo_s *sinfo, void *cbdata)

+{

+    struct vfydatafd_s *vd = cbdata;

+    char *vmsg, *msg;

+    size_t n;

+    size_t remainder = BUFSIZ - vd->len >= 0 ? BUFSIZ - vd->len : 0;

+

+    vmsg = rpmsinfoMsg(sinfo);

+    rasprintf(&msg, "    %s\n", vmsg);

+    n = rstrlcpy(vd->msg + vd->len, msg, remainder);

+    free(vmsg);

+    free(msg);

+    if(n <= remainder){

+	vd->len += n;

+    }

+    return 1;

+}

+

+

+int rpmcliVerifySignaturesFD(rpmts ts, FD_t fdi, char **msg)

+{

+    rpmRC rc = RPMRC_FAIL;

+    rpmKeyring keyring = rpmtsGetKeyring(ts, 1);

+    rpmVSFlags vsflags = rpmtsVfyFlags(ts);

+    int vfylevel = rpmtsVfyLevel(ts);

+    struct vfydatafd_s vd = {.len = 0};

+

+    vsflags |= rpmcliVSFlags;

+    if (rpmcliVfyLevelMask) {

+	vfylevel &= ~rpmcliVfyLevelMask;

+	rpmtsSetVfyLevel(ts, vfylevel);

+    }

+

+    if (!rpmpkgVerifySigs(keyring, vfylevel, vsflags, fdi, vfyFDCb, &vd)) {

+	rc = RPMRC_OK;

+    }

+    *msg = strdup(vd.msg);

+    rpmsqPoll();

+

+    rpmKeyringFree(keyring);

+    return rc;

+}

+

diff --git a/lib/rpmcli.h b/lib/rpmcli.h

index 3961418..450f7be 100644

--- a/lib/rpmcli.h

+++ b/lib/rpmcli.h

@@ -411,6 +411,16 @@ int rpmcliImportPubkeys(rpmts ts, ARGV_const_t argv);

  */

 int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv);

+

+/** \ingroup rpmcli

+ * Verify package signatures.

+ * @param ts		transaction set

+ * @param fd		a file descriptor to verify

+ * @param msg		a string containing textual information about the verification, similar to rpmcliVerifySignatures output.

+ * @return		0 on success

+ */

+int rpmcliVerifySignaturesFD(rpmts ts, FD_t fd, char **msg);

+

 #ifdef __cplusplus

 }

 #endif

diff --git a/lib/rpmextents.c b/lib/rpmextents.c

new file mode 100644

index 0000000..ac43264

--- /dev/null

+++ b/lib/rpmextents.c

@@ -0,0 +1,110 @@

+

+#include "system.h"

+

+#include <rpm/rpmlog.h>

+#include <rpm/rpmio.h>

+#include <string.h>

+#include <errno.h>

+

+

+#include "lib/rpmextents_internal.h"

+

+

+int extentsVerifySigs(FD_t fd, int print_content){

+    rpm_loff_t current;

+    int32_t rc;

+    size_t len;

+    uint64_t content_len;

+    char *content = NULL;

+    struct extents_footer_t footer;

+

+    current = Ftell(fd);

+

+    if(extentsFooterFromFD(fd, &footer) != RPMRC_OK) {

+	rc = -1;

+	goto exit;

+    }

+    if(Fseek(fd, footer.offsets.checksig_offset, SEEK_SET) < 0) {

+	rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to seek signature verification offset\n"));

+	rc = -1;

+	goto exit;

+    }

+    len = sizeof(rc);

+    if (Fread(&rc, len, 1, fd) != len) {

+	rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to read Signature Verification RC\n"));

+	rc = -1;

+	goto exit;

+    }

+

+    if(print_content) {

+	len = sizeof(content_len);

+	if (Fread(&content_len, len, 1, fd) != len) {

+	    rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to read signature content length\n"));

+	    goto exit;

+	}

+

+	content = rmalloc(content_len + 1);

+	if(content == NULL) {

+	    rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to allocate memory to read signature content\n"));

+	    goto exit;

+	}

+	content[content_len] = 0;

+	if (Fread(content, content_len, 1, fd) != content_len) {

+	    rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to read signature content\n"));

+	    goto exit;

+	}

+

+	rpmlog(RPMLOG_NOTICE, "%s", content);

+    }

+exit:

+    if(content){

+	rfree(content);

+    }

+    if (Fseek(fd, current, SEEK_SET) < 0) {

+	rpmlog(RPMLOG_ERR, _("extentsVerifySigs: unable to seek back to original location\n"));

+    }

+    return rc;

+

+}

+

+rpmRC extentsFooterFromFD(FD_t fd, struct extents_footer_t *footer) {

+

+    rpmRC rc = RPMRC_NOTFOUND;

+    rpm_loff_t current;

+    size_t len;

+

+    // If the file is not seekable, we cannot detect whether or not it is transcoded.

+    if(Fseek(fd, 0, SEEK_CUR) < 0) {

+        return RPMRC_FAIL;

+    }

+    current = Ftell(fd);