Compare SHA256 to check if versions are really the same

If we know both SHA256, and they're different, the packages are. This
approach stores the SHA256 only at runtime, avoiding the overhead of
storing it on-disk, because when we update repositories we update all
of them anyhow.

Note that pkgCacheGenerator is hidden, so we can just modify its
ABI, hooray.

Closes: #931175
LP: #2029268

Author: julian-klode

apt-pkg/deb/deblistparser.h

@@ -14,6 +14,9 @@
 #include <apt-pkg/pkgcache.h>
 #include <apt-pkg/pkgcachegen.h>
 #include <apt-pkg/tagfile.h>
+#ifdef APT_COMPILING_APT
+#include <apt-pkg/tagfile-keys.h>
+#endif
 
 #include <string>
 #include <vector>
@@ -90,6 +93,13 @@ class APT_HIDDEN debListParser : public pkgCacheListParser
 
    explicit debListParser(FileFd *File);
    virtual ~debListParser();
+
+#ifdef APT_COMPILING_APT
+   APT::StringView SHA256() const
+   {
+      return Section.Find(pkgTagSection::Key::SHA256);
+   }
+#endif
 };
 
 class APT_HIDDEN debDebFileParser : public debListParser

apt-pkg/pkgcachegen.cc

@@ -12,6 +12,7 @@
 #include <config.h>
 
 #include <apt-pkg/configuration.h>
+#include <apt-pkg/deblistparser.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/indexfile.h>
@@ -376,6 +377,12 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
    void const * oldMap = Map.Data();
 
    auto Hash = List.VersionHash();
+   APT::StringView ListSHA256;
+
+   bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
+   auto DebList = dynamic_cast<debListParser *>(&List);
+   if (DebList != nullptr)
+      ListSHA256 = DebList->SHA256();
    if (Ver.end() == false)
    {
       /* We know the list is sorted so we use that fact in the search.
@@ -392,8 +399,18 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
 	 // Versionstrings are equal - is hash also equal?
 	 if (Res == 0)
 	 {
-	    if (List.SameVersion(Hash, Ver) == true)
-	       break;
+	    if (List.SameVersion(Hash, Ver))
+	    {
+               // We do not have SHA256 for both, so we cannot compare them, trust the call from SameVersion()
+               if (ListSHA256.empty() || VersionExtra[Ver->ID].SHA256[0] == 0)
+                  break;
+               // We have SHA256 for both, so they must match.
+               if (ListSHA256 == APT::StringView(VersionExtra[Ver->ID].SHA256, 64))
+                  break;
+	       if (Debug)
+		  std::cerr << "Found differing SHA256 for " << Pkg.Name() << "=" << Version.to_string() << std::endl;
+	    }
+
 	    // sort (volatile) sources above not-sources like the status file
 	    if (CurrentFile == nullptr || (CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
 	    {
@@ -440,6 +457,8 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
    if (oldMap != Map.Data())
 	 LastVer = static_cast<map_pointer<pkgCache::Version> *>(Map.Data()) + (LastVer - static_cast<map_pointer<pkgCache::Version> const *>(oldMap));
    *LastVer = verindex;
+   if (ListSHA256.size() == 64)
+	 memcpy(VersionExtra[Ver->ID].SHA256, ListSHA256.data(), 64);
 
    if (unlikely(List.NewVersion(Ver) == false))
       return _error->Error(_("Error occurred while processing %s (%s%d)"),
@@ -869,6 +888,10 @@ map_pointer<pkgCache::Version> pkgCacheGenerator::NewVersion(pkgCache::VerIterat
    Ver->Hash = Hash;
    Ver->ID = Cache.HeaderP->VersionCount++;
 
+   // Allocate size for extra store
+   if (VersionExtra.size() <= Ver->ID)
+      VersionExtra.resize(Ver->ID + 1);
+
    // try to find the version string in the group for reuse
    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
    pkgCache::GrpIterator Grp = Pkg.Group();

apt-pkg/pkgcachegen.h

@@ -76,6 +76,12 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    std::unordered_set<string_pointer, hash> strSections;
 #endif
 
+   struct VersionExtra
+   {
+      char SHA256[64];
+   };
+   std::vector<VersionExtra> VersionExtra{32 * 1024};
+
    friend class pkgCacheListParser;
    typedef pkgCacheListParser ListParser;
 

test/integration/framework

@@ -1022,9 +1022,13 @@ insertpackage() {
 Priority: $PRIORITY
 Section: $SECTION
 Installed-Size: 42
-Size: 42
-SHA256: 0000000000000000000000000000000000000000000000000000000000000000
-Maintainer: Joe Sixpack <[email protected]>"
+Size: 42"
+                    if echo "$DEPENDENCIES" | grep -q SHA256:; then
+                        :
+                    else
+                        echo "SHA256: 0000000000000000000000000000000000000000000000000000000000000000"
+                    fi
+                    echo "Maintainer: Joe Sixpack <[email protected]>"
 					test "$arch" = 'none' || echo "Architecture: $arch"
 					echo "Version: $VERSION
 Filename: pool/${DISTSECTION}/${NAME}/${NAME}_${VERSION}_${arch}.deb"

test/integration/test-same-version-but-different

@@ -30,6 +30,9 @@ insertpackage 'testing' 'diff-size' 'all' '1' 'Size: 42'
 insertpackage 'unstable' 'diff-instsize' 'all' '1' 'Installed-Size: 21'
 insertpackage 'testing' 'diff-instsize' 'all' '1' 'Installed-Size: 42'
 
+insertpackage 'unstable' 'diff-sha256' 'all' '1' 'SHA256: 0000000000000000000000000000000000000000000000000000000000000000'
+insertpackage 'testing' 'diff-sha256' 'all' '1' 'SHA256: 0000000000000000000000000000000000000000000000000000000000000001'
+
 setupaptarchive
 
 APTARCHIVE="$(readlink -f ./aptarchive)"
@@ -96,3 +99,11 @@ testsuccessequal "diff-instsize:
         500 file:${APTARCHIVE} testing/main all Packages
      1 500
         500 file:${APTARCHIVE} unstable/main all Packages" apt policy diff-instsize
+testsuccessequal "diff-sha256:
+  Installed: (none)
+  Candidate: 1
+  Version table:
+     1 500
+        500 file:${APTARCHIVE} testing/main all Packages
+     1 500
+        500 file:${APTARCHIVE} unstable/main all Packages" apt policy diff-sha256