Merge branch 'same-version-sha256' into 'main'

Try to compare SHA256 to determine same version

See merge request apt-team/apt!305

Author: julian-klode

apt-pkg/deb/deblistparser.h

@@ -14,6 +14,9 @@
 #include <apt-pkg/pkgcache.h>
 #include <apt-pkg/pkgcachegen.h>
 #include <apt-pkg/tagfile.h>
+#ifdef APT_COMPILING_APT
+#include <apt-pkg/tagfile-keys.h>
+#endif
 
 #include <string>
 #include <vector>
@@ -90,6 +93,13 @@ class APT_HIDDEN debListParser : public pkgCacheListParser
 
    explicit debListParser(FileFd *File);
    virtual ~debListParser();
+
+#ifdef APT_COMPILING_APT
+   APT::StringView SHA256() const
+   {
+      return Section.Find(pkgTagSection::Key::SHA256);
+   }
+#endif
 };
 
 class APT_HIDDEN debDebFileParser : public debListParser

apt-pkg/pkgcachegen.cc

@@ -12,6 +12,7 @@
 #include <config.h>
 
 #include <apt-pkg/configuration.h>
+#include <apt-pkg/deblistparser.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/indexfile.h>
@@ -376,6 +377,12 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
    void const * oldMap = Map.Data();
 
    auto Hash = List.VersionHash();
+   APT::StringView ListSHA256;
+
+   bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
+   auto DebList = dynamic_cast<debListParser *>(&List);
+   if (DebList != nullptr)
+      ListSHA256 = DebList->SHA256();
    if (Ver.end() == false)
    {
       /* We know the list is sorted so we use that fact in the search.
@@ -392,8 +399,18 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
 	 // Versionstrings are equal - is hash also equal?
 	 if (Res == 0)
 	 {
-	    if (List.SameVersion(Hash, Ver) == true)
-	       break;
+	    if (List.SameVersion(Hash, Ver))
+	    {
+               // We do not have SHA256 for both, so we cannot compare them, trust the call from SameVersion()
+               if (ListSHA256.empty() || VersionExtra[Ver->ID].SHA256[0] == 0)
+                  break;
+               // We have SHA256 for both, so they must match.
+               if (ListSHA256 == APT::StringView(VersionExtra[Ver->ID].SHA256, 64))
+                  break;
+	       if (Debug)
+		  std::cerr << "Found differing SHA256 for " << Pkg.Name() << "=" << Version.to_string() << std::endl;
+	    }
+
 	    // sort (volatile) sources above not-sources like the status file
 	    if (CurrentFile == nullptr || (CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
 	    {
@@ -440,6 +457,8 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
    if (oldMap != Map.Data())
 	 LastVer = static_cast<map_pointer<pkgCache::Version> *>(Map.Data()) + (LastVer - static_cast<map_pointer<pkgCache::Version> const *>(oldMap));
    *LastVer = verindex;
+   if (ListSHA256.size() == 64)
+	 memcpy(VersionExtra[Ver->ID].SHA256, ListSHA256.data(), 64);
 
    if (unlikely(List.NewVersion(Ver) == false))
       return _error->Error(_("Error occurred while processing %s (%s%d)"),
@@ -869,6 +888,10 @@ map_pointer<pkgCache::Version> pkgCacheGenerator::NewVersion(pkgCache::VerIterat
    Ver->Hash = Hash;
    Ver->ID = Cache.HeaderP->VersionCount++;
 
+   // Allocate size for extra store
+   if (VersionExtra.size() <= Ver->ID)
+      VersionExtra.resize(Ver->ID + 1);
+
    // try to find the version string in the group for reuse
    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
    pkgCache::GrpIterator Grp = Pkg.Group();

apt-pkg/pkgcachegen.h

@@ -76,6 +76,12 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    std::unordered_set<string_pointer, hash> strSections;
 #endif
 
+   struct VersionExtra
+   {
+      char SHA256[64];
+   };
+   std::vector<VersionExtra> VersionExtra{32 * 1024};
+
    friend class pkgCacheListParser;
    typedef pkgCacheListParser ListParser;
 

test/integration/framework

@@ -1022,9 +1022,13 @@ insertpackage() {
 Priority: $PRIORITY
 Section: $SECTION
 Installed-Size: 42
-Size: 42
-SHA256: 0000000000000000000000000000000000000000000000000000000000000000
-Maintainer: Joe Sixpack <[email protected]>"
+Size: 42"
+                    if echo "$DEPENDENCIES" | grep -q SHA256:; then
+                        :
+                    else
+                        echo "SHA256: 0000000000000000000000000000000000000000000000000000000000000000"
+                    fi
+                    echo "Maintainer: Joe Sixpack <[email protected]>"
 					test "$arch" = 'none' || echo "Architecture: $arch"
 					echo "Version: $VERSION
 Filename: pool/${DISTSECTION}/${NAME}/${NAME}_${VERSION}_${arch}.deb"

test/integration/test-same-version-but-different

@@ -30,6 +30,9 @@ insertpackage 'testing' 'diff-size' 'all' '1' 'Size: 42'
 insertpackage 'unstable' 'diff-instsize' 'all' '1' 'Installed-Size: 21'
 insertpackage 'testing' 'diff-instsize' 'all' '1' 'Installed-Size: 42'
 
+insertpackage 'unstable' 'diff-sha256' 'all' '1' 'SHA256: 0000000000000000000000000000000000000000000000000000000000000000'
+insertpackage 'testing' 'diff-sha256' 'all' '1' 'SHA256: 0000000000000000000000000000000000000000000000000000000000000001'
+
 setupaptarchive
 
 APTARCHIVE="$(readlink -f ./aptarchive)"
@@ -96,3 +99,11 @@ testsuccessequal "diff-instsize:
         500 file:${APTARCHIVE} testing/main all Packages
      1 500
         500 file:${APTARCHIVE} unstable/main all Packages" apt policy diff-instsize
+testsuccessequal "diff-sha256:
+  Installed: (none)
+  Candidate: 1
+  Version table:
+     1 500
+        500 file:${APTARCHIVE} testing/main all Packages
+     1 500
+        500 file:${APTARCHIVE} unstable/main all Packages" apt policy diff-sha256