feature #378 Removing legacy tags significantly reduces memory and CPU usage (nicolas-grekas)

This PR was merged into the 1.0-dev branch.

Discussion
----------

Removing legacy tags significantly reduces memory and CPU usage

Before:
> Memory usage: 272.59MB (peak: 606.6MB), time: 5.79s

After:
> Memory usage: 145.56MB (peak: 158.56MB), time: 1.86s

Commits
-------

990b2d1 Removing legacy tags significantly reduces memory and CPU usage

Author: fabpot

src/Cache.php

@@ -0,0 +1,54 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\Flex;
+
+use Composer\Cache as BaseCache;
+
+/**
+ * @author Nicolas Grekas <[email protected]>
+ */
+class Cache extends BaseCache
+{
+    private static $lowestTags = [
+        'symfony/symfony' => 'v3.4.0',
+    ];
+
+    public function read($file)
+    {
+        $content = parent::read($file);
+
+        if (0 === strpos($file, 'provider-symfony$')) {
+            $content = json_encode($this->removeLegacyTags(json_decode($content, true)));
+        }
+
+        return $content;
+    }
+
+    public function removeLegacyTags(array $data): array
+    {
+        foreach (self::$lowestTags as $package => $lowestVersion) {
+            if (!isset($data['packages'][$package][$lowestVersion])) {
+                continue;
+            }
+            foreach ($data['packages'] as $package => $versions) {
+                foreach ($versions as $version => $composerJson) {
+                    if (version_compare($version, $lowestVersion, '<')) {
+                        unset($data['packages'][$package][$version]);
+                    }
+                }
+            }
+            break;
+        }
+
+        return $data;
+    }
+}

src/ComposerRepository.php

@@ -11,7 +11,11 @@
 
 namespace Symfony\Flex;
 
+use Composer\Config;
+use Composer\EventDispatcher\EventDispatcher;
+use Composer\IO\IOInterface;
 use Composer\Repository\ComposerRepository as BaseComposerRepository;
+use Composer\Util\RemoteFilesystem;
 
 /**
  * @author Nicolas Grekas <[email protected]>
@@ -20,6 +24,13 @@ class ComposerRepository extends BaseComposerRepository
 {
     private $providerFiles;
 
+    public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null)
+    {
+        parent::__construct($repoConfig, $io, $config, $eventDispatcher, $rfs);
+
+        $this->cache = new Cache($io, $config->get('cache-repo-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', $this->url), 'a-z0-9.$');
+    }
+
     protected function loadProviderListings($data)
     {
         if (null !== $this->providerFiles) {
@@ -53,6 +64,12 @@ protected function fetchFile($filename, $cacheKey = null, $sha256 = null, $store
             return [];
         }
 
-        return parent::fetchFile($filename, $cacheKey, $sha256, $storeLastModifiedTime);
+        $data = parent::fetchFile($filename, $cacheKey, $sha256, $storeLastModifiedTime);
+
+        if (0 === strpos($filename, 'http://packagist.org/p/symfony/') || 0 === strpos($filename, 'https://packagist.org/p/symfony/')) {
+            $data = $this->cache->removeLegacyTags($data);
+        }
+
+        return $data;
     }
 }

src/Flex.php

@@ -36,6 +36,8 @@
 use Composer\Plugin\PluginInterface;
 use Composer\Plugin\PreFileDownloadEvent;
 use Composer\Repository\ComposerRepository as BaseComposerRepository;
+use Composer\Repository\RepositoryFactory;
+use Composer\Repository\RepositoryManager;
 use Composer\Script\Event;
 use Composer\Script\ScriptEvents;
 use Symfony\Component\Console\Input\ArgvInput;
@@ -101,6 +103,13 @@ public function activate(Composer $composer, IOInterface $io)
 
         $rfs = Factory::createRemoteFilesystem($this->io, $this->config);
         $this->rfs = new ParallelDownloader($this->io, $this->config, $rfs->getOptions(), $rfs->isTlsDisabled());
+        $manager = new RepositoryManager($this->io, $this->config, $composer->getEventDispatcher(), $this->rfs);
+        $manager->setRepositoryClass('composer', ComposerRepository::class);
+        foreach (RepositoryFactory::defaultRepos(null, $this->config, $manager) as $repo) {
+            $manager->addRepository($repo);
+        }
+        $manager->setLocalRepository($composer->getRepositoryManager()->getLocalRepository());
+        $composer->setRepositoryManager($manager);
         $this->configurator = new Configurator($composer, $io, $this->options);
         $this->downloader = new Downloader($composer, $io, $this->rfs);
         $this->downloader->setFlexId($this->getFlexId());

tests/FlexTest.php

@@ -14,11 +14,14 @@
 use Composer\Composer;
 use Composer\Config;
 use Composer\DependencyResolver\Operation\InstallOperation;
+use Composer\Factory;
 use Composer\Installer\PackageEvent;
 use Composer\IO\BufferIO;
 use Composer\Package\Locker;
 use Composer\Package\Package;
 use Composer\Package\RootPackageInterface;
+use Composer\Repository\RepositoryManager;
+use Composer\Repository\WritableRepositoryInterface;
 use Composer\Script\Event;
 use PHPUnit\Framework\TestCase;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -160,12 +163,18 @@ public function testPostInstall()
 
     public function testActivateLoadsClasses()
     {
+        $io = new BufferIO('', OutputInterface::VERBOSITY_VERBOSE);
         $composer = new Composer();
-        $composer->setConfig($this->getMockBuilder(Config::class)->disableOriginalConstructor()->getMock());
+        $composer->setConfig(Factory::createConfig($io));
         $package = $this->getMockBuilder(RootPackageInterface::class)->disableOriginalConstructor()->getMock();
         $package->method('getExtra')->will($this->returnValue(['symfony' => ['allow-contrib' => true]]));
         $composer->setPackage($package);
-        $io = new BufferIO('', OutputInterface::VERBOSITY_VERBOSE);
+        $localRepo = $this->getMockBuilder(WritableRepositoryInterface::class)->disableOriginalConstructor()->getMock();
+        $manager = $this->getMockBuilder(RepositoryManager::class)->disableOriginalConstructor()->getMock();
+        $manager->expects($this->once())
+            ->method('getLocalRepository')
+            ->willReturn($localRepo);
+        $composer->setRepositoryManager($manager);
 
         $flex = new Flex();
         $flex->activate($composer, $io);