Skip to content

[perf] Cache nixpkgs resolution #2576

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 9, 2025
Merged

[perf] Cache nixpkgs resolution #2576

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aba4ebe
[perf] Cache nixpkgs resolution
mikeland73 Apr 8, 2025
8e1c2a1
Change ttl to 30 days
mikeland73 Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[perf] Cache nixpkgs resolution
  • Loading branch information
@mikeland73
mikeland73 committed Apr 8, 2025
commit aba4ebeb47250383bfd283fdb6594f21b6608e04
83 changes: 42 additions & 41 deletions devbox.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,173 +2,174 @@
"lockfile_version": "1",
"packages": {
"fd@latest": {
"last_modified": "2025-02-07T11:26:36Z",
"resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#fd",
"last_modified": "2025-03-11T17:52:14Z",
"resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#fd",
"source": "devbox-search",
"version": "10.2.0",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/ad5m54pfn9k39v80lpyhrnsh336nqrp5-fd-10.2.0",
"path": "/nix/store/40pdazk980kp3h26py4hjyx9rys1g14n-fd-10.2.0",
"default": true
}
],
"store_path": "/nix/store/ad5m54pfn9k39v80lpyhrnsh336nqrp5-fd-10.2.0"
"store_path": "/nix/store/40pdazk980kp3h26py4hjyx9rys1g14n-fd-10.2.0"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/vn3mny38qmf3lm809rpcvahxqwhkqb7m-fd-10.2.0",
"path": "/nix/store/76zcwa1d33vciy4gyqvk6jl2n3g1542q-fd-10.2.0",
"default": true
}
],
"store_path": "/nix/store/vn3mny38qmf3lm809rpcvahxqwhkqb7m-fd-10.2.0"
"store_path": "/nix/store/76zcwa1d33vciy4gyqvk6jl2n3g1542q-fd-10.2.0"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/6kdv4iy9j3svncq0vs47wxfvvn7flcr5-fd-10.2.0",
"path": "/nix/store/ys9qmljs0ag7j040radgg48l6pvjmv9l-fd-10.2.0",
"default": true
}
],
"store_path": "/nix/store/6kdv4iy9j3svncq0vs47wxfvvn7flcr5-fd-10.2.0"
"store_path": "/nix/store/ys9qmljs0ag7j040radgg48l6pvjmv9l-fd-10.2.0"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/x58pg72qw2xv1vvs4pbqw63zhkdkp331-fd-10.2.0",
"path": "/nix/store/rrdvpl7rym4ia0h7rfz1vmlcvvivj30j-fd-10.2.0",
"default": true
}
],
"store_path": "/nix/store/x58pg72qw2xv1vvs4pbqw63zhkdkp331-fd-10.2.0"
"store_path": "/nix/store/rrdvpl7rym4ia0h7rfz1vmlcvvivj30j-fd-10.2.0"
}
}
},
"git@latest": {
"last_modified": "2025-02-07T11:26:36Z",
"resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#git",
"last_modified": "2025-03-11T17:52:14Z",
"resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#git",
"source": "devbox-search",
"version": "2.47.2",
"version": "2.48.1",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/9z3jhc0rlj3zaw8nd1zka9vli6w0q11g-git-2.47.2",
"path": "/nix/store/b3sci30zzzlj3rzj1y89cijnd6zcwapk-git-2.48.1",
"default": true
},
{
"name": "doc",
"path": "/nix/store/rh151iwgy4h8yv8kxd5facw57cyj0bav-git-2.47.2-doc"
"path": "/nix/store/086knqdw7fjgzczp0i6nad95s2v6jbya-git-2.48.1-doc"
}
],
"store_path": "/nix/store/9z3jhc0rlj3zaw8nd1zka9vli6w0q11g-git-2.47.2"
"store_path": "/nix/store/b3sci30zzzlj3rzj1y89cijnd6zcwapk-git-2.48.1"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/gx5y37qcfqdvn0h6swjd04dmqjjh3nk7-git-2.47.2",
"path": "/nix/store/pck1dr5jxrd5b8nmfasbn13z422jhcfm-git-2.48.1",
"default": true
},
{
"name": "debug",
"path": "/nix/store/8vfpmf3vjgzl2psip76p0f9h11sb6y3p-git-2.47.2-debug"
"path": "/nix/store/xqqsvzlilh843rm6knykyng81apapr33-git-2.48.1-debug"
},
{
"name": "doc",
"path": "/nix/store/c25mq3q83dvw3k5pb0qr5333g3cycylq-git-2.47.2-doc"
"path": "/nix/store/485b32ys0s2dvjfisn7405ildmpqvfzk-git-2.48.1-doc"
}
],
"store_path": "/nix/store/gx5y37qcfqdvn0h6swjd04dmqjjh3nk7-git-2.47.2"
"store_path": "/nix/store/pck1dr5jxrd5b8nmfasbn13z422jhcfm-git-2.48.1"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/39xx5gx3hxigs1b5ldw5i2jr84vsn3rf-git-2.47.2",
"path": "/nix/store/9qjzgsf9mvdp6sfd7xyzhgrahl2qhhp6-git-2.48.1",
"default": true
},
{
"name": "doc",
"path": "/nix/store/xmh2djjrnbpiqqgpblrcbavnqh0nv4km-git-2.47.2-doc"
"path": "/nix/store/cgv7qa0ix059ma9a0qac0bywfvl3k7k2-git-2.48.1-doc"
}
],
"store_path": "/nix/store/39xx5gx3hxigs1b5ldw5i2jr84vsn3rf-git-2.47.2"
"store_path": "/nix/store/9qjzgsf9mvdp6sfd7xyzhgrahl2qhhp6-git-2.48.1"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/33g65w5cc9n8fr0hxj84282xmv4l7hyl-git-2.47.2",
"path": "/nix/store/lqx2rv26sdndpa2vyy2vxsahj03km69z-git-2.48.1",
"default": true
},
{
"name": "debug",
"path": "/nix/store/jyz4nvcd3bci4vg2sfsmvrq0fp9mzr5a-git-2.47.2-debug"
"name": "doc",
"path": "/nix/store/hjczhs1dm3hzij7mx5c91rkzqvkb89av-git-2.48.1-doc"
},
{
"name": "doc",
"path": "/nix/store/lb4nipdhlwrxdavz7gdkcik6lkz3cbdm-git-2.47.2-doc"
"name": "debug",
"path": "/nix/store/bk8xndavdnc2qgyvc6hcc8h29lk9jzqb-git-2.48.1-debug"
}
],
"store_path": "/nix/store/33g65w5cc9n8fr0hxj84282xmv4l7hyl-git-2.47.2"
"store_path": "/nix/store/lqx2rv26sdndpa2vyy2vxsahj03km69z-git-2.48.1"
}
}
},
"github:NixOS/nixpkgs/nixpkgs-unstable": {
"resolved": "github:NixOS/nixpkgs/73cf49b8ad837ade2de76f87eb53fc85ed5d4680?lastModified=1739866667&narHash=sha256-EO1ygNKZlsAC9avfcwHkKGMsmipUk1Uc0TbrEZpkn64%3D"
"last_modified": "2025-04-07T13:23:10Z",
"resolved": "github:NixOS/nixpkgs/b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b?lastModified=1744032190&narHash=sha256-KSlfrncSkcu1YE%2BuuJ%2FPTURsSlThoGkRqiGDVdbiE%2Fk%3D"
},
"go@latest": {
"last_modified": "2025-02-12T00:10:52Z",
"resolved": "github:NixOS/nixpkgs/83a2581c81ff5b06f7c1a4e7cc736a455dfcf7b4#go_1_24",
"last_modified": "2025-03-11T17:52:14Z",
"resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#go",
"source": "devbox-search",
"version": "1.24.0",
"version": "1.24.1",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/qldcnifalkvyah0wnv7m4zb854yd9l88-go-1.24.0",
"path": "/nix/store/ja4jxx60lh1qfqfl4z4p2rff56ia1c3c-go-1.24.1",
"default": true
}
],
"store_path": "/nix/store/qldcnifalkvyah0wnv7m4zb854yd9l88-go-1.24.0"
"store_path": "/nix/store/ja4jxx60lh1qfqfl4z4p2rff56ia1c3c-go-1.24.1"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/rrxgml7w4pfmibjbspkdvrw8vd2vnarb-go-1.24.0",
"path": "/nix/store/8ply43gnxk1xwichr81mpgbjcd9a1y5w-go-1.24.1",
"default": true
}
],
"store_path": "/nix/store/rrxgml7w4pfmibjbspkdvrw8vd2vnarb-go-1.24.0"
"store_path": "/nix/store/8ply43gnxk1xwichr81mpgbjcd9a1y5w-go-1.24.1"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/7imv22pl4qrjwvi6jzlfb305rc2min45-go-1.24.0",
"path": "/nix/store/87yxrfx5lh78bdz393i33cr5z23x06q4-go-1.24.1",
"default": true
}
],
"store_path": "/nix/store/7imv22pl4qrjwvi6jzlfb305rc2min45-go-1.24.0"
"store_path": "/nix/store/87yxrfx5lh78bdz393i33cr5z23x06q4-go-1.24.1"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/vh5d5bj1sljdhdypy80x1ydx2jx6rv2q-go-1.24.0",
"path": "/nix/store/cfjhl0kn7xc65466pha9fkrvigw3g72n-go-1.24.1",
"default": true
}
],
"store_path": "/nix/store/vh5d5bj1sljdhdypy80x1ydx2jx6rv2q-go-1.24.0"
"store_path": "/nix/store/cfjhl0kn7xc65466pha9fkrvigw3g72n-go-1.24.1"
}
}
}
Expand Down
27 changes: 15 additions & 12 deletions internal/devbox/update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package devbox
import (
"context"
"fmt"
"slices"

"github.com/pkg/errors"
"go.jetify.com/devbox/internal/devbox/devopt"
Expand All @@ -20,6 +21,20 @@ import (
)

func (d *Devbox) Update(ctx context.Context, opts devopt.UpdateOpts) error {
if len(opts.Pkgs) == 0 || slices.Contains(opts.Pkgs, "nixpkgs") {
if err := d.lockfile.UpdateStdenv(); err != nil {
return err
}
// if nixpkgs is the only package to update, just return here.
if len(opts.Pkgs) == 1 {
return nil
}
// Otherwise, remove nixpkgs and continue
opts.Pkgs = slices.DeleteFunc(opts.Pkgs, func(pkg string) bool {
return pkg == "nixpkgs"
})
}

inputs, err := d.inputsToUpdate(opts)
if err != nil {
return err
Expand Down Expand Up @@ -65,9 +80,6 @@ func (d *Devbox) Update(ctx context.Context, opts devopt.UpdateOpts) error {
}
}

if err := d.updateStdenv(); err != nil {
return err
}
mode := update
if opts.NoInstall {
mode = noInstall
Expand Down Expand Up @@ -110,15 +122,6 @@ func (d *Devbox) inputsToUpdate(
return pkgsToUpdate, nil
}

func (d *Devbox) updateStdenv() error {
err := d.lockfile.Remove(d.Stdenv().String())
if err != nil {
return err
}
d.lockfile.Stdenv() // will re-resolve the stdenv flake
return nil
}

func (d *Devbox) updateDevboxPackage(pkg *devpkg.Package) error {
resolved, err := d.lockfile.FetchResolvedPackage(pkg.Raw)
if err != nil {
Expand Down
17 changes: 17 additions & 0 deletions internal/lock/lockfile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,23 @@ func (f *File) Save() error {
return cuecfg.WriteFile(lockFilePath(f.devboxProject.ProjectDir()), f)
}

func (f *File) UpdateStdenv() error {
if err := nix.ClearFlakeCache(f.devboxProject.Stdenv()); err != nil {
return err
}
if err := f.Remove(f.devboxProject.Stdenv().String()); err != nil {
return err
}
return f.Add(f.devboxProject.Stdenv().String())
}

// TODO: We should improve a few issues with this function:
// * It shared the same name as Devbox.Stdenv() which is confusing.
// * Since File implements DevboxProject, IDEs really struggle to accurately find call sites.
// (side note, we should remove DevboxProject interface)
// * This function forces a resolution of the stdenv flake which is slow and doesn't give us a
// chance to "prep" the user for some waiting.
// * Should we rename to Nixpkgs() ? Stdenv feels a bit ambiguous.
func (f *File) Stdenv() flake.Ref {
unlocked := f.devboxProject.Stdenv()
pkg, err := f.Resolve(unlocked.String())
Expand Down
23 changes: 21 additions & 2 deletions internal/lock/resolve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ func (f *File) FetchResolvedPackage(pkg string) (*Package, error) {
return nil, err
}
return &Package{
Resolved: installable.String(),
Resolved: installable.String(),
LastModified: time.Unix(installable.Ref.LastModified, 0).UTC().Format(time.RFC3339),
}, nil
}

Expand Down Expand Up @@ -220,7 +221,25 @@ func lockFlake(ctx context.Context, ref flake.Ref) (flake.Ref, error) {
return ref, nil
}

meta, err := nix.ResolveFlake(ctx, ref)
var meta nix.FlakeMetadata
var err error
// For nixpkgs, we cache resolutions (currently flakeCacheTTL=90 days) to avoid downloading
// new nixpkgs too often which is really slow and rarely changes anything.
//
// Ideally we can do something similar for all packages (flake and otherwise)
// Specifically, if user adds [email protected] (or python@latest that resolves to 3.12) and that
// package is already installed, we should use it instead of using 3.12 from search service
// (which may have different store path). This would allow all devbox projects to share packages
// if the version resolution is the same.
//
// That said, the logic for caching resolved versions and non-locked flake references would not
// be the same.
if ref.IsNixpkgs() {
meta, err = nix.ResolveCachedFlake(ctx, ref)
} else {
meta, err = nix.ResolveFlake(ctx, ref)
}

if err != nil {
return ref, err
}
Expand Down
32 changes: 27 additions & 5 deletions internal/nix/flake.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,23 @@ package nix
import (
"context"
"encoding/json"
"time"

"go.jetify.com/devbox/nix/flake"
"go.jetify.com/pkg/filecache"
)

const flakeCacheTTL = time.Hour * 24 * 90

var flakeFileCache = filecache.New[FlakeMetadata]("devbox/flakes")

type FlakeMetadata struct {
Description string `json:"description"`
Original flake.Ref `json:"original"`
Resolved flake.Ref `json:"resolved"`
Locked flake.Ref `json:"locked"`
Path string `json:"path"`
Description string `json:"description"`
LastModified int64 `json:"lastModified"`
Locked flake.Ref `json:"locked"`
Original flake.Ref `json:"original"`
Path string `json:"path"`
Resolved flake.Ref `json:"resolved"`
}

func ResolveFlake(ctx context.Context, ref flake.Ref) (FlakeMetadata, error) {
Expand All @@ -28,3 +35,18 @@ func ResolveFlake(ctx context.Context, ref flake.Ref) (FlakeMetadata, error) {
}
return meta, nil
}

func ResolveCachedFlake(ctx context.Context, ref flake.Ref) (FlakeMetadata, error) {
return flakeFileCache.GetOrSet(ref.String(), func() (FlakeMetadata, time.Duration, error) {
meta, err := ResolveFlake(ctx, ref)
if err != nil {
return FlakeMetadata{}, 0, err
}
return meta, flakeCacheTTL, nil
})
}

func ClearFlakeCache(ref flake.Ref) error {
// TODO: Add unset to filecache
return flakeFileCache.Set(ref.String(), FlakeMetadata{}, -1)
}
Loading
Loading