Add support for large NGINX config file sizes

internal/config/config.go

@@ -366,13 +366,25 @@ func registerClientFlags(fs *flag.FlagSet) {
 	fs.Int(
 		ClientGRPCMaxMessageReceiveSizeKey,
 		DefMaxMessageRecieveSize,
-		"Updates the client grpc setting MaxRecvMsgSize with the specific value in MB.",
+		"Updates the client grpc setting MaxRecvMsgSize with the specific value in bytes.",
 	)
 
 	fs.Int(
 		ClientGRPCMaxMessageSendSizeKey,
 		DefMaxMessageSendSize,
-		"Updates the client grpc setting MaxSendMsgSize with the specific value in MB.",
+		"Updates the client grpc setting MaxSendMsgSize with the specific value in bytes.",
+	)
+
+	fs.Uint32(
+		ClientGRPCFileChunkSizeKey,
+		DefFileChunkSize,
+		"File chunk size in bytes.",
+	)
+
+	fs.Uint32(
+		ClientGRPCMaxFileSizeKey,
+		DefMaxFileSize,
+		"Max file size in bytes.",
 	)
 }
 
@@ -709,6 +721,8 @@ func resolveClient() *Client {
 			MaxMessageSize:        viperInstance.GetInt(ClientGRPCMaxMessageSizeKey),
 			MaxMessageReceiveSize: viperInstance.GetInt(ClientGRPCMaxMessageReceiveSizeKey),
 			MaxMessageSendSize:    viperInstance.GetInt(ClientGRPCMaxMessageSendSizeKey),
+			MaxFileSize:           viperInstance.GetUint32(ClientGRPCMaxFileSizeKey),
+			FileChunkSize:         viperInstance.GetUint32(ClientGRPCFileChunkSizeKey),
 		},
 		Backoff: &BackOff{
 			InitialInterval:     viperInstance.GetDuration(ClientBackoffInitialIntervalKey),

internal/config/config_test.go

@@ -104,6 +104,8 @@ func checkDefaultsClientValues(t *testing.T, viperInstance *viper.Viper) {
 	assert.Equal(t, DefMaxMessageSize, viperInstance.GetInt(ClientGRPCMaxMessageSizeKey))
 	assert.Equal(t, DefMaxMessageRecieveSize, viperInstance.GetInt(ClientGRPCMaxMessageReceiveSizeKey))
 	assert.Equal(t, DefMaxMessageSendSize, viperInstance.GetInt(ClientGRPCMaxMessageSendSizeKey))
+	assert.Equal(t, DefFileChunkSize, viperInstance.GetUint32(ClientGRPCFileChunkSizeKey))
+	assert.Equal(t, DefMaxFileSize, viperInstance.GetUint32(ClientGRPCMaxFileSizeKey))
 	assert.Equal(t, make(map[string]string), viperInstance.GetStringMapString(LabelsRootKey))
 }
 
@@ -781,6 +783,8 @@ func createConfig() *Config {
 				MaxMessageSize:        1048575,
 				MaxMessageReceiveSize: 1048575,
 				MaxMessageSendSize:    1048575,
+				MaxFileSize:           485753,
+				FileChunkSize:         48575,
 			},
 			Backoff: &BackOff{
 				InitialInterval:     200 * time.Millisecond,

internal/config/defaults.go

@@ -5,7 +5,6 @@
 package config
 
 import (
-	"math"
 	"time"
 
 	pkg "github.com/nginx/agent/v3/pkg/config"
@@ -28,9 +27,11 @@ const (
 	DefCommandTLServerNameKey  = ""
 
 	// Client GRPC Settings
-	DefMaxMessageSize        = 0       // 0 = unset
-	DefMaxMessageRecieveSize = 4194304 // default 4 MB
-	DefMaxMessageSendSize    = math.MaxInt32
+	DefMaxMessageSize               = 0       // 0 = unset
+	DefMaxMessageRecieveSize        = 4194304 // default 4 MB
+	DefMaxMessageSendSize           = 4194304 // default 4 MB
+	DefMaxFileSize           uint32 = 3145728 // 3MB
+	DefFileChunkSize         uint32 = 2097152 // 2MB
 
 	// Client HTTP Settings
 	DefHTTPTimeout = 10 * time.Second

internal/config/flags.go

@@ -36,6 +36,8 @@ var (
 	ClientGRPCMaxMessageSendSizeKey    = pre(ClientRootKey) + "grpc_max_message_send_size"
 	ClientGRPCMaxMessageReceiveSizeKey = pre(ClientRootKey) + "grpc_max_message_receive_size"
 	ClientGRPCMaxMessageSizeKey        = pre(ClientRootKey) + "grpc_max_message_size"
+	ClientGRPCMaxFileSizeKey           = pre(ClientRootKey) + "grpc_max_file_size"
+	ClientGRPCFileChunkSizeKey         = pre(ClientRootKey) + "grpc_file_chunk_size"
 
 	ClientBackoffInitialIntervalKey     = pre(ClientRootKey) + "backoff_initial_interval"
 	ClientBackoffMaxIntervalKey         = pre(ClientRootKey) + "backoff_max_interval"

internal/config/testdata/nginx-agent.conf

@@ -35,6 +35,8 @@ client:
        max_message_size: 1048575
        max_message_receive_size: 1048575
        max_message_send_size: 1048575
+       max_file_size: 485753
+       file_chunk_size: 48575
     backoff:
         initial_interval: 200ms
         max_interval: 10s

internal/config/types.go

@@ -83,9 +83,11 @@ type (
 		KeepAlive *KeepAlive `yaml:"keepalive" mapstructure:"keepalive"`
 		// if MaxMessageSize is size set then we use that value,
 		// otherwise MaxMessageRecieveSize and MaxMessageSendSize for individual settings
-		MaxMessageSize        int `yaml:"max_message_size"         mapstructure:"max_message_size"`
-		MaxMessageReceiveSize int `yaml:"max_message_receive_size" mapstructure:"max_message_receive_size"`
-		MaxMessageSendSize    int `yaml:"max_message_send_size"    mapstructure:"max_message_send_size"`
+		MaxMessageSize        int    `yaml:"max_message_size"         mapstructure:"max_message_size"`
+		MaxMessageReceiveSize int    `yaml:"max_message_receive_size" mapstructure:"max_message_receive_size"`
+		MaxMessageSendSize    int    `yaml:"max_message_send_size"    mapstructure:"max_message_send_size"`
+		MaxFileSize           uint32 `yaml:"max_file_size"            mapstructure:"max_file_size"`
+		FileChunkSize         uint32 `yaml:"file_chunk_size"          mapstructure:"file_chunk_size"`
 	}
 
 	KeepAlive struct {

internal/file/fake_file_stream_test.go

@@ -0,0 +1,116 @@
+// Copyright (c) F5, Inc.
+//
+// This source code is licensed under the Apache License, Version 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+
+package file
+
+import (
+	"context"
+	"sync/atomic"
+
+	mpi "github.com/nginx/agent/v3/api/grpc/mpi/v1"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+type FakeClientStreamingClient struct {
+	sendCount atomic.Int32
+}
+
+func (f *FakeClientStreamingClient) Send(req *mpi.FileDataChunk) error {
+	f.sendCount.Add(1)
+	return nil
+}
+
+func (f *FakeClientStreamingClient) CloseAndRecv() (*mpi.UpdateFileResponse, error) {
+	return &mpi.UpdateFileResponse{}, nil
+}
+
+func (f *FakeClientStreamingClient) Header() (metadata.MD, error) {
+	return metadata.MD{}, nil
+}
+
+func (f *FakeClientStreamingClient) Trailer() metadata.MD {
+	return nil
+}
+
+func (f *FakeClientStreamingClient) CloseSend() error {
+	return nil
+}
+
+func (f *FakeClientStreamingClient) Context() context.Context {
+	return context.Background()
+}
+
+func (f *FakeClientStreamingClient) SendMsg(m any) error {
+	return nil
+}
+
+func (f *FakeClientStreamingClient) RecvMsg(m any) error {
+	return nil
+}
+
+type FakeServerStreamingClient struct {
+	chunks         map[uint32][]byte
+	fileName       string
+	currentChunkID uint32
+}
+
+func (f *FakeServerStreamingClient) Recv() (*mpi.FileDataChunk, error) {
+	fileDataChunk := &mpi.FileDataChunk{
+		Meta: &mpi.MessageMeta{
+			MessageId:     "123",
+			CorrelationId: "1234",
+			Timestamp:     timestamppb.Now(),
+		},
+	}
+
+	if f.currentChunkID == 0 {
+		fileDataChunk.Chunk = &mpi.FileDataChunk_Header{
+			Header: &mpi.FileDataChunkHeader{
+				FileMeta: &mpi.FileMeta{
+					Name:        f.fileName,
+					Permissions: "666",
+				},
+				Chunks:    52,
+				ChunkSize: 1,
+			},
+		}
+	} else {
+		fileDataChunk.Chunk = &mpi.FileDataChunk_Content{
+			Content: &mpi.FileDataChunkContent{
+				ChunkId: f.currentChunkID,
+				Data:    f.chunks[f.currentChunkID-1],
+			},
+		}
+	}
+
+	f.currentChunkID++
+
+	return fileDataChunk, nil
+}
+
+func (f *FakeServerStreamingClient) Header() (metadata.MD, error) {
+	return metadata.MD{}, nil
+}
+
+func (f *FakeServerStreamingClient) Trailer() metadata.MD {
+	return metadata.MD{}
+}
+
+func (f *FakeServerStreamingClient) CloseSend() error {
+	return nil
+}
+
+func (f *FakeServerStreamingClient) Context() context.Context {
+	return context.Background()
+}
+
+func (f *FakeServerStreamingClient) SendMsg(m any) error {
+	return nil
+}
+
+func (f *FakeServerStreamingClient) RecvMsg(m any) error {
+	return nil
+}