User with silo collaborator role is unable to start an instance

[askfongjojo]

Customer reported that only silo admin is able to start a stopped instance. I've confirmed that the collaborator user could stop or reboot the instance in question. Here is the replication on rack2:

$ OXIDE_TOKEN=oxide-token-5709b47d82caf64ea1a572c72ede041ca06a6465 OXIDE_HOST=https://silo11.sys.rack2.eng.oxide.computer oxide instance reboot --project bob --instance silo-admin
success
Instance {
    description: "",
    hostname: "silo-admin",
    id: 60648710-35c7-4073-8201-b95caa92a3d6,
    memory: ByteCount(
        8589934592,
    ),
    name: Name(
        "silo-admin",
    ),
    ncpus: InstanceCpuCount(
        2,
    ),
    project_id: be6cc6d9-8b58-437f-ad95-c27d7c38bbcf,
    run_state: Rebooting,
    time_created: 2023-10-12T19:24:15.749916Z,
    time_modified: 2023-10-12T19:24:15.749916Z,
    time_run_state_updated: 2023-10-12T20:32:05.290802Z,
}
$ OXIDE_TOKEN=oxide-token-5709b47d82caf64ea1a572c72ede041ca06a6465 OXIDE_HOST=https://silo11.sys.rack2.eng.oxide.computer oxide instance stop --project bob --instance silo-admin
success
Instance {
    description: "",
    hostname: "silo-admin",
    id: 60648710-35c7-4073-8201-b95caa92a3d6,
    memory: ByteCount(
        8589934592,
    ),
    name: Name(
        "silo-admin",
    ),
    ncpus: InstanceCpuCount(
        2,
    ),
    project_id: be6cc6d9-8b58-437f-ad95-c27d7c38bbcf,
    run_state: Stopping,
    time_created: 2023-10-12T19:24:15.749916Z,
    time_modified: 2023-10-12T19:24:15.749916Z,
    time_run_state_updated: 2023-10-12T20:32:17.386049Z,
}
$ OXIDE_TOKEN=oxide-token-5709b47d82caf64ea1a572c72ede041ca06a6465 OXIDE_HOST=https://silo11.sys.rack2.eng.oxide.computer oxide instance start --project bob --instance silo-admin
error
Error Response: status: 403 Forbidden; headers: {"content-type": "application/json", "x-request-id": "dea4334c-732c-4ad3-bbfd-2af613dbf763", "content-length": "113", "date": "Thu, 12 Oct 2023 20:32:28 GMT"}; value: Error { error_code: Some("Forbidden"), message: "Forbidden", request_id: "dea4334c-732c-4ad3-bbfd-2af613dbf763" }

The error came from the saga's authorization check which is involved only in the start instance action currently:

21:10:32.095Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "external-authenticator")} }
21:10:32.097Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Read
    actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000003, .. })
    resource = SiloUser { parent: Silo { parent: Fleet, key: c7dd16b8-255e-4300-8e52-7396a72caf3e, lookup_type: ById(c7dd16b8-255e-4300-8e52-7396a72caf3e) }, key: 648bcf81-e2ab-4017-af45-7bafb728bef3, lookup_type: ById(648bcf81-e2ab-4017-af45-7bafb728bef3) }
    result = Ok(())
21:10:32.097Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    roles = RoleSet { roles: {} }
21:10:32.097Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Query
    actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000003, .. })
    resource = Database
    result = Ok(())
21:10:32.099Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    roles = RoleSet { roles: {} }
21:10:32.099Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Query
    actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000003, .. })
    resource = Database
    result = Ok(())
21:10:32.101Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    roles = RoleSet { roles: {} }
21:10:32.101Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Query
    actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000003, .. })
    resource = Database
    result = Ok(())
21:10:32.103Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "external-authenticator")} }
21:10:32.105Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Read
    actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000003, .. })
    resource = Silo { parent: Fleet, key: c7dd16b8-255e-4300-8e52-7396a72caf3e, lookup_type: ById(c7dd16b8-255e-4300-8e52-7396a72caf3e) }
    result = Ok(())
21:10:32.105Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    roles = RoleSet { roles: {} }
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.105Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): authorize result
    action = Query
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    resource = Database
    result = Ok(())
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.107Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    roles = RoleSet { roles: {} }
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.107Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): authorize result
    action = Query
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    resource = Database
    result = Ok(())
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.110Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    roles = RoleSet { roles: {} }
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.111Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): authorize result
    action = Query
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    resource = Database
    result = Ok(())
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.114Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    roles = RoleSet { roles: {} }
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.114Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): authorize result
    action = Query
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    resource = Database
    result = Ok(())
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.118Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    roles = RoleSet { roles: {} }
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.119Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): authorize result
    action = Query
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    resource = Database
    result = Ok(())
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.123Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    roles = RoleSet { roles: {} }
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.124Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): authorize result
    action = Query
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    resource = Database
    result = Ok(())
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.128Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    roles = RoleSet { roles: {} }
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.128Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): authorize result
    action = Query
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    resource = Database
    result = Ok(())
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.132Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    roles = RoleSet { roles: {(Silo, c7dd16b8-255e-4300-8e52-7396a72caf3e, "collaborator")} }
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.135Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (dropshot_external): authorize result
    action = Modify
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    local_addr = 172.30.2.5:443
    method = POST
    remote_addr = 172.20.17.42:62112
    req_id = 5e7797b0-bd31-4b2e-83f0-910473657147
    resource = Instance { parent: Project { parent: Silo { parent: Fleet, key: c7dd16b8-255e-4300-8e52-7396a72caf3e, lookup_type: ById(c7dd16b8-255e-4300-8e52-7396a72caf3e) }, key: be6cc6d9-8b58-437f-ad95-c27d7c38bbcf, lookup_type: ByName("bob") }, key: 60648710-35c7-4073-8201-b95caa92a3d6, lookup_type: ByName("silo-admin") }
    result = Ok(())
    uri = /v1/instances/silo-admin/start?project=bob
21:10:32.135Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga create
    dag = {"end_node":6,"graph":{"edge_property":"directed","edges":[[0,1,null],[1,2,null],[2,3,null],[3,4,null],[5,0,null],[4,6,null]],"node_holes":[],"nodes":[{"Action":{"action_name":"instance_start.mark_as_starting","label":"MarkAsStarting","name":"starting_state"}},{"Action":{"action_name":"instance_start.dpd_ensure","label":"DpdEnsure","name":"dpd_ensure"}},{"Action":{"action_name":"instance_start.v2p_ensure","label":"V2PEnsure","name":"v2p_ensure"}},{"Action":{"action_name":"instance_start.ensure_registered","label":"EnsureRegistered","name":"ensure_registered"}},{"Action":{"action_name":"instance_start.ensure_running","label":"EnsureRunning","name":"ensure_running"}},{"Start":{"params":{"ensure_network":true,"instance":{"identity":{"description":"","id":"60648710-35c7-4073-8201-b95caa92a3d6","name":"silo-admin","time_created":"2023-10-12T19:24:15.749916Z","time_deleted":null,"time_modified":"2023-10-12T19:24:15.749916Z"},"project_id":"be6cc6d9-8b58-437f-ad95-c27d7c38bbcf","runtime_state":{"boot_on_fault":false,"dst_propolis_id":null,"gen":45,"hostname":"silo-admin","memory":8589934592,"migration_id":null,"ncpus":2,"propolis_gen":4,"propolis_id":"098b3a55-e1b1-410d-b8b8-e98082c3f9ac","propolis_ip":"fd00:1122:3344:102::8e/128","sled_id":"a2adea92-b56e-44fc-8a0d-7d63b5fd3b93","state":"stopped","time_updated":"2023-10-12T20:32:17.410492Z"},"user_data":[]},"serialized_authn":{"kind":{"Authenticated":[{"actor":{"SiloUser":{"silo_id":"c7dd16b8-255e-4300-8e52-7396a72caf3e","silo_user_id":"648bcf81-e2ab-4017-af45-7bafb728bef3"}}},{"mapped_fleet_roles":{"admin":["admin"]}}]}}}}},"End"]},"saga_name":"instance-start","start_node":5}
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.135Z INFO 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): creating saga
    file = nexus/db-queries/src/db/sec_store.rs:45
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
21:10:32.173Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N005 started
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.173Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = Started
    node_id = 5
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.175Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N005 succeeded
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.175Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = Succeeded(Null)
    node_id = 5
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.178Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N000 started
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.178Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = Started
    node_id = 0
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.180Z INFO 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): moving instance to Starting state via saga
    file = nexus/src/app/sagas/instance_start.rs:95
    instance_id = 60648710-35c7-4073-8201-b95caa92a3d6
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
21:10:32.219Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N000 succeeded
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.220Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = Succeeded(Object {"boot_on_fault": Bool(false), "dst_propolis_id": Null, "gen": Number(46), "hostname": String("silo-admin"), "memory": Number(8589934592), "migration_id": Null, "ncpus": Number(2), "propolis_gen": Number(4), "propolis_id": String("098b3a55-e1b1-410d-b8b8-e98082c3f9ac"), "propolis_ip": String("fd00:1122:3344:102::8e/128"), "sled_id": String("a2adea92-b56e-44fc-8a0d-7d63b5fd3b93"), "state": String("starting"), "time_updated": String("2023-10-12T20:32:17.410492Z")})
    node_id = 0
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.222Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N001 started
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.222Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = Started
    node_id = 1
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.224Z INFO 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): start saga: ensuring instance dpd configuration
    file = nexus/src/app/sagas/instance_start.rs:218
    instance_id = 60648710-35c7-4073-8201-b95caa92a3d6
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
21:10:32.224Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    roles = RoleSet { roles: {} }
21:10:32.225Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Query
    actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000004ead, .. })
    resource = Database
    result = Ok(())
21:10:32.227Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    roles = RoleSet { roles: {} }
21:10:32.227Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Query
    actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000004ead, .. })
    resource = Database
    result = Ok(())
21:10:32.229Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "viewer")} }
21:10:32.231Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Read
    actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000004ead, .. })
    resource = Sled { parent: Fleet, key: a2adea92-b56e-44fc-8a0d-7d63b5fd3b93, lookup_type: ById(a2adea92-b56e-44fc-8a0d-7d63b5fd3b93) }
    result = Ok(())
21:10:32.231Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    roles = RoleSet { roles: {} }
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
    saga_node = DpdEnsure
21:10:32.232Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Query
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    resource = Database
    result = Ok(())
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
    saga_node = DpdEnsure
21:10:32.235Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    roles = RoleSet { roles: {} }
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
    saga_node = DpdEnsure
21:10:32.236Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Query
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    resource = Database
    result = Ok(())
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
    saga_node = DpdEnsure
21:10:32.239Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    roles = RoleSet { roles: {(Silo, c7dd16b8-255e-4300-8e52-7396a72caf3e, "collaborator")} }
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
    saga_node = DpdEnsure
21:10:32.241Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Read
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    resource = Fleet
    result = Err(Forbidden)
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
    saga_node = DpdEnsure
21:10:32.241Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N001 failed
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.241Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = Failed(ActionFailed { source_error: String("Forbidden") })
    node_id = 1
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.244Z INFO 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): update for saga cached state
    file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/steno-0.4.0/src/sec.rs:1332
    new_state = Unwinding
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.244Z INFO 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): updating state
    file = nexus/db-queries/src/db/sec_store.rs:72
    new_state = unwinding
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.281Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N000 undo_started
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.281Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = UndoStarted
    node_id = 0
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.284Z INFO 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): start saga failed; returning instance to Stopped
    file = nexus/src/app/sagas/instance_start.rs:179
    instance_id = 60648710-35c7-4073-8201-b95caa92a3d6
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
21:10:32.321Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N000 undo_finished
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.321Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = UndoFinished
    node_id = 0
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.324Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N005 undo_started
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.324Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = UndoStarted
    node_id = 5
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.326Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N005 undo_finished
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
21:10:32.326Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = UndoFinished
    node_id = 5
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
21:10:32.328Z INFO 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): update for saga cached state
    file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/steno-0.4.0/src/sec.rs:1332
    new_state = Done
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7

[askfongjojo]

Updated the nexus log snippet above with the complete chain of saga output. This line looks a bit suspicious:

21:10:32.229Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): roles
    roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "viewer")} }

[gjcolombo]

I think I see the problem. The failure point is here:

21:10:32.241Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): authorize result
    action = Read
    actor = Some(Actor::SiloUser { silo_user_id: 648bcf81-e2ab-4017-af45-7bafb728bef3, silo_id: c7dd16b8-255e-4300-8e52-7396a72caf3e, .. })
    actor_id = 648bcf81-e2ab-4017-af45-7bafb728bef3
    authenticated = true
    resource = Fleet
    result = Err(Forbidden)
    saga_id = 722f512d-f40b-46b8-ae35-904c3e18ac32
    saga_name = instance-start
    saga_node = DpdEnsure

sis_dpd_ensure needs Fleet access to call Nexus::boundary_switches. Previously (before the start saga was implemented), the instance create saga populated NAT entries by obtaining a boundary switch list in project_create_instance using a special operation context that has fleet access:

omicron/nexus/src/app/instance.rs

Lines 204 to 211 in a903d61

	let saga_params = sagas::instance_create::Params {
	serialized_authn: authn::saga::Serialized::for_opctx(opctx),
	project_id: authz_project.id(),
	create_params: params.clone(),
	boundary_switches: self
	.boundary_switches(&self.opctx_alloc)
	.await?,
	};

The start saga, on the other hand, uses a context derived from the calling user's token. Note that the saga is perfectly happy to use opctx_alloc elsewhere in this step:

omicron/nexus/src/app/sagas/instance_start.rs

Lines 326 to 337 in 7e88bdf

	let sled_uuid = sagactx.lookup::<Uuid>("sled_id")?;
	let (.., sled) = LookupPath::new(&osagactx.nexus().opctx_alloc, &datastore)
	.sled_id(sled_uuid)
	.fetch()
	.await
	.map_err(ActionError::action_failed)?;
	let boundary_switches = osagactx
	.nexus()
	.boundary_switches(&opctx)
	.await
	.map_err(ActionError::action_failed)?;

If this theory is correct, the fix should be to switch to using opctx_alloc to do the boundary switch query in addition to the sled query.

[gjcolombo]

I can reproduce this issue in an integration test and believe I have a fix, but am wrangling a bit with the integration test environment itself (most of the helper functions don't seem to be designed to be used with collaborator users). Once I get that sorted out I'll put up a PR.