utils/init_provider.go stopped working

[depohmel]

Since yesterday utils/init_provider.go OpenProviderSession fails with following error:

{"level":"info","ts":1560884939.663679,"caller":"utils/init_provider.go:38","msg":"Block softlayer provider volume registry done!"}
{"level":"info","ts":1560884939.6637144,"caller":"utils/init_provider.go:63","msg":"Provider registration done!!!"}
{"level":"info","ts":1560884939.6637185,"caller":"utils/init_provider.go:75","msg":"In OpenProviderSession methods"}
{"level":"info","ts":1560884939.663722,"caller":"utils/init_provider.go:102","msg":"Generating generateContextCredentials for ","Provider ID":"SOFTLAYER-BLOCK"}
{"level":"error","ts":1560884940.10645,"caller":"iam/token_exchange.go:173","msg":"IAM token exchange request failed with message","StatusCode":400,"ErrorMessage:":"Validation failed ","ErrorType:":"BXNIM0106E","Error":{"errorMessage":"Validation failed ","errorCode":"BXNIM0106E","errorDetails":"Validation of property 'response_type' with value '[ims_portal]' failed. Valid values: 'cloud_iam'","requirements":{"error":"","code":""}},"stacktrace":"vendor/go.uber.org/zap.Stack\n\t/repo/go/src/vendor/go.uber.org/zap/field.go:195\nvendor/go.uber.org/zap.(*Logger).check\n\t/repo/go/src/vendor/go.uber.org/zap/logger.go:301\nvendor/go.uber.org/zap.(*Logger).Error\n\t/repo/go/src/vendor/go.uber.org/zap/logger.go:202\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/iam.(*tokenExchangeRequest).sendTokenExchangeRequest\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/iam/token_exchange.go:173\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/iam.(*tokenExchangeRequest).exchangeForIMSToken\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/iam/token_exchange.go:112\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/iam.(*tokenExchangeService).ExchangeIAMAPIKeyForIMSToken\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/iam/token_exchange.go:90\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/softlayer/auth.(*contextCredentialsFactory).ForIAMAPIKey\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/softlayer/auth/ims_token.go:52\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils.GenerateContextCredentials\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils/init_provider.go:115\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils.OpenProviderSession\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils/init_provider.go:89\ngithub.com/kanisterio/kanister/pkg/blockstorage/ibm.newClient\n\t/repo/go/src/github.com/kanisterio/kanister/pkg/blockstorage/ibm/client.go:74\ngithub.com/kanisterio/kanister/pkg/blockstorage/ibm.(*ClientSuite).TestClient\n\t/repo/go/src/github.com/kanisterio/kanister/pkg/blockstorage/ibm/client_test.go:43\nreflect.Value.call\n\t/usr/local/go/src/reflect/value.go:447\nreflect.Value.Call\n\t/usr/local/go/src/reflect/value.go:308\nvendor/gopkg.in/check%2ev1.(*suiteRunner).forkTest.func1\n\t/repo/go/src/vendor/gopkg.in/check.v1/check.go:772\nvendor/gopkg.in/check%2ev1.(*suiteRunner).forkCall.func1\n\t/repo/go/src/vendor/gopkg.in/check.v1/check.go:666"}
{"level":"error","ts":1560884940.1073356,"caller":"auth/ims_token.go:55","msg":"Unable to retrieve IMS credentials from IAM API key","error":"IAM token exchange request failed: Validation failed ","stacktrace":"vendor/go.uber.org/zap.Stack\n\t/repo/go/src/vendor/go.uber.org/zap/field.go:195\nvendor/go.uber.org/zap.(*Logger).check\n\t/repo/go/src/vendor/go.uber.org/zap/logger.go:301\nvendor/go.uber.org/zap.(*Logger).Error\n\t/repo/go/src/vendor/go.uber.org/zap/logger.go:202\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/softlayer/auth.(*contextCredentialsFactory).ForIAMAPIKey\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/softlayer/auth/ims_token.go:55\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils.GenerateContextCredentials\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils/init_provider.go:115\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils.OpenProviderSession\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils/init_provider.go:89\ngithub.com/kanisterio/kanister/pkg/blockstorage/ibm.newClient\n\t/repo/go/src/github.com/kanisterio/kanister/pkg/blockstorage/ibm/client.go:74\ngithub.com/kanisterio/kanister/pkg/blockstorage/ibm.(*ClientSuite).TestClient\n\t/repo/go/src/github.com/kanisterio/kanister/pkg/blockstorage/ibm/client_test.go:43\nreflect.Value.call\n\t/usr/local/go/src/reflect/value.go:447\nreflect.Value.Call\n\t/usr/local/go/src/reflect/value.go:308\nvendor/gopkg.in/check%2ev1.(*suiteRunner).forkTest.func1\n\t/repo/go/src/vendor/gopkg.in/check.v1/check.go:772\nvendor/gopkg.in/check%2ev1.(*suiteRunner).forkCall.func1\n\t/repo/go/src/vendor/gopkg.in/check.v1/check.go:666"}
{"level":"error","ts":1560884940.1078017,"caller":"utils/init_provider.go:96","msg":"Failed to open provider session","error":"IAM token exchange request failed: Validation failed ","Fatal":false,"stacktrace":"vendor/go.uber.org/zap.Stack\n\t/repo/go/src/vendor/go.uber.org/zap/field.go:195\nvendor/go.uber.org/zap.(*Logger).check\n\t/repo/go/src/vendor/go.uber.org/zap/logger.go:301\nvendor/go.uber.org/zap.(*Logger).Error\n\t/repo/go/src/vendor/go.uber.org/zap/logger.go:202\nvendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils.OpenProviderSession\n\t/repo/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils/init_provider.go:96\ngithub.com/kanisterio/kanister/pkg/blockstorage/ibm.newClient\n\t/repo/go/src/github.com/kanisterio/kanister/pkg/blockstorage/ibm/client.go:74\ngithub.com/kanisterio/kanister/pkg/blockstorage/ibm.(*ClientSuite).TestClient\n\t/repo/go/src/github.com/kanisterio/kanister/pkg/blockstorage/ibm/client_test.go:43\nreflect.Value.call\n\t/usr/local/go/src/reflect/value.go:447\nreflect.Value.Call\n\t/usr/local/go/src/reflect/value.go:308\nvendor/gopkg.in/check%2ev1.(*suiteRunner).forkTest.func1\n\t/repo/go/src/vendor/gopkg.in/check.v1/check.go:772\nvendor/gopkg.in/check%2ev1.(*suiteRunner).forkCall.func1\n\t/repo/go/src/vendor/gopkg.in/check.v1/check.go:666"}

example config:

Bluemix:(*config.BluemixConfig){IamURL:\"https://iam.bluemix.net\", IamClientID:\"bx\", IamClientSecret:\"bx\", IamAPIKey:\"IBMCLOUD_ServiceAPIKey\", RefreshToken:\"\"}, Softlayer:(*config.SoftlayerConfig){SoftlayerBlockEnabled:true, SoftlayerBlockProviderName:\"SOFTLAYER-BLOCK\", SoftlayerFileEnabled:false, SoftlayerFileProviderName:\"SOFTLAYER-FILE\", SoftlayerUsername:\"\", SoftlayerAPIKey:\"\", SoftlayerEndpointURL:\"https://api.softlayer.com/rest/v3\", SoftlayerDataCenter:\"sjc03\", SoftlayerTimeout:\"20s\", SoftlayerVolProvisionTimeout:\"10m\", SoftlayerRetryInterval:\"5s\", SoftlayerJWTKID:\"\", SoftlayerJWTTTL:0, SoftlayerJWTValidFrom:0, SoftlayerIMSEndpointURL:\"https://api.softlayer.com/mobile/v3\", SoftlayerAPIDebug:false}, Gen2:(*config.Gen2Config){Gen2ProviderEnabled:false, Gen2Username:\"\", Gen2APIKey:\"\", Gen2URL:\"\"}

[narkarum]

Can you see whether the IAM token given is correct?

[depohmel]

Do you mean IamAPIKey? Or a return token from https://iam.bluemix.net ?

[narkarum]

IAM API KEY

[narkarum]

IamAPIKey:"IBMCLOUD_ServiceAPIKey"

Seems this is wrong.

Is it a correct API KEY?

[depohmel]

@narkarum, for security reason i'm not sharing my actual key. this is IamAPIKey:"IBMCLOUD_ServiceAPIKey" is not correct, but when i was testing i was using a correct key.

[narkarum]

Recently there was a change done to the config. I would check locally and revert back.

[narkarum]

There is a change in the config. Please set the API config to: gc_api_key

[VPC]
  vpc_enabled = true
  vpc_block_provider_name = "vpc-classic"
  gc_token_exchange_endpoint_url = "https://iam.stage1.bluemix.net"
  gc_riaas_endpoint_url = "https://us-east-stage02.iaasdev.cloud.ibm.com"
  gc_resource_group_id = "XXXX"
  gc_api_key = "XXXX"
  encryption = true
  vpc_timeout = "120s"
  max_retry_attempt  = 5
  max_retry_gap =  30
  api_version = "2019-01-01"

[IKS]
  iks_enabled = true
  iks_block_provider_name = "iks-vpc-classic"

[narkarum]

gc_api_key is the encrypted value with the cluster ID

[depohmel]

what do you mean encrypted value with cluster id? is there a way of using libs without cluster?
any code example on how this thing should look like?

[depohmel]

any plans to add backward compatibility?

[depohmel]

also is there any changes inside softlayer config part?
is this still relevant?

[softlayer]
softlayer_block_enabled = true
softlayer_block_provider_name = "SOFTLAYER-BLOCK"
softlayer_file_enabled = false
softlayer_file_provider_name = "SOFTLAYER-FILE"
softlayer_username = ""
softlayer_api_key = ""
softlayer_endpoint_url = "https://api.softlayer.com/rest/v3"
softlayer_iam_endpoint_url = "https://api.softlayer.com/mobile/v3"
softlayer_datacenter = "mex01"
softlayer_api_timeout = "20s"

[depohmel]

i've tried to make it work by doing following:

• updated github.com/IBM/ibmcloud-storage-volume-lib to 0e08a46
• updated github.com/softlayer/softlayer-go to ba6e7f29521771a05a8faa6c1782735b4b74dc36
• extarcted slclient.toml from iks cluster kube-system/storage-secret-store secret into libconfig.toml
• tried to establish new client with this toml. and still have errors:

{"level":"info","ts":1562182636.9882107,"caller":"utils/init_provider.go:44","msg":"Block softlayer provider volume registry done!"}
{"level":"info","ts":1562182636.988369,"caller":"utils/init_provider.go:93","msg":"Provider registration done!!!"}
{"level":"info","ts":1562182636.9884503,"caller":"utils/init_provider.go:133","msg":"Generating generateContextCredentials for ","Provider ID":"SOFTLAYER-BLOCK"}
{"level":"info","ts":1562182636.988554,"caller":"iam/token_exchange.go:180","msg":"Sending IAM token exchange request"}
... Panic: runtime error: invalid memory address or nil pointer dereference (PC=0x45B60A)

/usr/local/go/src/runtime/panic.go:513
  in gopanic
/usr/local/go/src/runtime/panic.go:82
  in panicmem
/usr/local/go/src/runtime/signal_unix.go:390
  in sigpanic
/home/ilya/k10/go/src/vendor/go.uber.org/zap/logger.go:264
  in Logger.check
/home/ilya/k10/go/src/vendor/go.uber.org/zap/logger.go:186
  in Logger.Info
/home/ilya/k10/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/softlayer/backend/utils.go:44
  in retry
/home/ilya/k10/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/softlayer/backend/account_service.go:99
  in AccountServiceSL.GetObject
/home/ilya/k10/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/volume-providers/softlayer/block/provider.go:143
  in SLBlockProvider.OpenSession
/home/ilya/k10/go/src/vendor/github.com/IBM/ibmcloud-storage-volume-lib/provider/utils/init_provider.go:121
  in OpenProviderSession

[prgavali]

@depohmel Can you paste your config file masking your credentials ?

[depohmel]

@prgavali here is my config

  iam_url = "https://iam.bluemix.net"
  iam_client_id = "bx"
  iam_client_secret = "bx"
  iam_api_key = "XXXX"
  refresh_token = ""
  pay_tier = "paid"
  containers_api_route = "https://us-south.containers.cloud.ibm.com"
  encryption = false

[Softlayer]
  encryption = true
  softlayer_username = ""
  softlayer_api_key = ""
  softlayer_endpoint_url = "https://api.service.softlayer.com/rest/v3"
  softlayer_iam_endpoint_url = "https://api.service.softlayer.com/mobile/v3"
  softlayer_datacenter = "sjc03"
  softlayer_token_exchange_endpoint_url = "https://iam.bluemix.net"

[VPC]
  gc_token_exchange_endpoint_url = "https://iam.bluemix.net"
  gc_riaas_endpoint_url = "https://us-south.iaas.cloud.ibm.com:443"
  gc_resource_group_id = "XXXX"
  gc_api_key = "XXXXXX"
  encryption = true

[prgavali]

@depohmel Remove service from below URLs .. These are private endpoints and works from IBM network only.
softlayer_endpoint_url = "https://api.service.softlayer.com/rest/v3" softlayer_iam_endpoint_url = "https://api.service.softlayer.com/mobile/v3"

Below are the public endpoints . They should work from outside
softlayer_endpoint_url = "https://api.softlayer.com/rest/v3" softlayer_iam_endpoint_url = "https://api.softlayer.com/mobile/v3"

Anyways ,I fixed the nil pointer reference to logger. So you should be able to see actual error message in the log