More workload identity, this time in the oppsoite direction. Specifically, running the workload in Amazon Web Services (AWS) Elastic Kubernetes Service (EKS), accessing Google Cloud Platform (GCP) services.
There have been many blog posts on this.
On GCP, we need a workload identity pool associated with the AWS account. Additionally, a service account with permissions allowing an AWS Role to impersonate it. Permissions to other GCP resources should be granted to the service account.
1terraform {
2 required_providers {
3 google = {
4 source = "hashicorp/google"
5 version = "4.43.0"
6 }
7 }
8}
9
10locals {
11 project = "project-id"
12 project_number = "000000000"
13 aws_account_id = "000000000"
14}
15
16resource "google_iam_workload_identity_pool" "p" {
17 project = local.project
18 workload_identity_pool_id = "workload-pool-id"
19}
20
21#
22# provider within the pool
23# maps to individual AWS accounts
24#
25resource "google_iam_workload_identity_pool_provider" "p" {
26 project = local.project
27 workload_identity_pool_id = google_iam_workload_identity_pool.p.workload_identity_pool_id
28 workload_identity_pool_provider_id = "workload-provider-id"
29 attribute_mapping = {
30 "google.subject" = "assertion.arn"
31 "attribute.aws_account" = "assertion.account"
32 # an attribute with the full arn of the assumed role
33 # source attribute is arn:aws:sts::account:assumed-role/role-to-be-assumed/id
34 "attribute.aws_role" = "assertion.arn.contains('assumed-role') ? assertion.arn.extract('{account_arn}assumed-role/') + 'assumed-role/' + assertion.arn.extract('assumed-role/{role_name}/') : assertion.arn"
35 }
36 aws {
37 account_id = local.aws_account_id
38 }
39}
40
41resource "google_service_account" "s" {
42 project = local.project
43 account_id = "gcp-service-account-name"
44}
45
46resource "google_service_account_iam_member" "s" {
47 service_account_id = google_service_account.a.id
48 role = "roles/iam.workloadIdentityUser"
49 # principalSet matches anything with the given attribute
50 # here we're matching on assumed role arn
51 member = "principalSet://iam.googleapis.com/projects/${local.project_number}/locations/global/workloadIdentityPools/workload-pool-id/attribute.aws_role/arn:aws:sts::account:assumed-role/role-for-irsa"
52}
We'll need an AWS Role and with a trust policy to allow the Kubernetes service account to assume the role. The OIDC provider is also necessary but out of scope.
1terraform {
2 required_providers {
3 aws = {
4 source = "hashicorp/aws"
5 version = "4.39.0"
6 }
7 }
8}
9
10locals {
11 oidc_provider = "..."
12 oidc_provider_arn = "arn:aws:iam::000000000:oidc-provider/${local.oidc_provider}"
13}
14
15resource "aws_iam_role" "r" {
16 name = "role-for-irsa"
17
18 assume_role_policy = jsonencode({
19 Version = "2012-10-17"
20 Statement = [{
21 Action = "sts:AssumeRoleWithWebIdentity"
22 Effect = "Allow"
23 Sid = "r1"
24 Principal = {
25 # built in federated identity provider
26 Federated = local.oidc_provider_arn
27 }
28 Condition = {
29 StringEquals = {
30 "${local.oidc_provider}:sub" = "system:serviceaccount:k8s-namespace:serviceaccount-name"
31 }
32 }
33 }]
34 })
35}
We'll need a kubernetes service account, annotated with the AWS Role it can use as an identity.
1apiVersion: v1
2kind: ServiceAccount
3metadata:
4 name: serviceaccoount-name
5 annotations:
6 eks.amazonaws.com/role-arn: arn:aws:iam::account:role/role-for-irsa
The GCP SDK can use a config file to handle the token exchange. It needs an audience referencing the workload identity pool provider, and the intended service account to impersonate.
1apiVersion: v1
2kind: ConfigMap
3metadata:
4 name: configmap-name
5data:
6 creds.json: |
7 {
8 "type": "external_account",
9 "audience": "//iam.googleapis.com/projects/project-number/locations/global/workloadIdentityPools/workload-pool-id/providers/workload-provider-id",
10 "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request",
11 "token_url": "https://sts.googleapis.com/v1/token",
12 "credential_source": {
13 "environment_id": "aws1",
14 "region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone",
15 "url": "http://169.254.169.254/latest/meta-data/iam/security-credentials",
16 "regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15",
17 "imdsv2_session_token_url": "http://169.254.169.254/latest/api/token"
18 },
19 "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/gcp-service-account-name@project-id.iam.gserviceaccount.com:generateAccessToken"
20 }
The pod needs to use the service account with the role annotation.
Additionally, it needs to mount the above config file, and point GOOGLE_APPLICATION_CREDENTIALS to it.
1apiVersion: v1
2kind: Pod
3metadata:
4 name: pod-name
5spec:
6 serviceAccountName: serviceaccount-name
7 containers:
8 - name: container-name
9 image: image-ref
10 env:
11 - name: GOOGLE_APPLICATION_CREDENTIALS
12 value: /var/run/secrets/google/creds.json
13 volumeMounts:
14 - name: google
15 mountPath: /var/run/secrets/google
16 volumes:
17 - name: google
18 configMap:
19 name: configmap-name
The GCP Go SDK has built in support to do the necessary token exchanges with AWS STS.
All it needs is for GOOGLE_APPLICATION_CREDENTIALS to point to a config file (see above).
1package main
2
3import (
4 "context"
5 "fmt"
6 "os"
7
8 "cloud.google.com/go/storage"
9 "golang.org/x/exp/slog"
10)
11
12func main() {
13 lgh := slog.HandlerOptions{
14 Level: slog.DebugLevel,
15 }.NewJSONHandler(os.Stderr)
16 lg := slog.New(lgh)
17
18 err := run(lg)
19 if err != nil {
20 lg.Error("run", err)
21 os.Exit(1)
22 }
23}
24
25func run(lg *slog.Logger) error {
26 ctx := context.Background()
27 // Example: GCP storage client
28 client, err := storage.NewClient(ctx)
29 if err != nil {
30 return fmt.Errorf("create GCP Cloud Storage client: %w", err)
31 }
32
33 _ = client
34 return nil
35}
If for whatever reason, IMDSv2 is broken on your cluster
(maybe someone forgot to set ec2:MetadataHttpPutResponseHopLimit to 2),
you can do a manual token exchange to generate the access tokens for sdk use.
Note: you'll want to remove the GOOGLE_APPLICATION_CREDENTIALS env and the file it points to.
1package main
2
3import (
4 "context"
5 "encoding/json"
6 "fmt"
7 "net/http"
8 "net/url"
9 "os"
10 "sort"
11 "strconv"
12 "strings"
13 "time"
14
15 credentials "cloud.google.com/go/iam/credentials/apiv1"
16 "cloud.google.com/go/iam/credentials/apiv1/credentialspb"
17 "cloud.google.com/go/storage"
18 "github.com/aws/aws-sdk-go-v2/aws"
19 signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
20 "github.com/aws/aws-sdk-go-v2/service/sts"
21 "golang.org/x/exp/slog"
22 "golang.org/x/oauth2"
23 "google.golang.org/api/option"
24 gcpsts "google.golang.org/api/sts/v1"
25)
26
27func main() {
28 lgh := slog.HandlerOptions{
29 Level: slog.DebugLevel,
30 }.NewJSONHandler(os.Stderr)
31 lg := slog.New(lgh)
32
33 err := run(lg)
34 if err != nil {
35 lg.Error("run", err)
36 os.Exit(1)
37 }
38}
39
40func run(lg *slog.Logger) error {
41 ctx := context.Background()
42
43 stsTS, err := NewAWSToGCPTokenSource(ctx)
44 if err != nil {
45 return fmt.Errorf("create gcp sts token source: %w", err)
46 }
47 ts, err := NewGCPServiceAccountTokenSource(ctx, stsTS)
48 if err != nil {
49 return fmt.Errorf("create gcp service account token source: %w", err)
50 }
51
52 // Example: GCP storage client
53 client, _ := storage.NewClient(ctx,
54 // use our access tokens
55 option.WithTokenSource(ts),
56 )
57 _ = client
58 return nil
59}
60
61type AWSToGCPTokenSource struct {
62 s *gcpsts.Service
63}
64
65func NewAWSToGCPTokenSource(ctx context.Context) (*AWSToGCPTokenSource, error) {
66 client, err := gcpsts.NewService(ctx, option.WithoutAuthentication())
67 if err != nil {
68 return nil, fmt.Errorf("create gcp sts service client: %w", err)
69 }
70
71 return &AWSToGCPTokenSource{
72 s: client,
73 }, nil
74}
75
76// Token implements oauth2.TokenSource by taking a local filesystem mounted EKS IRSA token,
77// assuming the associated AWS Role, and exchanging that for a GCP STS token.
78func (g *AWSToGCPTokenSource) Token() (*oauth2.Token, error) {
79 // potential input values
80 region := os.Getenv("AWS_REGION") // injected by IRSA
81 roleARN := os.Getenv("AWS_ROLE_ARN") // injected by IRSA
82 gcpProjectNumber := "000000000"
83 gcpWorkloadPool := "workload-pool-id"
84 gcpProvider := "workload-provider-id"
85 gcpTargetResource := "//iam.googleapis.com/projects/" + gcpProjectNumber + "/locations/global/workloadIdentityPools/" + gcpWorkloadPool + "/providers/" + gcpProvider
86
87 // get the projected OIDC token within EKS
88 // this has an audience of STS and is updated over time
89 b, err := os.ReadFile("/var/run/secrets/eks.amazonaws.com/serviceaccount/token")
90 if err != nil {
91 return nil, fmt.Errorf("read mounted eks token: %w", err)
92 }
93 webToken := string(b)
94
95 // assume the role mapped by IRSA using the OIDC token
96 ctx := context.Background()
97 tsNow := strconv.FormatInt(time.Now().Unix(), 10)
98 assumed, err := sts.New(sts.Options{
99 Region: region,
100 }).AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityInput{
101 RoleArn: &roleARN,
102 RoleSessionName: &tsNow,
103 WebIdentityToken: &webToken,
104 })
105 if err != nil {
106 return nil, fmt.Errorf("assume role: %w", err)
107 }
108
109 // prepare and sign an HTTP request equivalent to a call to AWS sts.GetCallerIdentity.
110 // hand the resulting URL / Method / Headers to GCP STS, which will use it to confirm our identity,
111 // and give us a token.
112 gciReq, err := http.NewRequest(
113 http.MethodPost,
114 "https://sts."+region+".amazonaws.com/?Action=GetCallerIdentity&Version=2011-06-15",
115 nil,
116 )
117 if err != nil {
118 return nil, fmt.Errorf("create fake request: %w", err)
119 }
120 // equivalent to setting an audience in oauth2
121 gciReq.Header.Set(
122 "x-goog-cloud-target-resource",
123 gcpTargetResource,
124 )
125 err = signer.NewSigner().SignHTTP(ctx,
126 aws.Credentials{
127 AccessKeyID: *assumed.Credentials.AccessKeyId,
128 SecretAccessKey: *assumed.Credentials.SecretAccessKey,
129 SessionToken: *assumed.Credentials.SessionToken,
130 CanExpire: true,
131 Expires: *assumed.Credentials.Expiration,
132 },
133 gciReq,
134 // empty string representing the body
135 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
136 "sts",
137 os.Getenv("AWS_DEFAULT_REGION"),
138 time.Now(),
139 )
140 if err != nil {
141 return nil, fmt.Errorf("sign fake GetCallerIdentity request: %w", err)
142 }
143
144 var headers []map[string]string
145 // host header is implicit in Go's net/http.Request
146 headers = append(headers, map[string]string{
147 "key": "host",
148 "value": gciReq.Host,
149 })
150 for k, vs := range gciReq.Header {
151 headers = append(headers, map[string]string{
152 "key": strings.ToLower(k),
153 "value": vs[0],
154 })
155 }
156 // header fields should be sorted?
157 sort.Slice(headers, func(i, j int) bool {
158 return headers[i]["key"] < headers[j]["key"]
159 })
160
161 // GCP STS expects a url encoded json object containing the request data
162 b, err = json.Marshal(map[string]any{
163 "url": gciReq.URL.String(),
164 "method": gciReq.Method,
165 "headers": headers,
166 })
167 if err != nil {
168 return nil, fmt.Errorf("json encode sts req: %w", err)
169 }
170 escaped := url.QueryEscape(string(b))
171
172 // exchange for a GCP STS token associated with our pool
173 res, err := g.s.V1.Token(&gcpsts.GoogleIdentityStsV1ExchangeTokenRequest{
174 Audience: gcpTargetResource,
175 GrantType: "urn:ietf:params:oauth:grant-type:token-exchange",
176 RequestedTokenType: "urn:ietf:params:oauth:token-type:access_token",
177 Scope: "https://www.googleapis.com/auth/cloud-platform",
178 SubjectToken: escaped,
179 SubjectTokenType: "urn:ietf:params:aws:token-type:aws4_request",
180 }).Do()
181 if err != nil {
182 return nil, fmt.Errorf("gcp sts exchange: %w", err)
183 }
184
185 exp := time.Now().Add(time.Duration(res.ExpiresIn) * time.Second)
186 return &oauth2.Token{
187 AccessToken: res.AccessToken,
188 Expiry: exp,
189 }, nil
190}
191
192type GCPServiceAccountTokenSource struct {
193 g *credentials.IamCredentialsClient
194}
195
196func NewGCPServiceAccountTokenSource(ctx context.Context, ts oauth2.TokenSource) (*GCPServiceAccountTokenSource, error) {
197 client, err := credentials.NewIamCredentialsClient(ctx, option.WithTokenSource(ts))
198 if err != nil {
199 return nil, fmt.Errorf("create gcp credentials client: %w", err)
200 }
201
202 return &GCPServiceAccountTokenSource{
203 g: client,
204 }, nil
205}
206
207// Token implements oauth2.TokenSource by exchanging a GCP STS token
208// for an GCP access token from the impersonated service account
209func (g *GCPServiceAccountTokenSource) Token() (*oauth2.Token, error) {
210 // potential input values
211 gcpEmail := "gcp-service-account-name@project-id.iam.gserviceaccount.com"
212
213 ctx := context.Background()
214 // impersonate the service accoutn and generate an access token
215 res, err := g.g.GenerateAccessToken(ctx, &credentialspb.GenerateAccessTokenRequest{
216 Name: "projects/-/serviceAccounts/" + gcpEmail,
217 Scope: []string{"https://www.googleapis.com/auth/cloud-platform"},
218 })
219 if err != nil {
220 return nil, fmt.Errorf("generate GCP access token: %w", err)
221 }
222
223 return &oauth2.Token{
224 AccessToken: res.AccessToken,
225 Expiry: res.ExpireTime.AsTime(),
226 }, nil
227}