diff --git a/CHANGELOG.md b/CHANGELOG.md index 9daa87d2f2c..507206c82d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828 * [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873 +* [FEATURE] Storage: Add support for Alibaba Cloud OSS as a backend for blocks, ruler, and alertmanager storage. Configured via `-.oss.*` flags with `backend: oss`. #7907 * [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539 * [ENHANCEMENT] gRPC: When `-auth.enabled=true`, a gRPC request without the `X-Scope-OrgID` metadata is now rejected with the `UNAUTHENTICATED` status code instead of `UNKNOWN`. #7873 * [ENHANCEMENT] Upgrade promql-engine and Thanos to latest. #7889 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 166b8f43635..15bea9a237d 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -386,7 +386,7 @@ The `blocks_storage_config` configures the blocks storage. ```yaml blocks_storage: # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, - # filesystem. + # oss, filesystem. # CLI flag: -blocks-storage.backend [backend: | default = "s3"] @@ -718,6 +718,24 @@ blocks_storage: # CLI flag: -blocks-storage.oci.request-retry-interval [request_retry_interval: | default = 10] + oss: + # The Alibaba Cloud OSS endpoint to connect to, e.g. + # oss-cn-hangzhou.aliyuncs.com. + # CLI flag: -blocks-storage.oss.endpoint + [endpoint: | default = ""] + + # The Alibaba Cloud OSS bucket name. + # CLI flag: -blocks-storage.oss.bucket + [bucket: | default = ""] + + # The Alibaba Cloud OSS access key ID. + # CLI flag: -blocks-storage.oss.access-key-id + [access_key_id: | default = ""] + + # The Alibaba Cloud OSS access key secret. + # CLI flag: -blocks-storage.oss.access-key-secret + [access_key_secret: | default = ""] + filesystem: # Local filesystem storage directory. # CLI flag: -blocks-storage.filesystem.dir diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 8c43bcb4653..0dc4d398023 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -439,7 +439,7 @@ The `blocks_storage_config` configures the blocks storage. ```yaml blocks_storage: # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, - # filesystem. + # oss, filesystem. # CLI flag: -blocks-storage.backend [backend: | default = "s3"] @@ -771,6 +771,24 @@ blocks_storage: # CLI flag: -blocks-storage.oci.request-retry-interval [request_retry_interval: | default = 10] + oss: + # The Alibaba Cloud OSS endpoint to connect to, e.g. + # oss-cn-hangzhou.aliyuncs.com. + # CLI flag: -blocks-storage.oss.endpoint + [endpoint: | default = ""] + + # The Alibaba Cloud OSS bucket name. + # CLI flag: -blocks-storage.oss.bucket + [bucket: | default = ""] + + # The Alibaba Cloud OSS access key ID. + # CLI flag: -blocks-storage.oss.access-key-id + [access_key_id: | default = ""] + + # The Alibaba Cloud OSS access key secret. + # CLI flag: -blocks-storage.oss.access-key-secret + [access_key_secret: | default = ""] + filesystem: # Local filesystem storage directory. # CLI flag: -blocks-storage.filesystem.dir diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 628ee3f9027..3f3fd6145a6 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -725,7 +725,7 @@ The `alertmanager_storage_config` configures the Cortex alertmanager storage bac ```yaml # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, -# filesystem, configdb, local. +# oss, filesystem, configdb, local. # CLI flag: -alertmanager-storage.backend [backend: | default = "s3"] @@ -1055,6 +1055,24 @@ oci: # CLI flag: -alertmanager-storage.oci.request-retry-interval [request_retry_interval: | default = 10] +oss: + # The Alibaba Cloud OSS endpoint to connect to, e.g. + # oss-cn-hangzhou.aliyuncs.com. + # CLI flag: -alertmanager-storage.oss.endpoint + [endpoint: | default = ""] + + # The Alibaba Cloud OSS bucket name. + # CLI flag: -alertmanager-storage.oss.bucket + [bucket: | default = ""] + + # The Alibaba Cloud OSS access key ID. + # CLI flag: -alertmanager-storage.oss.access-key-id + [access_key_id: | default = ""] + + # The Alibaba Cloud OSS access key secret. + # CLI flag: -alertmanager-storage.oss.access-key-secret + [access_key_secret: | default = ""] + filesystem: # Local filesystem storage directory. # CLI flag: -alertmanager-storage.filesystem.dir @@ -1097,7 +1115,7 @@ The `blocks_storage_config` configures the blocks storage. ```yaml # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, -# filesystem. +# oss, filesystem. # CLI flag: -blocks-storage.backend [backend: | default = "s3"] @@ -1427,6 +1445,24 @@ oci: # CLI flag: -blocks-storage.oci.request-retry-interval [request_retry_interval: | default = 10] +oss: + # The Alibaba Cloud OSS endpoint to connect to, e.g. + # oss-cn-hangzhou.aliyuncs.com. + # CLI flag: -blocks-storage.oss.endpoint + [endpoint: | default = ""] + + # The Alibaba Cloud OSS bucket name. + # CLI flag: -blocks-storage.oss.bucket + [bucket: | default = ""] + + # The Alibaba Cloud OSS access key ID. + # CLI flag: -blocks-storage.oss.access-key-id + [access_key_id: | default = ""] + + # The Alibaba Cloud OSS access key secret. + # CLI flag: -blocks-storage.oss.access-key-secret + [access_key_secret: | default = ""] + filesystem: # Local filesystem storage directory. # CLI flag: -blocks-storage.filesystem.dir @@ -6329,7 +6365,7 @@ The `ruler_storage_config` configures the Cortex ruler storage backend. ```yaml # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, -# filesystem, configdb, local. +# oss, filesystem, configdb, local. # CLI flag: -ruler-storage.backend [backend: | default = "s3"] @@ -6659,6 +6695,24 @@ oci: # CLI flag: -ruler-storage.oci.request-retry-interval [request_retry_interval: | default = 10] +oss: + # The Alibaba Cloud OSS endpoint to connect to, e.g. + # oss-cn-hangzhou.aliyuncs.com. + # CLI flag: -ruler-storage.oss.endpoint + [endpoint: | default = ""] + + # The Alibaba Cloud OSS bucket name. + # CLI flag: -ruler-storage.oss.bucket + [bucket: | default = ""] + + # The Alibaba Cloud OSS access key ID. + # CLI flag: -ruler-storage.oss.access-key-id + [access_key_id: | default = ""] + + # The Alibaba Cloud OSS access key secret. + # CLI flag: -ruler-storage.oss.access-key-secret + [access_key_secret: | default = ""] + filesystem: # Local filesystem storage directory. # CLI flag: -ruler-storage.filesystem.dir @@ -6709,7 +6763,7 @@ The `runtime_configuration_storage_config` configures the storage backend for th [file: | default = ""] # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, -# filesystem. +# oss, filesystem. # CLI flag: -runtime-config.backend [backend: | default = "filesystem"] @@ -7039,6 +7093,24 @@ oci: # CLI flag: -runtime-config.oci.request-retry-interval [request_retry_interval: | default = 10] +oss: + # The Alibaba Cloud OSS endpoint to connect to, e.g. + # oss-cn-hangzhou.aliyuncs.com. + # CLI flag: -runtime-config.oss.endpoint + [endpoint: | default = ""] + + # The Alibaba Cloud OSS bucket name. + # CLI flag: -runtime-config.oss.bucket + [bucket: | default = ""] + + # The Alibaba Cloud OSS access key ID. + # CLI flag: -runtime-config.oss.access-key-id + [access_key_id: | default = ""] + + # The Alibaba Cloud OSS access key secret. + # CLI flag: -runtime-config.oss.access-key-secret + [access_key_secret: | default = ""] + filesystem: # Local filesystem storage directory. # CLI flag: -runtime-config.filesystem.dir diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md index 9bba2b68bad..d1561ac9af5 100644 --- a/docs/configuration/v1-guarantees.md +++ b/docs/configuration/v1-guarantees.md @@ -47,6 +47,7 @@ Currently experimental features are: - TLS configuration in Etcd client. - OpenStack Swift storage support. - Oracle Cloud Infrastructure (OCI) Object Storage support. +- Alibaba Cloud OSS storage support. - Metric relabeling in the distributor. - Ingester: do not unregister from ring on shutdown (`-ingester.unregister-on-shutdown=false`) - Distributor: diff --git a/pkg/storage/bucket/client.go b/pkg/storage/bucket/client.go index 8e9e6fd2f96..375774b285c 100644 --- a/pkg/storage/bucket/client.go +++ b/pkg/storage/bucket/client.go @@ -18,6 +18,7 @@ import ( "github.com/cortexproject/cortex/pkg/storage/bucket/filesystem" "github.com/cortexproject/cortex/pkg/storage/bucket/gcs" "github.com/cortexproject/cortex/pkg/storage/bucket/oci" + "github.com/cortexproject/cortex/pkg/storage/bucket/oss" "github.com/cortexproject/cortex/pkg/storage/bucket/s3" "github.com/cortexproject/cortex/pkg/storage/bucket/swift" ) @@ -38,12 +39,15 @@ const ( // OCI is the value for the Oracle Cloud Infrastructure storage backend. OCI = "oci" + // OSS is the value for the Alibaba Cloud OSS storage backend. + OSS = "oss" + // Filesystem is the value for the filesystem storage backend. Filesystem = "filesystem" ) var ( - SupportedBackends = []string{S3, GCS, Azure, Swift, OCI, Filesystem} + SupportedBackends = []string{S3, GCS, Azure, Swift, OCI, OSS, Filesystem} ErrUnsupportedStorageBackend = errors.New("unsupported storage backend") @@ -59,6 +63,7 @@ type Config struct { Azure azure.Config `yaml:"azure"` Swift swift.Config `yaml:"swift"` OCI oci.Config `yaml:"oci"` + OSS oss.Config `yaml:"oss"` Filesystem filesystem.Config `yaml:"filesystem"` // Not used internally, meant to allow callers to wrap Buckets @@ -90,6 +95,7 @@ func (cfg *Config) RegisterFlagsWithPrefixAndBackend(prefix string, f *flag.Flag cfg.Azure.RegisterFlagsWithPrefix(prefix, f) cfg.Swift.RegisterFlagsWithPrefix(prefix, f) cfg.OCI.RegisterFlagsWithPrefix(prefix, f) + cfg.OSS.RegisterFlagsWithPrefix(prefix, f) cfg.Filesystem.RegisterFlagsWithPrefix(prefix, f) f.StringVar(&cfg.Backend, prefix+"backend", defaultBackend, fmt.Sprintf("Backend storage to use. Supported backends are: %s.", strings.Join(cfg.supportedBackends(), ", "))) @@ -123,6 +129,8 @@ func NewClient(ctx context.Context, cfg Config, hedgedRoundTripper func(rt http. client, err = swift.NewBucketClient(cfg.Swift, hedgedRoundTripper, name, logger) case OCI: client, err = oci.NewBucketClient(cfg.OCI, hedgedRoundTripper, name, logger) + case OSS: + client, err = oss.NewBucketClient(cfg.OSS, hedgedRoundTripper, name, logger) case Filesystem: client, err = filesystem.NewBucketClient(cfg.Filesystem) default: diff --git a/pkg/storage/bucket/oss/bucket_client.go b/pkg/storage/bucket/oss/bucket_client.go new file mode 100644 index 00000000000..3ee1dcab401 --- /dev/null +++ b/pkg/storage/bucket/oss/bucket_client.go @@ -0,0 +1,26 @@ +package oss + +import ( + "net/http" + + "github.com/go-kit/log" + "github.com/thanos-io/objstore" + "github.com/thanos-io/objstore/providers/oss" +) + +// NewBucketClient creates a new Alibaba Cloud OSS bucket client. +func NewBucketClient(cfg Config, hedgedRoundTripper func(rt http.RoundTripper) http.RoundTripper, name string, logger log.Logger) (objstore.Bucket, error) { + bucketConfig := oss.Config{ + Endpoint: cfg.Endpoint, + Bucket: cfg.Bucket, + AccessKeyID: cfg.AccessKeyID, + AccessKeySecret: cfg.AccessKeySecret.Value, + } + + // Check the error explicitly so a nil *oss.Bucket is not returned as a non-nil objstore.Bucket. + bkt, err := oss.NewBucketWithConfig(logger, bucketConfig, name, hedgedRoundTripper) + if err != nil { + return nil, err + } + return bkt, nil +} diff --git a/pkg/storage/bucket/oss/config.go b/pkg/storage/bucket/oss/config.go new file mode 100644 index 00000000000..8018761610a --- /dev/null +++ b/pkg/storage/bucket/oss/config.go @@ -0,0 +1,28 @@ +package oss + +import ( + "flag" + + "github.com/cortexproject/cortex/pkg/util/flagext" +) + +// Config holds the config options for an Alibaba Cloud OSS backend +type Config struct { + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` + AccessKeyID string `yaml:"access_key_id"` + AccessKeySecret flagext.Secret `yaml:"access_key_secret"` +} + +// RegisterFlags registers the flags for Alibaba Cloud OSS storage +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.RegisterFlagsWithPrefix("", f) +} + +// RegisterFlagsWithPrefix registers the flags for Alibaba Cloud OSS storage with the provided prefix +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.Endpoint, prefix+"oss.endpoint", "", "The Alibaba Cloud OSS endpoint to connect to, e.g. oss-cn-hangzhou.aliyuncs.com.") + f.StringVar(&cfg.Bucket, prefix+"oss.bucket", "", "The Alibaba Cloud OSS bucket name.") + f.StringVar(&cfg.AccessKeyID, prefix+"oss.access-key-id", "", "The Alibaba Cloud OSS access key ID.") + f.Var(&cfg.AccessKeySecret, prefix+"oss.access-key-secret", "The Alibaba Cloud OSS access key secret.") +} diff --git a/pkg/storage/bucket/oss/config_test.go b/pkg/storage/bucket/oss/config_test.go new file mode 100644 index 00000000000..1208b1ca200 --- /dev/null +++ b/pkg/storage/bucket/oss/config_test.go @@ -0,0 +1,74 @@ +package oss + +import ( + "testing" + + "github.com/go-kit/log" + "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v2" + + "github.com/cortexproject/cortex/pkg/util/flagext" +) + +func TestConfig(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + config string + expectedConfig Config + expectedErr error + }{ + "default config": { + config: "", + expectedConfig: Config{}, + expectedErr: nil, + }, + "custom config": { + config: ` +endpoint: oss-cn-hangzhou.aliyuncs.com +bucket: test-bucket +access_key_id: test-access-key-id +access_key_secret: test-access-key-secret +`, + expectedConfig: Config{ + Endpoint: "oss-cn-hangzhou.aliyuncs.com", + Bucket: "test-bucket", + AccessKeyID: "test-access-key-id", + AccessKeySecret: flagext.Secret{Value: "test-access-key-secret"}, + }, + expectedErr: nil, + }, + "invalid type": { + config: `bucket: [foo]`, + expectedConfig: Config{}, + expectedErr: &yaml.TypeError{Errors: []string{"line 1: cannot unmarshal !!seq into string"}}, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + cfg := Config{} + flagext.DefaultValues(&cfg) + + err := yaml.Unmarshal([]byte(testData.config), &cfg) + require.Equal(t, testData.expectedErr, err) + require.Equal(t, testData.expectedConfig, cfg) + }) + } +} + +func TestNewBucketClient(t *testing.T) { + t.Parallel() + + _, err := NewBucketClient(Config{}, nil, "test", log.NewNopLogger()) + require.ErrorContains(t, err, "endpoint or bucket is not present") + + bkt, err := NewBucketClient(Config{ + Endpoint: "oss-cn-hangzhou.aliyuncs.com", + Bucket: "test-bucket", + AccessKeyID: "id", + AccessKeySecret: flagext.Secret{Value: "secret"}, + }, nil, "test", log.NewNopLogger()) + require.NoError(t, err) + require.Equal(t, "test-bucket", bkt.Name()) +} diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index b48a37271d3..280f9e9d1d0 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -640,7 +640,7 @@ }, "backend": { "default": "s3", - "description": "Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, filesystem, configdb, local.", + "description": "Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, oss, filesystem, configdb, local.", "type": "string", "x-cli-flag": "alertmanager-storage.backend" }, @@ -751,6 +751,31 @@ }, "type": "object" }, + "oss": { + "properties": { + "access_key_id": { + "description": "The Alibaba Cloud OSS access key ID.", + "type": "string", + "x-cli-flag": "alertmanager-storage.oss.access-key-id" + }, + "access_key_secret": { + "description": "The Alibaba Cloud OSS access key secret.", + "type": "string", + "x-cli-flag": "alertmanager-storage.oss.access-key-secret" + }, + "bucket": { + "description": "The Alibaba Cloud OSS bucket name.", + "type": "string", + "x-cli-flag": "alertmanager-storage.oss.bucket" + }, + "endpoint": { + "description": "The Alibaba Cloud OSS endpoint to connect to, e.g. oss-cn-hangzhou.aliyuncs.com.", + "type": "string", + "x-cli-flag": "alertmanager-storage.oss.endpoint" + } + }, + "type": "object" + }, "s3": { "properties": { "access_key_id": { @@ -1136,7 +1161,7 @@ }, "backend": { "default": "s3", - "description": "Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, filesystem.", + "description": "Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, oss, filesystem.", "type": "string", "x-cli-flag": "blocks-storage.backend" }, @@ -3186,6 +3211,31 @@ }, "type": "object" }, + "oss": { + "properties": { + "access_key_id": { + "description": "The Alibaba Cloud OSS access key ID.", + "type": "string", + "x-cli-flag": "blocks-storage.oss.access-key-id" + }, + "access_key_secret": { + "description": "The Alibaba Cloud OSS access key secret.", + "type": "string", + "x-cli-flag": "blocks-storage.oss.access-key-secret" + }, + "bucket": { + "description": "The Alibaba Cloud OSS bucket name.", + "type": "string", + "x-cli-flag": "blocks-storage.oss.bucket" + }, + "endpoint": { + "description": "The Alibaba Cloud OSS endpoint to connect to, e.g. oss-cn-hangzhou.aliyuncs.com.", + "type": "string", + "x-cli-flag": "blocks-storage.oss.endpoint" + } + }, + "type": "object" + }, "s3": { "properties": { "access_key_id": { @@ -8065,7 +8115,7 @@ }, "backend": { "default": "s3", - "description": "Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, filesystem, configdb, local.", + "description": "Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, oss, filesystem, configdb, local.", "type": "string", "x-cli-flag": "ruler-storage.backend" }, @@ -8176,6 +8226,31 @@ }, "type": "object" }, + "oss": { + "properties": { + "access_key_id": { + "description": "The Alibaba Cloud OSS access key ID.", + "type": "string", + "x-cli-flag": "ruler-storage.oss.access-key-id" + }, + "access_key_secret": { + "description": "The Alibaba Cloud OSS access key secret.", + "type": "string", + "x-cli-flag": "ruler-storage.oss.access-key-secret" + }, + "bucket": { + "description": "The Alibaba Cloud OSS bucket name.", + "type": "string", + "x-cli-flag": "ruler-storage.oss.bucket" + }, + "endpoint": { + "description": "The Alibaba Cloud OSS endpoint to connect to, e.g. oss-cn-hangzhou.aliyuncs.com.", + "type": "string", + "x-cli-flag": "ruler-storage.oss.endpoint" + } + }, + "type": "object" + }, "s3": { "properties": { "access_key_id": { @@ -8561,7 +8636,7 @@ }, "backend": { "default": "filesystem", - "description": "Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, filesystem.", + "description": "Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci, oss, filesystem.", "type": "string", "x-cli-flag": "runtime-config.backend" }, @@ -8664,6 +8739,31 @@ }, "type": "object" }, + "oss": { + "properties": { + "access_key_id": { + "description": "The Alibaba Cloud OSS access key ID.", + "type": "string", + "x-cli-flag": "runtime-config.oss.access-key-id" + }, + "access_key_secret": { + "description": "The Alibaba Cloud OSS access key secret.", + "type": "string", + "x-cli-flag": "runtime-config.oss.access-key-secret" + }, + "bucket": { + "description": "The Alibaba Cloud OSS bucket name.", + "type": "string", + "x-cli-flag": "runtime-config.oss.bucket" + }, + "endpoint": { + "description": "The Alibaba Cloud OSS endpoint to connect to, e.g. oss-cn-hangzhou.aliyuncs.com.", + "type": "string", + "x-cli-flag": "runtime-config.oss.endpoint" + } + }, + "type": "object" + }, "period": { "default": "10s", "description": "How often to check runtime config file.",