Add Alibaba Cloud OSS storage backend

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 * [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828
 * [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873
+* [FEATURE] Storage: Add support for Alibaba Cloud OSS as a backend for blocks, ruler, and alertmanager storage. Configured via `-<prefix>.oss.*` flags with `backend: oss`. #7907
 * [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539
 * [ENHANCEMENT] gRPC: When `-auth.enabled=true`, a gRPC request without the `X-Scope-OrgID` metadata is now rejected with the `UNAUTHENTICATED` status code instead of `UNKNOWN`. #7873
 * [ENHANCEMENT] Upgrade promql-engine and Thanos to latest. #7889

docs/blocks-storage/querier.md

@@ -386,7 +386,7 @@ The `blocks_storage_config` configures the blocks storage.
 ```yaml
 blocks_storage:
   # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci,
-  # filesystem.
+  # oss, filesystem.
   # CLI flag: -blocks-storage.backend
   [backend: <string> | default = "s3"]
 
@@ -718,6 +718,24 @@ blocks_storage:
     # CLI flag: -blocks-storage.oci.request-retry-interval
     [request_retry_interval: <int> | default = 10]
 
+  oss:
+    # The Alibaba Cloud OSS endpoint to connect to, e.g.
+    # oss-cn-hangzhou.aliyuncs.com.
+    # CLI flag: -blocks-storage.oss.endpoint
+    [endpoint: <string> | default = ""]
+
+    # The Alibaba Cloud OSS bucket name.
+    # CLI flag: -blocks-storage.oss.bucket
+    [bucket: <string> | default = ""]
+
+    # The Alibaba Cloud OSS access key ID.
+    # CLI flag: -blocks-storage.oss.access-key-id
+    [access_key_id: <string> | default = ""]
+
+    # The Alibaba Cloud OSS access key secret.
+    # CLI flag: -blocks-storage.oss.access-key-secret
+    [access_key_secret: <string> | default = ""]
+
   filesystem:
     # Local filesystem storage directory.
     # CLI flag: -blocks-storage.filesystem.dir

docs/blocks-storage/store-gateway.md

@@ -439,7 +439,7 @@ The `blocks_storage_config` configures the blocks storage.
 ```yaml
 blocks_storage:
   # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci,
-  # filesystem.
+  # oss, filesystem.
   # CLI flag: -blocks-storage.backend
   [backend: <string> | default = "s3"]
 
@@ -771,6 +771,24 @@ blocks_storage:
     # CLI flag: -blocks-storage.oci.request-retry-interval
     [request_retry_interval: <int> | default = 10]
 
+  oss:
+    # The Alibaba Cloud OSS endpoint to connect to, e.g.
+    # oss-cn-hangzhou.aliyuncs.com.
+    # CLI flag: -blocks-storage.oss.endpoint
+    [endpoint: <string> | default = ""]
+
+    # The Alibaba Cloud OSS bucket name.
+    # CLI flag: -blocks-storage.oss.bucket
+    [bucket: <string> | default = ""]
+
+    # The Alibaba Cloud OSS access key ID.
+    # CLI flag: -blocks-storage.oss.access-key-id
+    [access_key_id: <string> | default = ""]
+
+    # The Alibaba Cloud OSS access key secret.
+    # CLI flag: -blocks-storage.oss.access-key-secret
+    [access_key_secret: <string> | default = ""]
+
   filesystem:
     # Local filesystem storage directory.
     # CLI flag: -blocks-storage.filesystem.dir

docs/configuration/config-file-reference.md

@@ -725,7 +725,7 @@ The `alertmanager_storage_config` configures the Cortex alertmanager storage bac
 
 ```yaml
 # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci,
-# filesystem, configdb, local.
+# oss, filesystem, configdb, local.
 # CLI flag: -alertmanager-storage.backend
 [backend: <string> | default = "s3"]
 
@@ -1055,6 +1055,24 @@ oci:
   # CLI flag: -alertmanager-storage.oci.request-retry-interval
   [request_retry_interval: <int> | default = 10]
 
+oss:
+  # The Alibaba Cloud OSS endpoint to connect to, e.g.
+  # oss-cn-hangzhou.aliyuncs.com.
+  # CLI flag: -alertmanager-storage.oss.endpoint
+  [endpoint: <string> | default = ""]
+
+  # The Alibaba Cloud OSS bucket name.
+  # CLI flag: -alertmanager-storage.oss.bucket
+  [bucket: <string> | default = ""]
+
+  # The Alibaba Cloud OSS access key ID.
+  # CLI flag: -alertmanager-storage.oss.access-key-id
+  [access_key_id: <string> | default = ""]
+
+  # The Alibaba Cloud OSS access key secret.
+  # CLI flag: -alertmanager-storage.oss.access-key-secret
+  [access_key_secret: <string> | default = ""]
+
 filesystem:
   # Local filesystem storage directory.
   # CLI flag: -alertmanager-storage.filesystem.dir
@@ -1097,7 +1115,7 @@ The `blocks_storage_config` configures the blocks storage.
 
 ```yaml
 # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci,
-# filesystem.
+# oss, filesystem.
 # CLI flag: -blocks-storage.backend
 [backend: <string> | default = "s3"]
 
@@ -1427,6 +1445,24 @@ oci:
   # CLI flag: -blocks-storage.oci.request-retry-interval
   [request_retry_interval: <int> | default = 10]
 
+oss:
+  # The Alibaba Cloud OSS endpoint to connect to, e.g.
+  # oss-cn-hangzhou.aliyuncs.com.
+  # CLI flag: -blocks-storage.oss.endpoint
+  [endpoint: <string> | default = ""]
+
+  # The Alibaba Cloud OSS bucket name.
+  # CLI flag: -blocks-storage.oss.bucket
+  [bucket: <string> | default = ""]
+
+  # The Alibaba Cloud OSS access key ID.
+  # CLI flag: -blocks-storage.oss.access-key-id
+  [access_key_id: <string> | default = ""]
+
+  # The Alibaba Cloud OSS access key secret.
+  # CLI flag: -blocks-storage.oss.access-key-secret
+  [access_key_secret: <string> | default = ""]
+
 filesystem:
   # Local filesystem storage directory.
   # CLI flag: -blocks-storage.filesystem.dir
@@ -6329,7 +6365,7 @@ The `ruler_storage_config` configures the Cortex ruler storage backend.
 
 ```yaml
 # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci,
-# filesystem, configdb, local.
+# oss, filesystem, configdb, local.
 # CLI flag: -ruler-storage.backend
 [backend: <string> | default = "s3"]
 
@@ -6659,6 +6695,24 @@ oci:
   # CLI flag: -ruler-storage.oci.request-retry-interval
   [request_retry_interval: <int> | default = 10]
 
+oss:
+  # The Alibaba Cloud OSS endpoint to connect to, e.g.
+  # oss-cn-hangzhou.aliyuncs.com.
+  # CLI flag: -ruler-storage.oss.endpoint
+  [endpoint: <string> | default = ""]
+
+  # The Alibaba Cloud OSS bucket name.
+  # CLI flag: -ruler-storage.oss.bucket
+  [bucket: <string> | default = ""]
+
+  # The Alibaba Cloud OSS access key ID.
+  # CLI flag: -ruler-storage.oss.access-key-id
+  [access_key_id: <string> | default = ""]
+
+  # The Alibaba Cloud OSS access key secret.
+  # CLI flag: -ruler-storage.oss.access-key-secret
+  [access_key_secret: <string> | default = ""]
+
 filesystem:
   # Local filesystem storage directory.
   # CLI flag: -ruler-storage.filesystem.dir
@@ -6709,7 +6763,7 @@ The `runtime_configuration_storage_config` configures the storage backend for th
 [file: <string> | default = ""]
 
 # Backend storage to use. Supported backends are: s3, gcs, azure, swift, oci,
-# filesystem.
+# oss, filesystem.
 # CLI flag: -runtime-config.backend
 [backend: <string> | default = "filesystem"]
 
@@ -7039,6 +7093,24 @@ oci:
   # CLI flag: -runtime-config.oci.request-retry-interval
   [request_retry_interval: <int> | default = 10]
 
+oss:
+  # The Alibaba Cloud OSS endpoint to connect to, e.g.
+  # oss-cn-hangzhou.aliyuncs.com.
+  # CLI flag: -runtime-config.oss.endpoint
+  [endpoint: <string> | default = ""]
+
+  # The Alibaba Cloud OSS bucket name.
+  # CLI flag: -runtime-config.oss.bucket
+  [bucket: <string> | default = ""]
+
+  # The Alibaba Cloud OSS access key ID.
+  # CLI flag: -runtime-config.oss.access-key-id
+  [access_key_id: <string> | default = ""]
+
+  # The Alibaba Cloud OSS access key secret.
+  # CLI flag: -runtime-config.oss.access-key-secret
+  [access_key_secret: <string> | default = ""]
+
 filesystem:
   # Local filesystem storage directory.
   # CLI flag: -runtime-config.filesystem.dir

docs/configuration/v1-guarantees.md

@@ -47,6 +47,7 @@ Currently experimental features are:
 - TLS configuration in Etcd client.
 - OpenStack Swift storage support.
 - Oracle Cloud Infrastructure (OCI) Object Storage support.
+- Alibaba Cloud OSS storage support.
 - Metric relabeling in the distributor.
 - Ingester: do not unregister from ring on shutdown (`-ingester.unregister-on-shutdown=false`)
 - Distributor:

pkg/storage/bucket/client.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/storage/bucket/filesystem"
 	"github.com/cortexproject/cortex/pkg/storage/bucket/gcs"
 	"github.com/cortexproject/cortex/pkg/storage/bucket/oci"
+	"github.com/cortexproject/cortex/pkg/storage/bucket/oss"
 	"github.com/cortexproject/cortex/pkg/storage/bucket/s3"
 	"github.com/cortexproject/cortex/pkg/storage/bucket/swift"
 )
@@ -38,12 +39,15 @@ const (
 	// OCI is the value for the Oracle Cloud Infrastructure storage backend.
 	OCI = "oci"
 
+	// OSS is the value for the Alibaba Cloud OSS storage backend.
+	OSS = "oss"
+
 	// Filesystem is the value for the filesystem storage backend.
 	Filesystem = "filesystem"
 )
 
 var (
-	SupportedBackends = []string{S3, GCS, Azure, Swift, OCI, Filesystem}
+	SupportedBackends = []string{S3, GCS, Azure, Swift, OCI, OSS, Filesystem}
 
 	ErrUnsupportedStorageBackend = errors.New("unsupported storage backend")
 
@@ -59,6 +63,7 @@ type Config struct {
 	Azure      azure.Config      `yaml:"azure"`
 	Swift      swift.Config      `yaml:"swift"`
 	OCI        oci.Config        `yaml:"oci"`
+	OSS        oss.Config        `yaml:"oss"`
 	Filesystem filesystem.Config `yaml:"filesystem"`
 
 	// Not used internally, meant to allow callers to wrap Buckets
@@ -90,6 +95,7 @@ func (cfg *Config) RegisterFlagsWithPrefixAndBackend(prefix string, f *flag.Flag
 	cfg.Azure.RegisterFlagsWithPrefix(prefix, f)
 	cfg.Swift.RegisterFlagsWithPrefix(prefix, f)
 	cfg.OCI.RegisterFlagsWithPrefix(prefix, f)
+	cfg.OSS.RegisterFlagsWithPrefix(prefix, f)
 	cfg.Filesystem.RegisterFlagsWithPrefix(prefix, f)
 
 	f.StringVar(&cfg.Backend, prefix+"backend", defaultBackend, fmt.Sprintf("Backend storage to use. Supported backends are: %s.", strings.Join(cfg.supportedBackends(), ", ")))
@@ -123,6 +129,8 @@ func NewClient(ctx context.Context, cfg Config, hedgedRoundTripper func(rt http.
 		client, err = swift.NewBucketClient(cfg.Swift, hedgedRoundTripper, name, logger)
 	case OCI:
 		client, err = oci.NewBucketClient(cfg.OCI, hedgedRoundTripper, name, logger)
+	case OSS:
+		client, err = oss.NewBucketClient(cfg.OSS, hedgedRoundTripper, name, logger)
 	case Filesystem:
 		client, err = filesystem.NewBucketClient(cfg.Filesystem)
 	default:

pkg/storage/bucket/oss/bucket_client.go

@@ -0,0 +1,26 @@
+package oss
+
+import (
+	"net/http"
+
+	"github.com/go-kit/log"
+	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/providers/oss"
+)
+
+// NewBucketClient creates a new Alibaba Cloud OSS bucket client.
+func NewBucketClient(cfg Config, hedgedRoundTripper func(rt http.RoundTripper) http.RoundTripper, name string, logger log.Logger) (objstore.Bucket, error) {
+	bucketConfig := oss.Config{
+		Endpoint:        cfg.Endpoint,
+		Bucket:          cfg.Bucket,
+		AccessKeyID:     cfg.AccessKeyID,
+		AccessKeySecret: cfg.AccessKeySecret.Value,
+	}
+
+	// Check the error explicitly so a nil *oss.Bucket is not returned as a non-nil objstore.Bucket.
+	bkt, err := oss.NewBucketWithConfig(logger, bucketConfig, name, hedgedRoundTripper)
+	if err != nil {
+		return nil, err
+	}
+	return bkt, nil
+}

pkg/storage/bucket/oss/config.go

@@ -0,0 +1,28 @@
+package oss
+
+import (
+	"flag"
+
+	"github.com/cortexproject/cortex/pkg/util/flagext"
+)
+
+// Config holds the config options for an Alibaba Cloud OSS backend
+type Config struct {
+	Endpoint        string         `yaml:"endpoint"`
+	Bucket          string         `yaml:"bucket"`
+	AccessKeyID     string         `yaml:"access_key_id"`
+	AccessKeySecret flagext.Secret `yaml:"access_key_secret"`
+}
+
+// RegisterFlags registers the flags for Alibaba Cloud OSS storage
+func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
+	cfg.RegisterFlagsWithPrefix("", f)
+}
+
+// RegisterFlagsWithPrefix registers the flags for Alibaba Cloud OSS storage with the provided prefix
+func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.Endpoint, prefix+"oss.endpoint", "", "The Alibaba Cloud OSS endpoint to connect to, e.g. oss-cn-hangzhou.aliyuncs.com.")
+	f.StringVar(&cfg.Bucket, prefix+"oss.bucket", "", "The Alibaba Cloud OSS bucket name.")
+	f.StringVar(&cfg.AccessKeyID, prefix+"oss.access-key-id", "", "The Alibaba Cloud OSS access key ID.")
+	f.Var(&cfg.AccessKeySecret, prefix+"oss.access-key-secret", "The Alibaba Cloud OSS access key secret.")
+}