Add compatibility for sort without ext labels

In thanos-io#6299 we changed the comparison algorithm in the store proxy to ignore store-specific external labels. This causes problems during long rollouts because old stores and new stores will sort data differently, leading to incorrectly deduplicated results. This commit adds a new flag to the store info message to indicate whether the store sorts data with or without external labels. The querier uses this flag to decide whether to resort the store response set before starting a merge. Signed-off-by: Filip Petkovski <[email protected]>
fpetkovski · Jun 16, 2023 · d71ea7f · d71ea7f
1 parent d430269
commit d71ea7f
Show file tree

Hide file tree

Showing 15 changed files with 199 additions and 96 deletions.
diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go
@@ -783,11 +783,12 @@ func runQuery(
 				if httpProbe.IsReady() {
 					mint, maxt := proxy.TimeRange()
 					return &infopb.StoreInfo{
-						MinTime:                      mint,
-						MaxTime:                      maxt,
-						SupportsSharding:             true,
-						SupportsWithoutReplicaLabels: true,
-						TsdbInfos:                    proxy.TSDBInfos(),
+						MinTime:                           mint,
+						MaxTime:                           maxt,
+						SupportsSharding:                  true,
+						SupportsWithoutReplicaLabels:      true,
+						SupportsSortWithoutExternalLabels: true,
+						TsdbInfos:                         proxy.TSDBInfos(),
 					}
 				}
 				return nil

diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go
@@ -339,11 +339,12 @@ func runReceive(
 				if httpProbe.IsReady() {
 					minTime, maxTime := proxy.TimeRange()
 					return &infopb.StoreInfo{
-						MinTime:                      minTime,
-						MaxTime:                      maxTime,
-						SupportsSharding:             true,
-						SupportsWithoutReplicaLabels: true,
-						TsdbInfos:                    proxy.TSDBInfos(),
+						MinTime:                           minTime,
+						MaxTime:                           maxTime,
+						SupportsSharding:                  true,
+						SupportsWithoutReplicaLabels:      true,
+						SupportsSortWithoutExternalLabels: true,
+						TsdbInfos:                         proxy.TSDBInfos(),
 					}
 				}
 				return nil

diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go
@@ -628,11 +628,12 @@ func runRule(
 				if httpProbe.IsReady() {
 					mint, maxt := tsdbStore.TimeRange()
 					return &infopb.StoreInfo{
-						MinTime:                      mint,
-						MaxTime:                      maxt,
-						SupportsSharding:             true,
-						SupportsWithoutReplicaLabels: true,
-						TsdbInfos:                    tsdbStore.TSDBInfos(),
+						MinTime:                           mint,
+						MaxTime:                           maxt,
+						SupportsSharding:                  true,
+						SupportsWithoutReplicaLabels:      true,
+						SupportsSortWithoutExternalLabels: true,
+						TsdbInfos:                         tsdbStore.TSDBInfos(),
 					}
 				}
 				return nil

diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go
@@ -268,11 +268,12 @@ func runSidecar(
 				if httpProbe.IsReady() {
 					mint, maxt := promStore.Timestamps()
 					return &infopb.StoreInfo{
-						MinTime:                      mint,
-						MaxTime:                      maxt,
-						SupportsSharding:             true,
-						SupportsWithoutReplicaLabels: true,
-						TsdbInfos:                    promStore.TSDBInfos(),
+						MinTime:                           mint,
+						MaxTime:                           maxt,
+						SupportsSharding:                  true,
+						SupportsWithoutReplicaLabels:      true,
+						SupportsSortWithoutExternalLabels: true,
+						TsdbInfos:                         promStore.TSDBInfos(),
 					}
 				}
 				return nil

diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go
@@ -456,11 +456,12 @@ func runStore(
 			if httpProbe.IsReady() {
 				mint, maxt := bs.TimeRange()
 				return &infopb.StoreInfo{
-					MinTime:                      mint,
-					MaxTime:                      maxt,
-					SupportsSharding:             true,
-					SupportsWithoutReplicaLabels: true,
-					TsdbInfos:                    bs.TSDBInfos(),
+					MinTime:                           mint,
+					MaxTime:                           maxt,
+					SupportsSharding:                  true,
+					SupportsWithoutReplicaLabels:      true,
+					SupportsSortWithoutExternalLabels: true,
+					TsdbInfos:                         bs.TSDBInfos(),
 				}
 			}
 			return nil

diff --git a/pkg/info/infopb/rpc.pb.go b/pkg/info/infopb/rpc.pb.go
diff --git a/pkg/info/infopb/rpc.proto b/pkg/info/infopb/rpc.proto
@@ -64,6 +64,10 @@ message StoreInfo {
 
     // TSDBInfos holds metadata for all TSDBs exposed by the store.
     repeated TSDBInfo tsdb_infos = 6 [(gogoproto.nullable) = false];
+
+    // supports_sort_without_external_labels indicates whether the store ignores external labels
+    // when comparing two label sets.
+    bool supports_sort_without_external_labels = 7;
 }
 
 // RulesInfo holds the metadata related to Rules API exposed by the component.

diff --git a/pkg/query/endpointset.go b/pkg/query/endpointset.go
@@ -843,6 +843,17 @@ func (er *endpointRef) SupportsWithoutReplicaLabels() bool {
 	return er.metadata.Store.SupportsWithoutReplicaLabels
 }
 
+func (er *endpointRef) SupportsSortWithoutExternalLabels() bool {
+	er.mtx.RLock()
+	defer er.mtx.RUnlock()
+
+	if er.metadata == nil || er.metadata.Store == nil {
+		return false
+	}
+
+	return er.metadata.Store.SupportsSortWithoutExternalLabels
+}
+
 func (er *endpointRef) String() string {
 	mint, maxt := er.TimeRange()
 	return fmt.Sprintf(

diff --git a/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go b/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go
@@ -215,6 +215,10 @@ func (s *storeRef) SupportsWithoutReplicaLabels() bool {
 	return false
 }
 
+func (s *storeRef) SupportsSortWithoutExternalLabels() bool {
+	return false
+}
+
 func (s *storeRef) String() string {
 	mint, maxt := s.TimeRange()
 	return fmt.Sprintf(

diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go
@@ -160,6 +160,10 @@ func (l *localClient) SupportsWithoutReplicaLabels() bool {
 	return true
 }
 
+func (l *localClient) SupportsSortWithoutExternalLabels() bool {
+	return true
+}
+
 type tenant struct {
 	readyS        *ReadyStorage
 	storeTSDB     *store.TSDBStore

diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go
@@ -64,6 +64,10 @@ type Client interface {
 	// and sorted response is supported by the underlying store.
 	SupportsWithoutReplicaLabels() bool
 
+	// SupportsSortWithoutExternalLabels returns true if the store sorts series
+	// with external labels excluded from the comparison function.
+	SupportsSortWithoutExternalLabels() bool
+
 	// String returns the string representation of the store client.
 	String() string