Revert "merge generalizer state and structurally relate aliases"

This reverts commit 917713b4db.

compiler/rustc_infer/src/infer/relate/generalize.rs

@@ -267,14 +267,12 @@ fn generalize<T: Into<Term<'tcx>> + Relate<TyCtxt<'tcx>>>(
         let mut generalizer = Generalizer {
             infcx: self,
             span,
+            structurally_relate_aliases,
             root_vid,
             for_universe,
             root_term: source_term.into(),
             ambient_variance,
-            state: match structurally_relate_aliases {
-                StructurallyRelateAliases::No => GeneralizerState::Default,
-                StructurallyRelateAliases::Yes => GeneralizerState::StructurallyRelateAliases,
-            },
+            state: GeneralizationState::Default,
             cache: Default::default(),
             normalize,
         };
@@ -324,13 +322,10 @@ fn visit_region(&mut self, r: ty::Region<'tcx>) {
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-enum GeneralizerState {
-    /// Treat aliases as potentially normalizable.
+enum GeneralizationState {
     Default,
-    IncompletelyRelateHigherRankedAlias,
-    /// Only one layer
-    ShallowStructurallyRelateAliases,
-    StructurallyRelateAliases,
+    InAlias,
+    InNormalizedAlias,
 }
 
 /// The "generalizer" is used when handling inference variables.
@@ -351,6 +346,10 @@ struct Generalizer<'me, 'tcx> {
 
     span: Span,
 
+    /// Whether aliases should be related structurally. If not, we have to
+    /// be careful when generalizing aliases.
+    structurally_relate_aliases: StructurallyRelateAliases,
+
     /// The vid of the type variable that is in the process of being
     /// instantiated. If we find this within the value we are folding,
     /// that means we would have created a cyclic value.
@@ -373,9 +372,10 @@ struct Generalizer<'me, 'tcx> {
     /// This is necessary to correctly handle
     /// `<T as Bar<<?0 as Foo>::Assoc>::Assoc == ?0`. This equality can
     /// hold by either normalizing the outer or the inner associated type.
-    state: GeneralizerState,
+    // TODO: update doc comment
+    state: GeneralizationState,
 
-    cache: SsoHashMap<(Ty<'tcx>, ty::Variance, GeneralizerState), Ty<'tcx>>,
+    cache: SsoHashMap<(Ty<'tcx>, ty::Variance, GeneralizationState), Ty<'tcx>>,
 
     /// Normalize an alias in the trait solver.
     /// If normalization fails, a fresh infer var is returned.
@@ -415,51 +415,44 @@ fn next_ty_var_for_alias(&self) -> Ty<'tcx> {
     ///   continue generalizing the alias. This ends up pulling down the universe of the
     ///   inference variable and is incomplete in case the alias would normalize to a type
     ///   which does not mention that inference variable.
-    fn handle_alias_ty(
+    fn generalize_alias_ty(
         &mut self,
-        alias_ty: Ty<'tcx>,
         alias: ty::AliasTy<'tcx>,
     ) -> Result<Ty<'tcx>, TypeError<'tcx>> {
-        match self.state {
-            GeneralizerState::ShallowStructurallyRelateAliases => {
-                // We can switch back to default, we've treated one layer as rigid by doing this operation.
-                self.state = GeneralizerState::Default;
-                let res = relate::structurally_relate_tys(self, alias_ty, alias_ty);
-                self.state = GeneralizerState::ShallowStructurallyRelateAliases;
-                return res;
+        // We do not eagerly replace aliases with inference variables if they have
+        // escaping bound vars, see the method comment for details. However, when we
+        // are inside of an alias with escaping bound vars replacing nested aliases
+        // with inference variables can cause incorrect ambiguity.
+        //
+        // cc trait-system-refactor-initiative#110
+        if self.infcx.next_trait_solver()
+            && !alias.has_escaping_bound_vars()
+            && match self.state {
+                GeneralizationState::Default => true,
+                GeneralizationState::InAlias => false,
+                // When generalizing an alias after normalizing,
+                // the outer alias should be treated as rigid and we shouldn't try generalizing it again.
+                // If we recursively find more aliases, the state should have been set back to InAlias.
+                GeneralizationState::InNormalizedAlias => unreachable!(),
             }
-            GeneralizerState::StructurallyRelateAliases => {
-                return relate::structurally_relate_tys(self, alias_ty, alias_ty);
-            }
-            GeneralizerState::Default
-                if self.infcx.next_trait_solver() && !alias.has_escaping_bound_vars() =>
-            {
-                // We do not eagerly replace aliases with inference variables if they have
-                // escaping bound vars, see the method comment for details. However, when we
-                // are inside of an alias with escaping bound vars replacing nested aliases
-                // with inference variables can cause incorrect ambiguity.
-                //
-                // cc trait-system-refactor-initiative#110
-                let normalized_alias = (self.normalize)(alias);
+        {
+            let normalized_alias = (self.normalize)(alias);
 
-                self.state = GeneralizerState::ShallowStructurallyRelateAliases;
-                // recursively generalize, treat the outer alias as rigid to avoid infinite recursion
-                let res = self.relate(normalized_alias, normalized_alias);
+            self.state = GeneralizationState::InNormalizedAlias;
+            // recursively generalize, treat the outer alias as rigid to avoid infinite recursion
+            let res = self.relate(normalized_alias, normalized_alias);
 
-                // only one way to get here
-                self.state = GeneralizerState::Default;
+            // only one way to get here
+            self.state = GeneralizationState::Default;
 
-                return res;
-            }
-            GeneralizerState::Default | GeneralizerState::IncompletelyRelateHigherRankedAlias => {}
+            return res;
         }
 
-        let previous_state =
-            mem::replace(&mut self.state, GeneralizerState::IncompletelyRelateHigherRankedAlias);
+        let previous_state = mem::replace(&mut self.state, GeneralizationState::InAlias);
         let result = match self.relate(alias, alias) {
             Ok(alias) => Ok(alias.to_ty(self.cx())),
             Err(e) => match previous_state {
-                GeneralizerState::Default => {
+                GeneralizationState::Default => {
                     let mut visitor = MaxUniverse::new();
                     alias.visit_with(&mut visitor);
                     let infer_replacement_is_complete =
@@ -472,11 +465,11 @@ fn handle_alias_ty(
                     debug!("generalization failure in alias");
                     Ok(self.next_ty_var_for_alias())
                 }
-                GeneralizerState::IncompletelyRelateHigherRankedAlias => return Err(e),
-
-                // Early return.
-                GeneralizerState::ShallowStructurallyRelateAliases
-                | GeneralizerState::StructurallyRelateAliases => unreachable!(),
+                GeneralizationState::InAlias => return Err(e),
+                // When generalizing an alias after normalizing,
+                // the outer alias should be treated as rigid and we shouldn't try generalizing it again.
+                // If we recursively find more aliases, the state should have been set back to InAlias.
+                GeneralizationState::InNormalizedAlias => unreachable!(),
             },
         };
         self.state = previous_state;
@@ -600,9 +593,11 @@ fn tys(&mut self, t: Ty<'tcx>, t2: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> {
                                 && !matches!(self.infcx.typing_mode(), TypingMode::Coherence)
                             {
                                 match self.state {
-                                    GeneralizerState::IncompletelyRelateHigherRankedAlias => {
+                                    GeneralizationState::InAlias => {
                                         inner.type_variables().equate(vid, new_var_id);
                                     }
+                                    GeneralizationState::Default
+                                    | GeneralizationState::InNormalizedAlias => {}
                                 }
                                 GeneralizerState::Default
                                 | GeneralizerState::ShallowStructurallyRelateAliases
@@ -635,7 +630,21 @@ fn tys(&mut self, t: Ty<'tcx>, t2: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> {
                 }
             }
 
-            ty::Alias(_, data) => self.handle_alias_ty(t, data),
+            ty::Alias(_, data) => match self.structurally_relate_aliases {
+                StructurallyRelateAliases::No => match self.state {
+                    GeneralizationState::Default | GeneralizationState::InAlias => {
+                        self.generalize_alias_ty(data)
+                    }
+                    GeneralizationState::InNormalizedAlias => {
+                        // We can switch back to default, we've treated one layer as rigid by doing this operation.
+                        self.state = GeneralizationState::Default;
+                        let res = relate::structurally_relate_tys(self, t, t);
+                        self.state = GeneralizationState::InNormalizedAlias;
+                        res
+                    }
+                },
+                StructurallyRelateAliases::Yes => relate::structurally_relate_tys(self, t, t),
+            },
 
             _ => relate::structurally_relate_tys(self, t, t),
         }?;
@@ -733,9 +742,11 @@ fn consts(
                                 && !matches!(self.infcx.typing_mode(), TypingMode::Coherence)
                             {
                                 match self.state {
-                                    GeneralizerState::IncompletelyRelateHigherRankedAlias => {
+                                    GeneralizationState::InAlias => {
                                         variable_table.union(vid, new_var_id);
                                     }
+                                    GeneralizationState::Default
+                                    | GeneralizationState::InNormalizedAlias => {}
                                 }
                                 GeneralizerState::Default
                                 | GeneralizerState::ShallowStructurallyRelateAliases