diff --git a/src/hotspot/share/opto/castnode.cpp b/src/hotspot/share/opto/castnode.cpp index 6b3058d94f6ff..82012edf7a759 100644 --- a/src/hotspot/share/opto/castnode.cpp +++ b/src/hotspot/share/opto/castnode.cpp @@ -33,10 +33,16 @@ #include "castnode.hpp" #include "utilities/checkedCast.hpp" +const ConstraintCastNode::DependencyType ConstraintCastNode::RegularDependency(true, true, true, "regular dependency"); // not pinned, narrows type +const ConstraintCastNode::DependencyType ConstraintCastNode::WidenTypeDependency(true, false, false, "widen type dependency"); // not pinned, doesn't narrow type +const ConstraintCastNode::DependencyType ConstraintCastNode::StrongDependency(false, true, true, "strong dependency"); // pinned, narrows type +const ConstraintCastNode::DependencyType ConstraintCastNode::UnconditionalDependency(false, false, true, "unconditional dependency"); // pinned, doesn't narrow type +const ConstraintCastNode::DependencyType ConstraintCastNode::PinnedWidenTypeDependency(false, false, false, "widen type dependency"); // not pinned, doesn't narrow type + //============================================================================= // If input is already higher or equal to cast type, then this is an identity. Node* ConstraintCastNode::Identity(PhaseGVN* phase) { - if (_dependency == UnconditionalDependency) { + if (!_dependency.narrows_type()) { return this; } Node* dom = dominating_cast(phase, phase); @@ -51,6 +57,10 @@ Node* ConstraintCastNode::Identity(PhaseGVN* phase) { const Type* ConstraintCastNode::Value(PhaseGVN* phase) const { if (in(0) && phase->type(in(0)) == Type::TOP) return Type::TOP; + if (!_dependency.constant_folds()) { + return _type; + } + const Type* in_type = phase->type(in(1)); const Type* ft = in_type->filter_speculative(_type); @@ -107,7 +117,7 @@ Node* ConstraintCastNode::Ideal(PhaseGVN* phase, bool can_reshape) { } uint ConstraintCastNode::hash() const { - return TypeNode::hash() + (int)_dependency + (_extra_types != nullptr ? _extra_types->hash() : 0); + return TypeNode::hash() + _dependency.hash() + (_extra_types != nullptr ? _extra_types->hash() : 0); } bool ConstraintCastNode::cmp(const Node &n) const { @@ -115,7 +125,7 @@ bool ConstraintCastNode::cmp(const Node &n) const { return false; } ConstraintCastNode& cast = (ConstraintCastNode&) n; - if (cast._dependency != _dependency) { + if (!cast._dependency.cmp(_dependency)) { return false; } if (_extra_types == nullptr || cast._extra_types == nullptr) { @@ -128,7 +138,7 @@ uint ConstraintCastNode::size_of() const { return sizeof(*this); } -Node* ConstraintCastNode::make_cast_for_basic_type(Node* c, Node* n, const Type* t, DependencyType dependency, BasicType bt) { +Node* ConstraintCastNode::make_cast_for_basic_type(Node* c, Node* n, const Type* t, const DependencyType& dependency, BasicType bt) { switch(bt) { case T_INT: return new CastIINode(c, n, t, dependency); @@ -141,7 +151,7 @@ Node* ConstraintCastNode::make_cast_for_basic_type(Node* c, Node* n, const Type* } TypeNode* ConstraintCastNode::dominating_cast(PhaseGVN* gvn, PhaseTransform* pt) const { - if (_dependency == UnconditionalDependency) { + if (!_dependency.narrows_type()) { return nullptr; } Node* val = in(1); @@ -203,30 +213,21 @@ void ConstraintCastNode::dump_spec(outputStream *st) const { st->print(" extra types: "); _extra_types->dump_on(st); } - if (_dependency != RegularDependency) { - st->print(" %s dependency", _dependency == StrongDependency ? "strong" : "unconditional"); - } + st->print(" "); + _dependency.dump_on(st); } #endif -const Type* CastIINode::Value(PhaseGVN* phase) const { - const Type *res = ConstraintCastNode::Value(phase); - if (res == Type::TOP) { - return Type::TOP; - } - assert(res->isa_int(), "res must be int"); - - // Similar to ConvI2LNode::Value() for the same reasons - // see if we can remove type assertion after loop opts - res = widen_type(phase, res, T_INT); +CastIINode* CastIINode::make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const { + return new CastIINode(in(0), parent, type, dependency, _range_check_dependency, _extra_types); +} - return res; +CastLLNode* CastLLNode::make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const { + return new CastLLNode(in(0), parent, type, dependency, _extra_types); } -Node* ConstraintCastNode::find_or_make_integer_cast(PhaseIterGVN* igvn, Node* parent, const TypeInteger* type) const { - Node* n = clone(); - n->set_req(1, parent); - n->as_ConstraintCast()->set_type(type); +Node* ConstraintCastNode::find_or_make_integer_cast(PhaseIterGVN* igvn, Node* parent, const TypeInteger* type, const DependencyType& dependency) const { + Node* n = make_with(parent, type, dependency); Node* existing = igvn->hash_find_insert(n); if (existing != nullptr) { n->destruct(igvn); @@ -240,14 +241,13 @@ Node *CastIINode::Ideal(PhaseGVN *phase, bool can_reshape) { if (progress != nullptr) { return progress; } - if (can_reshape && !phase->C->post_loop_opts_phase()) { - // makes sure we run ::Value to potentially remove type assertion after loop opts - phase->C->record_for_post_loop_opts_igvn(this); + if (!phase->C->widen_types_phase()) { + // makes sure we run widen_type() to potentially common type assertions after loop opts + phase->C->record_for_widen_types_igvn(this); } - if (!_range_check_dependency || phase->C->post_loop_opts_phase()) { + if (!_range_check_dependency || phase->C->widen_types_phase()) { return optimize_integer_cast(phase, T_INT); } - phase->C->record_for_post_loop_opts_igvn(this); return nullptr; } @@ -277,9 +277,9 @@ void CastIINode::dump_spec(outputStream* st) const { #endif CastIINode* CastIINode::pin_array_access_node() const { - assert(_dependency == RegularDependency, "already pinned"); + assert(depends_only_on_test(), "already pinned"); if (has_range_check()) { - return new CastIINode(in(0), in(1), bottom_type(), StrongDependency, has_range_check()); + return new CastIINode(in(0), in(1), bottom_type(), _dependency.pinned_dependency(), has_range_check()); } return nullptr; } @@ -313,24 +313,14 @@ void CastIINode::remove_range_check_cast(Compile* C) { } -const Type* CastLLNode::Value(PhaseGVN* phase) const { - const Type* res = ConstraintCastNode::Value(phase); - if (res == Type::TOP) { - return Type::TOP; - } - assert(res->isa_long(), "res must be long"); - - return widen_type(phase, res, T_LONG); -} - Node* CastLLNode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* progress = ConstraintCastNode::Ideal(phase, can_reshape); if (progress != nullptr) { return progress; } - if (!phase->C->post_loop_opts_phase()) { - // makes sure we run ::Value to potentially remove type assertion after loop opts - phase->C->record_for_post_loop_opts_igvn(this); + if (!phase->C->widen_types_phase()) { + // makes sure we run widen_type() to potentially common type assertions after loop opts + phase->C->record_for_widen_types_igvn(this); } // transform (CastLL (ConvI2L ..)) into (ConvI2L (CastII ..)) if the type of the CastLL is narrower than the type of // the ConvI2L. @@ -341,11 +331,10 @@ Node* CastLLNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (t != Type::TOP && t_in != Type::TOP) { const TypeLong* tl = t->is_long(); const TypeLong* t_in_l = t_in->is_long(); - assert(tl->_lo >= t_in_l->_lo && tl->_hi <= t_in_l->_hi, "CastLL type should be narrower than or equal to the type of its input"); - assert((tl != t_in_l) == (tl->_lo > t_in_l->_lo || tl->_hi < t_in_l->_hi), "if type differs then this nodes's type must be narrower"); - if (tl != t_in_l) { + if (tl->_lo > t_in_l->_lo || tl->_hi < t_in_l->_hi) { const TypeInt* ti = TypeInt::make(checked_cast(tl->_lo), checked_cast(tl->_hi), tl->_widen); - Node* castii = phase->transform(new CastIINode(in(0), in1->in(1), ti)); + assert(_extra_types == nullptr, ""); + Node* castii = phase->transform(new CastIINode(in(0), in1->in(1), ti, _dependency)); Node* convi2l = in1->clone(); convi2l->set_req(1, castii); return convi2l; @@ -475,7 +464,7 @@ Node* CastP2XNode::Identity(PhaseGVN* phase) { return this; } -Node* ConstraintCastNode::make_cast_for_type(Node* c, Node* in, const Type* type, DependencyType dependency, +Node* ConstraintCastNode::make_cast_for_type(Node* c, Node* in, const Type* type, const DependencyType& dependency, const TypeTuple* types) { if (type->isa_int()) { return new CastIINode(c, in, type, dependency, false, types); @@ -496,7 +485,52 @@ Node* ConstraintCastNode::make_cast_for_type(Node* c, Node* in, const Type* type return nullptr; } -Node* ConstraintCastNode::optimize_integer_cast(PhaseGVN* phase, BasicType bt) { +bool ConstraintCastNode::follow_uses_until_pinned_accesses(PhaseIterGVN* igvn) { + ResourceMark rm; + Unique_Node_List wq; + wq.push(this); + for (uint i = 0; i < wq.size(); ++i) { + Node* n = wq.at(i); + for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { + Node* u = n->fast_out(j); + if (u->in(0) == nullptr) { + int opcode = u->Opcode(); + // A node that can fault + if (u->is_Load() || + opcode == Op_DivI || + opcode == Op_DivL || + opcode == Op_ModI || + opcode == Op_ModL || + opcode == Op_UModI || + opcode == Op_UModL) { + return false; + } + wq.push(u); + } else if (u->is_CallStaticJava() && u->as_CallStaticJava()->is_uncommon_trap()) { + // ignore uncommon traps + } else { + if (!igvn->is_dominator(in(0), u->in(0))) { + return false; + } + if (!u->is_Mem()) { + return false; + } + if (!wq.member(u->in(MemNode::Address))) { + return false; + } + if (u->depends_only_on_test()) { + return false; + } + } + } + if (wq.size() > 10) { + return false; + } + } + return true; +} + +Node* ConstraintCastNode::optimize_integer_cast_of_add(PhaseGVN* phase, BasicType bt) { PhaseIterGVN *igvn = phase->is_IterGVN(); const TypeInteger* this_type = this->type()->is_integer(bt); Node* z = in(1); @@ -514,24 +548,62 @@ Node* ConstraintCastNode::optimize_integer_cast(PhaseGVN* phase, BasicType bt) { Node* x = z->in(1); Node* y = z->in(2); - Node* cx = find_or_make_integer_cast(igvn, x, rx); - Node* cy = find_or_make_integer_cast(igvn, y, ry); + const TypeInteger* tx = phase->type(x)->is_integer(bt); + const TypeInteger* ty = phase->type(y)->is_integer(bt); + + bool only_used_by_pinned_access = false; + if (!tx->is_con() && !ty->is_con() && !igvn->C->widen_types_phase()) { + only_used_by_pinned_access = follow_uses_until_pinned_accesses(igvn); + if (!only_used_by_pinned_access) { + phase->C->record_for_widen_types_igvn(this); + return nullptr; + } + } + + // If both inputs are not constant then, with the Cast pushed through the Add/Sub, the cast gets less precised types, + // and the resulting Add/Sub's type is wider than that of the Cast before pushing. + const DependencyType& dependency = (!tx->is_con() && !ty->is_con() && !only_used_by_pinned_access) ? _dependency.widen_type_dependency() : _dependency; + Node* cx = find_or_make_integer_cast(igvn, x, rx, dependency); + Node* cy = find_or_make_integer_cast(igvn, y, ry, dependency); if (op == Op_Add(bt)) { - return AddNode::make(cx, cy, bt); + Node* res = AddNode::make(cx, cy, bt); + assert(res->Value(phase) == Value(phase) || !tx->is_con() || !ty->is_con() || igvn->_worklist.member(z), "type widening"); + return res; } else { assert(op == Op_Sub(bt), ""); - return SubNode::make(cx, cy, bt); + Node* res = SubNode::make(cx, cy, bt); + assert(res->Value(phase) == Value(phase) || !tx->is_con() || !ty->is_con() || igvn->_worklist.member(z), "type widening"); + return res; } return nullptr; } return nullptr; } -const Type* ConstraintCastNode::widen_type(const PhaseGVN* phase, const Type* res, BasicType bt) const { - if (!phase->C->post_loop_opts_phase()) { +Node* ConstraintCastNode::optimize_integer_cast(PhaseGVN* phase, BasicType bt) { + Node* res = optimize_integer_cast_of_add(phase, bt); + if (res != nullptr) { return res; } + const Type* t = Value(phase); + if (t != Type::TOP) { + const TypeInteger* this_type = t->is_integer(bt); + const TypeInteger* wide_t = widen_type(phase, this_type, bt); + if (wide_t->lo_as_long() < this_type->lo_as_long() || wide_t->hi_as_long() > this_type->hi_as_long()) { + // Widening the type of the Cast (to allow some commoning) causes the Cast to change how it can be optimized (if + // type of its input is narrower than the Cast's type, we can't remove it to not loose the dependency). + return make_with(in(1), wide_t, _dependency.widen_type_dependency()); + } + } + return nullptr; +} + +const TypeInteger* ConstraintCastNode::widen_type(const PhaseGVN* phase, const TypeInteger* this_type, BasicType bt) const { + if (!phase->C->widen_types_phase()) { + return this_type; + } + // At VerifyConstraintCasts == 1, we verify the ConstraintCastNodes that are present during code // emission. This allows us detecting possible mis-scheduling due to these nodes being pinned at // the wrong control nodes. @@ -540,10 +612,9 @@ const Type* ConstraintCastNode::widen_type(const PhaseGVN* phase, const Type* re // mis-transformations that may happen due to these nodes being pinned at the wrong control // nodes. if (VerifyConstraintCasts > 1) { - return res; + return this_type; } - const TypeInteger* this_type = res->is_integer(bt); const TypeInteger* in_type = phase->type(in(1))->isa_integer(bt); if (in_type != nullptr && (in_type->lo_as_long() != this_type->lo_as_long() || @@ -564,5 +635,5 @@ const Type* ConstraintCastNode::widen_type(const PhaseGVN* phase, const Type* re MIN2(in_type->hi_as_long(), hi1), MAX2((int)in_type->_widen, w1), bt); } - return res; + return this_type; } diff --git a/src/hotspot/share/opto/castnode.hpp b/src/hotspot/share/opto/castnode.hpp index 1b848e5efdf24..143e8642350d4 100644 --- a/src/hotspot/share/opto/castnode.hpp +++ b/src/hotspot/share/opto/castnode.hpp @@ -32,22 +32,97 @@ //------------------------------ConstraintCastNode----------------------------- // cast to a different range class ConstraintCastNode: public TypeNode { -public: - enum DependencyType { - RegularDependency, // if cast doesn't improve input type, cast can be removed - StrongDependency, // leave cast in even if _type doesn't improve input type, can be replaced by stricter dominating cast if one exist - UnconditionalDependency // leave cast in unconditionally +protected: + // Cast nodes are subject to a few optimizations: + // + // 1- if the type carried by the Cast doesn't narrow the type of its input, the cast can be replaced by its input. + // Similarly, if a dominating Cast with the same input and a narrower type constraint is found, it can replace the + // current cast. + // + // 2- if the condition that the Cast is control dependent is hoisted, the Cast is hoisted as well + // + // 1- and 2- are not always applied depending on what constraint are applied to the Cast: there are cases where 1- + // and 2- apply, where neither 1- nor 2- apply and where one or the other apply. This class abstract away these + // details. + class DependencyType { + public: + DependencyType(bool depends_on_test, bool narrows_type, bool constant_folds, const char* desc) + : _depends_only_on_test(depends_on_test), + _narrows_type(narrows_type), + _constant_folds(constant_folds), + _desc(desc) { + } + NONCOPYABLE(DependencyType); + + bool depends_only_on_test() const { + return _depends_only_on_test; + } + + bool narrows_type() const { + return _narrows_type; + } + + bool constant_folds() const { + return _constant_folds; + } + + void dump_on(outputStream *st) const { + st->print("%s", _desc); + } + + uint hash() const { + return (_depends_only_on_test ? 1 : 0) + (_narrows_type ? 2 : 0) + (_constant_folds ? 4 : 0); + } + + bool cmp(const DependencyType& other) const { + return _depends_only_on_test == other._depends_only_on_test && _narrows_type == other._narrows_type && + _constant_folds == other._constant_folds; + } + + const DependencyType& widen_type_dependency() const { + if (_depends_only_on_test) { + return WidenTypeDependency; + } + return PinnedWidenTypeDependency; + } + + const DependencyType& pinned_dependency() const { + assert(_constant_folds, ""); + if (_narrows_type) { + return StrongDependency; + } + return UnconditionalDependency; + } + + private: + const bool _depends_only_on_test; // Does this Cast depends on its control input or is it pinned? + const bool _narrows_type; // Does this Cast narrows the type i.e. if input type is narrower can it be removed? + const bool _constant_folds; + const char* _desc; }; +public: + + static const DependencyType RegularDependency; + static const DependencyType WidenTypeDependency; + static const DependencyType PinnedWidenTypeDependency; + static const DependencyType StrongDependency; + static const DependencyType UnconditionalDependency; + protected: - const DependencyType _dependency; + const DependencyType& _dependency; virtual bool cmp( const Node &n ) const; virtual uint size_of() const; virtual uint hash() const; // Check the type - const Type* widen_type(const PhaseGVN* phase, const Type* res, BasicType bt) const; - Node* find_or_make_integer_cast(PhaseIterGVN* igvn, Node* parent, const TypeInteger* type) const; + const TypeInteger* widen_type(const PhaseGVN* phase, const TypeInteger* this_type, BasicType bt) const; + + virtual ConstraintCastNode* make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const { + ShouldNotReachHere(); + return nullptr; + } + + Node* find_or_make_integer_cast(PhaseIterGVN* igvn, Node* parent, const TypeInteger* type, const DependencyType& dependency) const; - private: // PhiNode::Ideal() transforms a Phi that merges a single uncasted value into a single cast pinned at the region. // The types of cast nodes eliminated as a consequence of this transformation are collected and stored here so the // type dependencies carried by the cast are known. The cast can then be eliminated if the type of its input is @@ -55,7 +130,7 @@ class ConstraintCastNode: public TypeNode { const TypeTuple* _extra_types; public: - ConstraintCastNode(Node* ctrl, Node* n, const Type* t, ConstraintCastNode::DependencyType dependency, + ConstraintCastNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency, const TypeTuple* extra_types) : TypeNode(t,2), _dependency(dependency), _extra_types(extra_types) { init_class_id(Class_ConstraintCast); @@ -67,18 +142,22 @@ class ConstraintCastNode: public TypeNode { virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); virtual int Opcode() const; virtual uint ideal_reg() const = 0; - virtual bool depends_only_on_test() const { return _dependency == RegularDependency; } - bool carry_dependency() const { return _dependency != RegularDependency; } + bool carry_dependency() const { return !_dependency.cmp(RegularDependency); } + virtual bool depends_only_on_test() const { return _dependency.depends_only_on_test(); } + const DependencyType& dependency() const { return _dependency; } TypeNode* dominating_cast(PhaseGVN* gvn, PhaseTransform* pt) const; - static Node* make_cast_for_basic_type(Node* c, Node* n, const Type* t, DependencyType dependency, BasicType bt); + static Node* make_cast_for_basic_type(Node* c, Node* n, const Type* t, const DependencyType& dependency, BasicType bt); #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; #endif - static Node* make_cast_for_type(Node* c, Node* in, const Type* type, DependencyType dependency, + static Node* make_cast_for_type(Node* c, Node* in, const Type* type, const DependencyType& dependency, const TypeTuple* types); + bool follow_uses_until_pinned_accesses(PhaseIterGVN* igvn); + + Node* optimize_integer_cast_of_add(PhaseGVN* phase, BasicType bt); Node* optimize_integer_cast(PhaseGVN* phase, BasicType bt); bool higher_equal_types(PhaseGVN* phase, const Node* other) const; @@ -102,7 +181,7 @@ class CastIINode: public ConstraintCastNode { virtual uint size_of() const; public: - CastIINode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, bool range_check_dependency = false, const TypeTuple* types = nullptr) + CastIINode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = RegularDependency, bool range_check_dependency = false, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types), _range_check_dependency(range_check_dependency) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastII); @@ -110,7 +189,6 @@ class CastIINode: public ConstraintCastNode { virtual int Opcode() const; virtual uint ideal_reg() const { return Op_RegI; } virtual Node* Identity(PhaseGVN* phase); - virtual const Type* Value(PhaseGVN* phase) const; virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); bool has_range_check() const { #ifdef _LP64 @@ -122,6 +200,7 @@ class CastIINode: public ConstraintCastNode { } CastIINode* pin_array_access_node() const; + CastIINode* make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const; void remove_range_check_cast(Compile* C); #ifndef PRODUCT @@ -131,21 +210,21 @@ class CastIINode: public ConstraintCastNode { class CastLLNode: public ConstraintCastNode { public: - CastLLNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastLLNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = RegularDependency, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastLL); } - virtual const Type* Value(PhaseGVN* phase) const; virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); virtual int Opcode() const; virtual uint ideal_reg() const { return Op_RegL; } + CastLLNode* make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const; }; class CastHHNode: public ConstraintCastNode { public: - CastHHNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastHHNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = RegularDependency, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastHH); @@ -156,7 +235,7 @@ class CastHHNode: public ConstraintCastNode { class CastFFNode: public ConstraintCastNode { public: - CastFFNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastFFNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = RegularDependency, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastFF); @@ -167,7 +246,7 @@ class CastFFNode: public ConstraintCastNode { class CastDDNode: public ConstraintCastNode { public: - CastDDNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastDDNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = RegularDependency, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastDD); @@ -178,7 +257,7 @@ class CastDDNode: public ConstraintCastNode { class CastVVNode: public ConstraintCastNode { public: - CastVVNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastVVNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = RegularDependency, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastVV); @@ -192,7 +271,7 @@ class CastVVNode: public ConstraintCastNode { // cast pointer to pointer (different type) class CastPPNode: public ConstraintCastNode { public: - CastPPNode (Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastPPNode (Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = RegularDependency, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { init_class_id(Class_CastPP); } @@ -204,7 +283,7 @@ class CastPPNode: public ConstraintCastNode { // for _checkcast, cast pointer to pointer (different type), without JOIN, class CheckCastPPNode: public ConstraintCastNode { public: - CheckCastPPNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CheckCastPPNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = RegularDependency, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CheckCastPP); diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 10846a326262a..6dad053ef17b6 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -406,6 +406,9 @@ void Compile::remove_useless_node(Node* dead) { if (dead->for_merge_stores_igvn()) { remove_from_merge_stores_igvn(dead); } + if (dead->for_widen_types_igvn()) { + remove_from_widen_types_igvn(dead); + } if (dead->is_Call()) { remove_useless_late_inlines( &_late_inlines, dead); remove_useless_late_inlines( &_string_late_inlines, dead); @@ -459,6 +462,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass + remove_useless_nodes(_for_widen_types_igvn, useful); remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes #ifdef ASSERT @@ -634,6 +638,7 @@ Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci, _max_node_limit(MaxNodeLimit), _post_loop_opts_phase(false), _merge_stores_phase(false), + _widen_type_phase(false), _allow_macro_nodes(true), _inlining_progress(false), _inlining_incrementally(false), @@ -659,6 +664,7 @@ Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci, _expensive_nodes(comp_arena(), 8, 0, nullptr), _for_post_loop_igvn(comp_arena(), 8, 0, nullptr), _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr), + _for_widen_types_igvn(comp_arena(), 8, 0, nullptr), _unstable_if_traps(comp_arena(), 8, 0, nullptr), _coarsened_locks(comp_arena(), 8, 0, nullptr), _congraph(nullptr), @@ -914,6 +920,7 @@ Compile::Compile(ciEnv* ci_env, _max_node_limit(MaxNodeLimit), _post_loop_opts_phase(false), _merge_stores_phase(false), + _widen_type_phase(false), _allow_macro_nodes(true), _inlining_progress(false), _inlining_incrementally(false), @@ -933,6 +940,7 @@ Compile::Compile(ciEnv* ci_env, _first_failure_details(nullptr), _for_post_loop_igvn(comp_arena(), 8, 0, nullptr), _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr), + _for_widen_types_igvn(comp_arena(), 8, 0, nullptr), _congraph(nullptr), NOT_PRODUCT(_igv_printer(nullptr) COMMA) _unique(0), @@ -1923,6 +1931,35 @@ void Compile::process_for_merge_stores_igvn(PhaseIterGVN& igvn) { } } +void Compile::record_for_widen_types_igvn(Node* n) { + if (!n->for_widen_types_igvn()) { + assert(!_for_widen_types_igvn.contains(n), "duplicate"); + n->add_flag(Node::NodeFlags::Flag_for_widen_types_igvn); + _for_widen_types_igvn.append(n); + } +} + +void Compile::remove_from_widen_types_igvn(Node* n) { + n->remove_flag(Node::NodeFlags::Flag_for_widen_types_igvn); + _for_widen_types_igvn.remove(n); +} + +void Compile::process_for_widen_types_igvn(PhaseIterGVN& igvn) { + C->set_widen_types_phase(); // no more loop opts allowed + + if (_for_widen_types_igvn.length() > 0) { + while (_for_widen_types_igvn.length() > 0) { + Node* n = _for_widen_types_igvn.pop(); + n->remove_flag(Node::NodeFlags::Flag_for_widen_types_igvn); + igvn._worklist.push(n); + } + igvn.optimize(); + if (failing()) return; + assert(_for_widen_types_igvn.length() == 0, "no more delayed nodes allowed"); + print_method(PHASE_AFTER_WIDEN_TYPES, 3); + } +} + void Compile::record_unstable_if_trap(UnstableIfTrap* trap) { if (OptimizeUnstableIf) { _unstable_if_traps.append(trap); @@ -2531,6 +2568,10 @@ void Compile::Optimize() { if (failing()) return; } + process_for_widen_types_igvn(igvn); + + if (failing()) return; + DEBUG_ONLY( _modified_nodes = nullptr; ) assert(igvn._worklist.size() == 0, "not empty"); @@ -3506,6 +3547,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f break; } case Op_CastII: { + assert(!UseNewCode || !n->in(1)->is_Con(), ""); n->as_CastII()->remove_range_check_cast(this); break; } diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 6d43948964e90..e5cceb5ae1b46 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -319,6 +319,7 @@ class Compile : public Phase { bool _post_loop_opts_phase; // Loop opts are finished. bool _merge_stores_phase; // Phase for merging stores, after post loop opts phase. + bool _widen_type_phase; bool _allow_macro_nodes; // True if we allow creation of macro nodes. int _major_progress; // Count of something big happening @@ -377,6 +378,7 @@ class Compile : public Phase { GrowableArray _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common GrowableArray _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over GrowableArray _for_merge_stores_igvn; // List of nodes for IGVN merge stores + GrowableArray _for_widen_types_igvn; GrowableArray _unstable_if_traps; // List of ifnodes after IGVN GrowableArray _coarsened_locks; // List of coarsened Lock and Unlock nodes ConnectionGraph* _congraph; @@ -778,6 +780,12 @@ class Compile : public Phase { void remove_from_merge_stores_igvn(Node* n); void process_for_merge_stores_igvn(PhaseIterGVN& igvn); + bool widen_types_phase() { return _widen_type_phase; } + void set_widen_types_phase() { _widen_type_phase = true; } + void record_for_widen_types_igvn(Node* n); + void remove_from_widen_types_igvn(Node* n); + void process_for_widen_types_igvn(PhaseIterGVN& igvn); + void shuffle_macro_nodes(); void sort_macro_nodes(); diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 3875c6ce32571..b76de3e043eab 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -513,6 +513,9 @@ Node *Node::clone() const { // If it is applicable, it will happen anyway when the cloned node is registered with IGVN. n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn); } + if (for_widen_types_igvn()) { + n->remove_flag(Node::NodeFlags::Flag_for_widen_types_igvn); + } if (n->is_ParsePredicate()) { C->add_parse_predicate(n->as_ParsePredicate()); } @@ -626,6 +629,9 @@ void Node::destruct(PhaseValues* phase) { if (for_merge_stores_igvn()) { compile->remove_from_merge_stores_igvn(this); } + if (for_widen_types_igvn()) { + compile->remove_from_widen_types_igvn(this); + } if (is_SafePoint()) { as_SafePoint()->delete_replaced_nodes(); @@ -1081,7 +1087,7 @@ void Node::init_NodeProperty() { //-----------------------------max_flags--------------------------------------- juint Node::max_flags() { - return (PD::_last_flag << 1) - 1; // allow flags combination + return (((juint)PD::_last_flag) << 1) - 1; // allow flags combination } #endif diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index 843baf48cf8ae..efe112d83e2de 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -837,7 +837,8 @@ class Node { Flag_for_merge_stores_igvn = 1 << 16, Flag_is_removed_by_peephole = 1 << 17, Flag_is_predicated_using_blend = 1 << 18, - _last_flag = Flag_is_predicated_using_blend + Flag_for_widen_types_igvn = 1 << 19, + _last_flag = Flag_for_widen_types_igvn }; class PD; @@ -1081,6 +1082,7 @@ class Node { bool for_post_loop_opts_igvn() const { return (_flags & Flag_for_post_loop_opts_igvn) != 0; } bool for_merge_stores_igvn() const { return (_flags & Flag_for_merge_stores_igvn) != 0; } + bool for_widen_types_igvn() const { return (_flags & Flag_for_widen_types_igvn) != 0; } // Is 'n' possibly a loop entry (i.e. a Parse Predicate projection)? static bool may_be_loop_entry(Node* n) { diff --git a/src/hotspot/share/opto/phasetype.hpp b/src/hotspot/share/opto/phasetype.hpp index b3076dd80911e..a567d55891adb 100644 --- a/src/hotspot/share/opto/phasetype.hpp +++ b/src/hotspot/share/opto/phasetype.hpp @@ -91,6 +91,7 @@ flags(PHASEIDEALLOOP_ITERATIONS, "PhaseIdealLoop iterations") \ flags(AFTER_LOOP_OPTS, "After Loop Optimizations") \ flags(AFTER_MERGE_STORES, "After Merge Stores") \ + flags(AFTER_WIDEN_TYPES, "After Widen Types") \ flags(BEFORE_MACRO_EXPANSION , "Before Macro Expansion") \ flags(AFTER_MACRO_EXPANSION_STEP, "After Macro Expansion Step") \ flags(AFTER_MACRO_EXPANSION, "After Macro Expansion") \ @@ -121,7 +122,7 @@ #define table_entry(name, description) PHASE_##name, enum CompilerPhaseType { - COMPILER_PHASES(table_entry) + COMPILER_PHASES(table_entry) PHASE_NUM_TYPES, PHASE_NONE }; diff --git a/test/hotspot/jtreg/compiler/c2/irTests/TestPushAddThruCast.java b/test/hotspot/jtreg/compiler/c2/irTests/TestPushAddThruCast.java index 7c5fa05f14788..abb9dcf8d8ad7 100644 --- a/test/hotspot/jtreg/compiler/c2/irTests/TestPushAddThruCast.java +++ b/test/hotspot/jtreg/compiler/c2/irTests/TestPushAddThruCast.java @@ -44,13 +44,13 @@ public static void main(String[] args) { TestFramework.run(); } - final static int length = RANDOM.nextInt(Integer.MAX_VALUE); - final static long llength = RANDOM.nextInt(Integer.MAX_VALUE); + final static int length = RANDOM.nextInt(5, Integer.MAX_VALUE); + final static long llength = RANDOM.nextInt(2, Integer.MAX_VALUE); static int i; static long l; @Test - @IR(counts = { IRNode.CAST_II, "1" }) + @IR(counts = { IRNode.CAST_II, "2" }) public static int test1() { int j = Objects.checkIndex(i, length); int k = Objects.checkIndex(i + 1, length); @@ -67,7 +67,7 @@ public static void test1_runner() { } @Test - @IR(counts = { IRNode.CAST_LL, "1" }) + @IR(counts = { IRNode.CAST_LL, "2" }) public static long test2() { long j = Objects.checkIndex(l, llength); long k = Objects.checkIndex(l + 1, llength); @@ -82,4 +82,24 @@ public static void test2_runner() { throw new RuntimeException("incorrect result: " + res); } } + + // Test commoning of Casts after loop opts when they are at the same control + @Test + @IR(counts = { IRNode.CAST_II, "2" }) + public static int test3() { + int j = Objects.checkIndex(i - 3, length); + j += Objects.checkIndex(i, length); + j += Objects.checkIndex(i - 2, length); + j += Objects.checkIndex(i - 1, length); + return j; + } + + @Run(test = "test3") + public static void test3_runner() { + i = RANDOM.nextInt(3, length-1); + int res = test3(); + if (res != i * 4 - 6) { + throw new RuntimeException("incorrect result: " + res + " for i = " + i); + } + } } diff --git a/test/hotspot/jtreg/compiler/rangechecks/TestArrayAccessAboveRCAfterRCCastIIEliminated.java b/test/hotspot/jtreg/compiler/rangechecks/TestArrayAccessAboveRCAfterRCCastIIEliminated.java index 87e87842af337..9adfff7a5938f 100644 --- a/test/hotspot/jtreg/compiler/rangechecks/TestArrayAccessAboveRCAfterRCCastIIEliminated.java +++ b/test/hotspot/jtreg/compiler/rangechecks/TestArrayAccessAboveRCAfterRCCastIIEliminated.java @@ -44,7 +44,6 @@ public class TestArrayAccessAboveRCAfterRCCastIIEliminated { private static volatile int volatileField; public static void main(String[] args) { - int[] array = new int[100]; for (int i = 0; i < 20_000; i++) { test1(9, 10, 1, true); test1(9, 10, 1, false); @@ -72,6 +71,15 @@ public static void main(String[] args) { test12(9, 10, 1, false); test13(9, 10, 1, true); test13(9, 10, 1, false); + test14(8, 0, 1, true); + test14(8, 0, 1, false); + inlined14(0, 0); + test15(8, 0, 1, true); + test15(8, 0, 1, false); + inlined15(0, 0); + test16(0, 9, 1, true, false); + test16(0, 9, 1, false, false); + inlined16_2(9, 1, 0, arrayField, true, 0); } try { test1(-1, 10, 1, true); @@ -125,6 +133,14 @@ public static void main(String[] args) { test13(-1, 10, 1, true); } catch (ArrayIndexOutOfBoundsException arrayIndexOutOfBoundsException) { } + try { + test14(Integer.MAX_VALUE, 10, 1, true); + } catch (ArrayIndexOutOfBoundsException arrayIndexOutOfBoundsException) { + } + try { + test15(Integer.MAX_VALUE, 10, 1, true); + } catch (ArrayIndexOutOfBoundsException arrayIndexOutOfBoundsException) { + } } private static void test1(int i, int j, int flag, boolean flag2) { @@ -468,6 +484,120 @@ private static void test13(int i, int j, int flag, boolean flag2) { } } + // Widened range check cast type after loop opts causes control dependency to be lost + private static void test14(int i, int j, int flag, boolean flag2) { + int l = 0; + for (; l < 10; l++); + j = inlined14(j, l); + int[] array = new int[10]; + notInlined(array); + if (flag == 0) { + } + if (flag2) { + float[] newArray = new float[10]; + newArray[i+j] = 42; // i+j in [0, 9] + float[] otherArray = new float[i+j]; // i+j in [0, max] + if (flag == 0) { + } + intField = array[otherArray.length]; + } else { + float[] newArray = new float[10]; + newArray[i+j] = 42; // i+j in [0, 9] + float[] otherArray = new float[i+j]; // i+j in [0, max] + if (flag == 0) { + } + intField = array[otherArray.length]; + } + } + + private static int inlined14(int j, int l) { + if (l == 10) { + j = 1; + } + return j; + } + + private static void test15(int i, int j, int flag, boolean flag2) { + i = Integer.max(i, Integer.MIN_VALUE + 1); + int l = 0; + for (; l < 10; l++); + j = inlined15(j, l); + int[] array = new int[10]; + notInlined(array); + if (flag == 0) { + } + if (flag2) { + float[] newArray = new float[10]; + newArray[i+j] = 42; // i+j in [0, 9] + float[] otherArray = new float[i+j]; // i+j in [0, max] + if (flag == 0) { + } + intField = array[otherArray.length]; + } else { + float[] newArray = new float[10]; + newArray[i+j] = 42; // i+j in [0, 9] + float[] otherArray = new float[i+j]; // i+j in [0, max] + if (flag == 0) { + } + intField = array[otherArray.length]; + } + } + + private static int inlined15(int j, int l) { + if (l == 10) { + j = Integer.max(j, Integer.MIN_VALUE + 10); + } + return j; + } + + private static void test16(int k, int j, int flag, boolean flag2, boolean flag3) { + int l = 0; + for (; l < 10; l++); + int m = inlined14(j, l); + + int i = inlined16(k); + j = Integer.min(j, 9); + int[] array = new int[10]; + notInlined(array); + if (flag == 0) { + throw new RuntimeException(""); + } + if (flag2) { + inlined16_2(j, flag, i, array, flag3, m); + } else { + inlined16_2(j, flag, i, array, flag3, m); + } + } + + private static void inlined16_2(int j, int flag, int i, int[] array, boolean flag2, int m) { + if (flag2) { + float[] newArray = new float[j + 1]; + // RC i