1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/int64-lowering.h"
6 #include "src/compiler/common-operator.h"
7 #include "src/compiler/diamond.h"
8 #include "src/compiler/graph.h"
9 #include "src/compiler/linkage.h"
10 #include "src/compiler/machine-operator.h"
11 #include "src/compiler/node-matchers.h"
12 #include "src/compiler/node-properties.h"
13 
14 #include "src/compiler/node.h"
15 #include "src/wasm/wasm-module.h"
16 #include "src/zone/zone.h"
17 
18 namespace v8 {
19 namespace internal {
20 namespace compiler {
21 
Int64Lowering(Graph * graph,MachineOperatorBuilder * machine,CommonOperatorBuilder * common,Zone * zone,Signature<MachineRepresentation> * signature)22 Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine,
23                              CommonOperatorBuilder* common, Zone* zone,
24                              Signature<MachineRepresentation>* signature)
25     : zone_(zone),
26       graph_(graph),
27       machine_(machine),
28       common_(common),
29       state_(graph, 3),
30       stack_(zone),
31       replacements_(nullptr),
32       signature_(signature),
33       placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"),
34                                   graph->start())) {
35   DCHECK_NOT_NULL(graph);
36   DCHECK_NOT_NULL(graph->end());
37   replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
38   memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
39 }
40 
LowerGraph()41 void Int64Lowering::LowerGraph() {
42   if (!machine()->Is32()) {
43     return;
44   }
45   stack_.push_back({graph()->end(), 0});
46   state_.Set(graph()->end(), State::kOnStack);
47 
48   while (!stack_.empty()) {
49     NodeState& top = stack_.back();
50     if (top.input_index == top.node->InputCount()) {
51       // All inputs of top have already been lowered, now lower top.
52       stack_.pop_back();
53       state_.Set(top.node, State::kVisited);
54       LowerNode(top.node);
55     } else {
56       // Push the next input onto the stack.
57       Node* input = top.node->InputAt(top.input_index++);
58       if (state_.Get(input) == State::kUnvisited) {
59         if (input->opcode() == IrOpcode::kPhi) {
60           // To break cycles with phi nodes we push phis on a separate stack so
61           // that they are processed after all other nodes.
62           PreparePhiReplacement(input);
63           stack_.push_front({input, 0});
64         } else if (input->opcode() == IrOpcode::kEffectPhi) {
65           stack_.push_front({input, 0});
66         } else {
67           stack_.push_back({input, 0});
68         }
69         state_.Set(input, State::kOnStack);
70       }
71     }
72   }
73 }
74 
GetParameterIndexAfterLowering(Signature<MachineRepresentation> * signature,int old_index)75 static int GetParameterIndexAfterLowering(
76     Signature<MachineRepresentation>* signature, int old_index) {
77   int result = old_index;
78   for (int i = 0; i < old_index; i++) {
79     if (signature->GetParam(i) == MachineRepresentation::kWord64) {
80       result++;
81     }
82   }
83   return result;
84 }
85 
GetParameterCountAfterLowering(Signature<MachineRepresentation> * signature)86 int Int64Lowering::GetParameterCountAfterLowering(
87     Signature<MachineRepresentation>* signature) {
88   // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
89   // after lowering.
90   return GetParameterIndexAfterLowering(
91       signature, static_cast<int>(signature->parameter_count()));
92 }
93 
GetReturnCountAfterLowering(Signature<MachineRepresentation> * signature)94 static int GetReturnCountAfterLowering(
95     Signature<MachineRepresentation>* signature) {
96   int result = static_cast<int>(signature->return_count());
97   for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
98     if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
99       result++;
100     }
101   }
102   return result;
103 }
104 
GetIndexNodes(Node * index,Node * & index_low,Node * & index_high)105 void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low,
106                                   Node*& index_high) {
107 #if defined(V8_TARGET_LITTLE_ENDIAN)
108   index_low = index;
109   index_high = graph()->NewNode(machine()->Int32Add(), index,
110                                 graph()->NewNode(common()->Int32Constant(4)));
111 #elif defined(V8_TARGET_BIG_ENDIAN)
112   index_low = graph()->NewNode(machine()->Int32Add(), index,
113                                graph()->NewNode(common()->Int32Constant(4)));
114   index_high = index;
115 #endif
116 }
117 
118 #if defined(V8_TARGET_LITTLE_ENDIAN)
119 const int Int64Lowering::kLowerWordOffset = 0;
120 const int Int64Lowering::kHigherWordOffset = 4;
121 #elif defined(V8_TARGET_BIG_ENDIAN)
122 const int Int64Lowering::kLowerWordOffset = 4;
123 const int Int64Lowering::kHigherWordOffset = 0;
124 #endif
125 
LowerNode(Node * node)126 void Int64Lowering::LowerNode(Node* node) {
127   switch (node->opcode()) {
128     case IrOpcode::kInt64Constant: {
129       int64_t value = OpParameter<int64_t>(node);
130       Node* low_node = graph()->NewNode(
131           common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
132       Node* high_node = graph()->NewNode(
133           common()->Int32Constant(static_cast<int32_t>(value >> 32)));
134       ReplaceNode(node, low_node, high_node);
135       break;
136     }
137     case IrOpcode::kLoad:
138     case IrOpcode::kUnalignedLoad: {
139       MachineRepresentation rep;
140       if (node->opcode() == IrOpcode::kLoad) {
141         rep = LoadRepresentationOf(node->op()).representation();
142       } else {
143         DCHECK(node->opcode() == IrOpcode::kUnalignedLoad);
144         rep = UnalignedLoadRepresentationOf(node->op()).representation();
145       }
146 
147       if (rep == MachineRepresentation::kWord64) {
148         Node* base = node->InputAt(0);
149         Node* index = node->InputAt(1);
150         Node* index_low;
151         Node* index_high;
152         GetIndexNodes(index, index_low, index_high);
153         const Operator* load_op;
154 
155         if (node->opcode() == IrOpcode::kLoad) {
156           load_op = machine()->Load(MachineType::Int32());
157         } else {
158           DCHECK(node->opcode() == IrOpcode::kUnalignedLoad);
159           load_op = machine()->UnalignedLoad(MachineType::Int32());
160         }
161 
162         Node* high_node;
163         if (node->InputCount() > 2) {
164           Node* effect_high = node->InputAt(2);
165           Node* control_high = node->InputAt(3);
166           high_node = graph()->NewNode(load_op, base, index_high, effect_high,
167                                        control_high);
168           // change the effect change from old_node --> old_effect to
169           // old_node --> high_node --> old_effect.
170           node->ReplaceInput(2, high_node);
171         } else {
172           high_node = graph()->NewNode(load_op, base, index_high);
173         }
174         node->ReplaceInput(1, index_low);
175         NodeProperties::ChangeOp(node, load_op);
176         ReplaceNode(node, node, high_node);
177       } else {
178         DefaultLowering(node);
179       }
180       break;
181     }
182     case IrOpcode::kStore:
183     case IrOpcode::kUnalignedStore: {
184       MachineRepresentation rep;
185       if (node->opcode() == IrOpcode::kStore) {
186         rep = StoreRepresentationOf(node->op()).representation();
187       } else {
188         DCHECK(node->opcode() == IrOpcode::kUnalignedStore);
189         rep = UnalignedStoreRepresentationOf(node->op());
190       }
191 
192       if (rep == MachineRepresentation::kWord64) {
193         // We change the original store node to store the low word, and create
194         // a new store node to store the high word. The effect and control edges
195         // are copied from the original store to the new store node, the effect
196         // edge of the original store is redirected to the new store.
197         Node* base = node->InputAt(0);
198         Node* index = node->InputAt(1);
199         Node* index_low;
200         Node* index_high;
201         GetIndexNodes(index, index_low, index_high);
202         Node* value = node->InputAt(2);
203         DCHECK(HasReplacementLow(value));
204         DCHECK(HasReplacementHigh(value));
205 
206         const Operator* store_op;
207         if (node->opcode() == IrOpcode::kStore) {
208           WriteBarrierKind write_barrier_kind =
209               StoreRepresentationOf(node->op()).write_barrier_kind();
210           store_op = machine()->Store(StoreRepresentation(
211               MachineRepresentation::kWord32, write_barrier_kind));
212         } else {
213           DCHECK(node->opcode() == IrOpcode::kUnalignedStore);
214           store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
215         }
216 
217         Node* high_node;
218         if (node->InputCount() > 3) {
219           Node* effect_high = node->InputAt(3);
220           Node* control_high = node->InputAt(4);
221           high_node = graph()->NewNode(store_op, base, index_high,
222                                        GetReplacementHigh(value), effect_high,
223                                        control_high);
224           node->ReplaceInput(3, high_node);
225 
226         } else {
227           high_node = graph()->NewNode(store_op, base, index_high,
228                                        GetReplacementHigh(value));
229         }
230 
231         node->ReplaceInput(1, index_low);
232         node->ReplaceInput(2, GetReplacementLow(value));
233         NodeProperties::ChangeOp(node, store_op);
234         ReplaceNode(node, node, high_node);
235       } else {
236         if (HasReplacementLow(node->InputAt(2))) {
237           node->ReplaceInput(2, GetReplacementLow(node->InputAt(2)));
238         }
239       }
240       break;
241     }
242     case IrOpcode::kStart: {
243       int parameter_count = GetParameterCountAfterLowering(signature());
244       // Only exchange the node if the parameter count actually changed.
245       if (parameter_count != static_cast<int>(signature()->parameter_count())) {
246         int delta =
247             parameter_count - static_cast<int>(signature()->parameter_count());
248         int new_output_count = node->op()->ValueOutputCount() + delta;
249         NodeProperties::ChangeOp(node, common()->Start(new_output_count));
250       }
251       break;
252     }
253     case IrOpcode::kParameter: {
254       DCHECK(node->InputCount() == 1);
255       // Only exchange the node if the parameter count actually changed. We do
256       // not even have to do the default lowering because the the start node,
257       // the only input of a parameter node, only changes if the parameter count
258       // changes.
259       if (GetParameterCountAfterLowering(signature()) !=
260           static_cast<int>(signature()->parameter_count())) {
261         int old_index = ParameterIndexOf(node->op());
262         int new_index = GetParameterIndexAfterLowering(signature(), old_index);
263         NodeProperties::ChangeOp(node, common()->Parameter(new_index));
264 
265         Node* high_node = nullptr;
266         if (signature()->GetParam(old_index) ==
267             MachineRepresentation::kWord64) {
268           high_node = graph()->NewNode(common()->Parameter(new_index + 1),
269                                        graph()->start());
270         }
271         ReplaceNode(node, node, high_node);
272       }
273       break;
274     }
275     case IrOpcode::kReturn: {
276       DefaultLowering(node);
277       int new_return_count = GetReturnCountAfterLowering(signature());
278       if (static_cast<int>(signature()->return_count()) != new_return_count) {
279         NodeProperties::ChangeOp(node, common()->Return(new_return_count));
280       }
281       break;
282     }
283     case IrOpcode::kCall: {
284       // TODO(turbofan): Make WASM code const-correct wrt. CallDescriptor.
285       CallDescriptor* descriptor =
286           const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
287       if (DefaultLowering(node) ||
288           (descriptor->ReturnCount() == 1 &&
289            descriptor->GetReturnType(0) == MachineType::Int64())) {
290         // We have to adjust the call descriptor.
291         const Operator* op = common()->Call(
292             wasm::ModuleEnv::GetI32WasmCallDescriptor(zone(), descriptor));
293         NodeProperties::ChangeOp(node, op);
294       }
295       if (descriptor->ReturnCount() == 1 &&
296           descriptor->GetReturnType(0) == MachineType::Int64()) {
297         // We access the additional return values through projections.
298         Node* low_node =
299             graph()->NewNode(common()->Projection(0), node, graph()->start());
300         Node* high_node =
301             graph()->NewNode(common()->Projection(1), node, graph()->start());
302         ReplaceNode(node, low_node, high_node);
303       }
304       break;
305     }
306     case IrOpcode::kWord64And: {
307       DCHECK(node->InputCount() == 2);
308       Node* left = node->InputAt(0);
309       Node* right = node->InputAt(1);
310 
311       Node* low_node =
312           graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
313                            GetReplacementLow(right));
314       Node* high_node =
315           graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
316                            GetReplacementHigh(right));
317       ReplaceNode(node, low_node, high_node);
318       break;
319     }
320     case IrOpcode::kTruncateInt64ToInt32: {
321       DCHECK(node->InputCount() == 1);
322       Node* input = node->InputAt(0);
323       ReplaceNode(node, GetReplacementLow(input), nullptr);
324       node->NullAllInputs();
325       break;
326     }
327     case IrOpcode::kInt64Add: {
328       DCHECK(node->InputCount() == 2);
329 
330       Node* right = node->InputAt(1);
331       node->ReplaceInput(1, GetReplacementLow(right));
332       node->AppendInput(zone(), GetReplacementHigh(right));
333 
334       Node* left = node->InputAt(0);
335       node->ReplaceInput(0, GetReplacementLow(left));
336       node->InsertInput(zone(), 1, GetReplacementHigh(left));
337 
338       NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
339       // We access the additional return values through projections.
340       Node* low_node =
341           graph()->NewNode(common()->Projection(0), node, graph()->start());
342       Node* high_node =
343           graph()->NewNode(common()->Projection(1), node, graph()->start());
344       ReplaceNode(node, low_node, high_node);
345       break;
346     }
347     case IrOpcode::kInt64Sub: {
348       DCHECK(node->InputCount() == 2);
349 
350       Node* right = node->InputAt(1);
351       node->ReplaceInput(1, GetReplacementLow(right));
352       node->AppendInput(zone(), GetReplacementHigh(right));
353 
354       Node* left = node->InputAt(0);
355       node->ReplaceInput(0, GetReplacementLow(left));
356       node->InsertInput(zone(), 1, GetReplacementHigh(left));
357 
358       NodeProperties::ChangeOp(node, machine()->Int32PairSub());
359       // We access the additional return values through projections.
360       Node* low_node =
361           graph()->NewNode(common()->Projection(0), node, graph()->start());
362       Node* high_node =
363           graph()->NewNode(common()->Projection(1), node, graph()->start());
364       ReplaceNode(node, low_node, high_node);
365       break;
366     }
367     case IrOpcode::kInt64Mul: {
368       DCHECK(node->InputCount() == 2);
369 
370       Node* right = node->InputAt(1);
371       node->ReplaceInput(1, GetReplacementLow(right));
372       node->AppendInput(zone(), GetReplacementHigh(right));
373 
374       Node* left = node->InputAt(0);
375       node->ReplaceInput(0, GetReplacementLow(left));
376       node->InsertInput(zone(), 1, GetReplacementHigh(left));
377 
378       NodeProperties::ChangeOp(node, machine()->Int32PairMul());
379       // We access the additional return values through projections.
380       Node* low_node =
381           graph()->NewNode(common()->Projection(0), node, graph()->start());
382       Node* high_node =
383           graph()->NewNode(common()->Projection(1), node, graph()->start());
384       ReplaceNode(node, low_node, high_node);
385       break;
386     }
387     case IrOpcode::kWord64Or: {
388       DCHECK(node->InputCount() == 2);
389       Node* left = node->InputAt(0);
390       Node* right = node->InputAt(1);
391 
392       Node* low_node =
393           graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
394                            GetReplacementLow(right));
395       Node* high_node =
396           graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
397                            GetReplacementHigh(right));
398       ReplaceNode(node, low_node, high_node);
399       break;
400     }
401     case IrOpcode::kWord64Xor: {
402       DCHECK(node->InputCount() == 2);
403       Node* left = node->InputAt(0);
404       Node* right = node->InputAt(1);
405 
406       Node* low_node =
407           graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
408                            GetReplacementLow(right));
409       Node* high_node =
410           graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
411                            GetReplacementHigh(right));
412       ReplaceNode(node, low_node, high_node);
413       break;
414     }
415     case IrOpcode::kWord64Shl: {
416       // TODO(turbofan): if the shift count >= 32, then we can set the low word
417       // of the output to 0 and just calculate the high word.
418       DCHECK(node->InputCount() == 2);
419       Node* shift = node->InputAt(1);
420       if (HasReplacementLow(shift)) {
421         // We do not have to care about the high word replacement, because
422         // the shift can only be between 0 and 63 anyways.
423         node->ReplaceInput(1, GetReplacementLow(shift));
424       }
425 
426       Node* value = node->InputAt(0);
427       node->ReplaceInput(0, GetReplacementLow(value));
428       node->InsertInput(zone(), 1, GetReplacementHigh(value));
429 
430       NodeProperties::ChangeOp(node, machine()->Word32PairShl());
431       // We access the additional return values through projections.
432       Node* low_node =
433           graph()->NewNode(common()->Projection(0), node, graph()->start());
434       Node* high_node =
435           graph()->NewNode(common()->Projection(1), node, graph()->start());
436       ReplaceNode(node, low_node, high_node);
437       break;
438     }
439     case IrOpcode::kWord64Shr: {
440       // TODO(turbofan): if the shift count >= 32, then we can set the low word
441       // of the output to 0 and just calculate the high word.
442       DCHECK(node->InputCount() == 2);
443       Node* shift = node->InputAt(1);
444       if (HasReplacementLow(shift)) {
445         // We do not have to care about the high word replacement, because
446         // the shift can only be between 0 and 63 anyways.
447         node->ReplaceInput(1, GetReplacementLow(shift));
448       }
449 
450       Node* value = node->InputAt(0);
451       node->ReplaceInput(0, GetReplacementLow(value));
452       node->InsertInput(zone(), 1, GetReplacementHigh(value));
453 
454       NodeProperties::ChangeOp(node, machine()->Word32PairShr());
455       // We access the additional return values through projections.
456       Node* low_node =
457           graph()->NewNode(common()->Projection(0), node, graph()->start());
458       Node* high_node =
459           graph()->NewNode(common()->Projection(1), node, graph()->start());
460       ReplaceNode(node, low_node, high_node);
461       break;
462     }
463     case IrOpcode::kWord64Sar: {
464       // TODO(turbofan): if the shift count >= 32, then we can set the low word
465       // of the output to 0 and just calculate the high word.
466       DCHECK(node->InputCount() == 2);
467       Node* shift = node->InputAt(1);
468       if (HasReplacementLow(shift)) {
469         // We do not have to care about the high word replacement, because
470         // the shift can only be between 0 and 63 anyways.
471         node->ReplaceInput(1, GetReplacementLow(shift));
472       }
473 
474       Node* value = node->InputAt(0);
475       node->ReplaceInput(0, GetReplacementLow(value));
476       node->InsertInput(zone(), 1, GetReplacementHigh(value));
477 
478       NodeProperties::ChangeOp(node, machine()->Word32PairSar());
479       // We access the additional return values through projections.
480       Node* low_node =
481           graph()->NewNode(common()->Projection(0), node, graph()->start());
482       Node* high_node =
483           graph()->NewNode(common()->Projection(1), node, graph()->start());
484       ReplaceNode(node, low_node, high_node);
485       break;
486     }
487     case IrOpcode::kWord64Equal: {
488       DCHECK(node->InputCount() == 2);
489       Node* left = node->InputAt(0);
490       Node* right = node->InputAt(1);
491 
492       // TODO(wasm): Use explicit comparisons and && here?
493       Node* replacement = graph()->NewNode(
494           machine()->Word32Equal(),
495           graph()->NewNode(
496               machine()->Word32Or(),
497               graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
498                                GetReplacementLow(right)),
499               graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
500                                GetReplacementHigh(right))),
501           graph()->NewNode(common()->Int32Constant(0)));
502 
503       ReplaceNode(node, replacement, nullptr);
504       break;
505     }
506     case IrOpcode::kInt64LessThan: {
507       LowerComparison(node, machine()->Int32LessThan(),
508                       machine()->Uint32LessThan());
509       break;
510     }
511     case IrOpcode::kInt64LessThanOrEqual: {
512       LowerComparison(node, machine()->Int32LessThan(),
513                       machine()->Uint32LessThanOrEqual());
514       break;
515     }
516     case IrOpcode::kUint64LessThan: {
517       LowerComparison(node, machine()->Uint32LessThan(),
518                       machine()->Uint32LessThan());
519       break;
520     }
521     case IrOpcode::kUint64LessThanOrEqual: {
522       LowerComparison(node, machine()->Uint32LessThan(),
523                       machine()->Uint32LessThanOrEqual());
524       break;
525     }
526     case IrOpcode::kChangeInt32ToInt64: {
527       DCHECK(node->InputCount() == 1);
528       Node* input = node->InputAt(0);
529       if (HasReplacementLow(input)) {
530         input = GetReplacementLow(input);
531       }
532       // We use SAR to preserve the sign in the high word.
533       ReplaceNode(
534           node, input,
535           graph()->NewNode(machine()->Word32Sar(), input,
536                            graph()->NewNode(common()->Int32Constant(31))));
537       node->NullAllInputs();
538       break;
539     }
540     case IrOpcode::kChangeUint32ToUint64: {
541       DCHECK(node->InputCount() == 1);
542       Node* input = node->InputAt(0);
543       if (HasReplacementLow(input)) {
544         input = GetReplacementLow(input);
545       }
546       ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
547       node->NullAllInputs();
548       break;
549     }
550     case IrOpcode::kBitcastInt64ToFloat64: {
551       DCHECK(node->InputCount() == 1);
552       Node* input = node->InputAt(0);
553       Node* stack_slot = graph()->NewNode(
554           machine()->StackSlot(MachineRepresentation::kWord64));
555 
556       Node* store_high_word = graph()->NewNode(
557           machine()->Store(
558               StoreRepresentation(MachineRepresentation::kWord32,
559                                   WriteBarrierKind::kNoWriteBarrier)),
560           stack_slot,
561           graph()->NewNode(common()->Int32Constant(kHigherWordOffset)),
562           GetReplacementHigh(input), graph()->start(), graph()->start());
563 
564       Node* store_low_word = graph()->NewNode(
565           machine()->Store(
566               StoreRepresentation(MachineRepresentation::kWord32,
567                                   WriteBarrierKind::kNoWriteBarrier)),
568           stack_slot,
569           graph()->NewNode(common()->Int32Constant(kLowerWordOffset)),
570           GetReplacementLow(input), store_high_word, graph()->start());
571 
572       Node* load =
573           graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
574                            graph()->NewNode(common()->Int32Constant(0)),
575                            store_low_word, graph()->start());
576 
577       ReplaceNode(node, load, nullptr);
578       break;
579     }
580     case IrOpcode::kBitcastFloat64ToInt64: {
581       DCHECK(node->InputCount() == 1);
582       Node* input = node->InputAt(0);
583       if (HasReplacementLow(input)) {
584         input = GetReplacementLow(input);
585       }
586       Node* stack_slot = graph()->NewNode(
587           machine()->StackSlot(MachineRepresentation::kWord64));
588       Node* store = graph()->NewNode(
589           machine()->Store(
590               StoreRepresentation(MachineRepresentation::kFloat64,
591                                   WriteBarrierKind::kNoWriteBarrier)),
592           stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
593           graph()->start(), graph()->start());
594 
595       Node* high_node = graph()->NewNode(
596           machine()->Load(MachineType::Int32()), stack_slot,
597           graph()->NewNode(common()->Int32Constant(kHigherWordOffset)), store,
598           graph()->start());
599 
600       Node* low_node = graph()->NewNode(
601           machine()->Load(MachineType::Int32()), stack_slot,
602           graph()->NewNode(common()->Int32Constant(kLowerWordOffset)), store,
603           graph()->start());
604       ReplaceNode(node, low_node, high_node);
605       break;
606     }
607     case IrOpcode::kWord64Ror: {
608       DCHECK(node->InputCount() == 2);
609       Node* input = node->InputAt(0);
610       Node* shift = HasReplacementLow(node->InputAt(1))
611                         ? GetReplacementLow(node->InputAt(1))
612                         : node->InputAt(1);
613       Int32Matcher m(shift);
614       if (m.HasValue()) {
615         // Precondition: 0 <= shift < 64.
616         int32_t shift_value = m.Value() & 0x3f;
617         if (shift_value == 0) {
618           ReplaceNode(node, GetReplacementLow(input),
619                       GetReplacementHigh(input));
620         } else if (shift_value == 32) {
621           ReplaceNode(node, GetReplacementHigh(input),
622                       GetReplacementLow(input));
623         } else {
624           Node* low_input;
625           Node* high_input;
626           if (shift_value < 32) {
627             low_input = GetReplacementLow(input);
628             high_input = GetReplacementHigh(input);
629           } else {
630             low_input = GetReplacementHigh(input);
631             high_input = GetReplacementLow(input);
632           }
633           int32_t masked_shift_value = shift_value & 0x1f;
634           Node* masked_shift =
635               graph()->NewNode(common()->Int32Constant(masked_shift_value));
636           Node* inv_shift = graph()->NewNode(
637               common()->Int32Constant(32 - masked_shift_value));
638 
639           Node* low_node = graph()->NewNode(
640               machine()->Word32Or(),
641               graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
642               graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
643           Node* high_node = graph()->NewNode(
644               machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
645                                                       high_input, masked_shift),
646               graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
647           ReplaceNode(node, low_node, high_node);
648         }
649       } else {
650         Node* safe_shift = shift;
651         if (!machine()->Word32ShiftIsSafe()) {
652           safe_shift =
653               graph()->NewNode(machine()->Word32And(), shift,
654                                graph()->NewNode(common()->Int32Constant(0x1f)));
655         }
656 
657         // By creating this bit-mask with SAR and SHL we do not have to deal
658         // with shift == 0 as a special case.
659         Node* inv_mask = graph()->NewNode(
660             machine()->Word32Shl(),
661             graph()->NewNode(machine()->Word32Sar(),
662                              graph()->NewNode(common()->Int32Constant(
663                                  std::numeric_limits<int32_t>::min())),
664                              safe_shift),
665             graph()->NewNode(common()->Int32Constant(1)));
666 
667         Node* bit_mask =
668             graph()->NewNode(machine()->Word32Xor(), inv_mask,
669                              graph()->NewNode(common()->Int32Constant(-1)));
670 
671         // We have to mask the shift value for this comparison. If
672         // !machine()->Word32ShiftIsSafe() then the masking should already be
673         // part of the graph.
674         Node* masked_shift6 = shift;
675         if (machine()->Word32ShiftIsSafe()) {
676           masked_shift6 =
677               graph()->NewNode(machine()->Word32And(), shift,
678                                graph()->NewNode(common()->Int32Constant(0x3f)));
679         }
680 
681         Diamond lt32(
682             graph(), common(),
683             graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
684                              graph()->NewNode(common()->Int32Constant(32))));
685 
686         // The low word and the high word can be swapped either at the input or
687         // at the output. We swap the inputs so that shift does not have to be
688         // kept for so long in a register.
689         Node* input_low =
690             lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
691                      GetReplacementHigh(input));
692         Node* input_high =
693             lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
694                      GetReplacementLow(input));
695 
696         Node* rotate_low =
697             graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
698         Node* rotate_high =
699             graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
700 
701         Node* low_node = graph()->NewNode(
702             machine()->Word32Or(),
703             graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
704             graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
705 
706         Node* high_node = graph()->NewNode(
707             machine()->Word32Or(),
708             graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
709             graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
710 
711         ReplaceNode(node, low_node, high_node);
712       }
713       break;
714     }
715     case IrOpcode::kWord64Clz: {
716       DCHECK(node->InputCount() == 1);
717       Node* input = node->InputAt(0);
718       Diamond d(
719           graph(), common(),
720           graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
721                            graph()->NewNode(common()->Int32Constant(0))));
722 
723       Node* low_node = d.Phi(
724           MachineRepresentation::kWord32,
725           graph()->NewNode(machine()->Int32Add(),
726                            graph()->NewNode(machine()->Word32Clz(),
727                                             GetReplacementLow(input)),
728                            graph()->NewNode(common()->Int32Constant(32))),
729           graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
730       ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
731       break;
732     }
733     case IrOpcode::kWord64Ctz: {
734       DCHECK(node->InputCount() == 1);
735       DCHECK(machine()->Word32Ctz().IsSupported());
736       Node* input = node->InputAt(0);
737       Diamond d(
738           graph(), common(),
739           graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
740                            graph()->NewNode(common()->Int32Constant(0))));
741       Node* low_node =
742           d.Phi(MachineRepresentation::kWord32,
743                 graph()->NewNode(machine()->Int32Add(),
744                                  graph()->NewNode(machine()->Word32Ctz().op(),
745                                                   GetReplacementHigh(input)),
746                                  graph()->NewNode(common()->Int32Constant(32))),
747                 graph()->NewNode(machine()->Word32Ctz().op(),
748                                  GetReplacementLow(input)));
749       ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
750       break;
751     }
752     case IrOpcode::kWord64Popcnt: {
753       DCHECK(node->InputCount() == 1);
754       Node* input = node->InputAt(0);
755       // We assume that a Word64Popcnt node only has been created if
756       // Word32Popcnt is actually supported.
757       DCHECK(machine()->Word32Popcnt().IsSupported());
758       ReplaceNode(node, graph()->NewNode(
759                             machine()->Int32Add(),
760                             graph()->NewNode(machine()->Word32Popcnt().op(),
761                                              GetReplacementLow(input)),
762                             graph()->NewNode(machine()->Word32Popcnt().op(),
763                                              GetReplacementHigh(input))),
764                   graph()->NewNode(common()->Int32Constant(0)));
765       break;
766     }
767     case IrOpcode::kPhi: {
768       MachineRepresentation rep = PhiRepresentationOf(node->op());
769       if (rep == MachineRepresentation::kWord64) {
770         // The replacement nodes have already been created, we only have to
771         // replace placeholder nodes.
772         Node* low_node = GetReplacementLow(node);
773         Node* high_node = GetReplacementHigh(node);
774         for (int i = 0; i < node->op()->ValueInputCount(); i++) {
775           low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
776           high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
777         }
778       } else {
779         DefaultLowering(node);
780       }
781       break;
782     }
783     case IrOpcode::kProjection: {
784       Node* call = node->InputAt(0);
785       DCHECK_EQ(IrOpcode::kCall, call->opcode());
786       CallDescriptor* descriptor =
787           const_cast<CallDescriptor*>(CallDescriptorOf(call->op()));
788       for (size_t i = 0; i < descriptor->ReturnCount(); i++) {
789         if (descriptor->GetReturnType(i) == MachineType::Int64()) {
790           UNREACHABLE();  // TODO(titzer): implement multiple i64 returns.
791         }
792       }
793       break;
794     }
795     case IrOpcode::kWord64ReverseBytes: {
796       Node* input = node->InputAt(0);
797       ReplaceNode(node, graph()->NewNode(machine()->Word32ReverseBytes().op(),
798                                          GetReplacementHigh(input)),
799                   graph()->NewNode(machine()->Word32ReverseBytes().op(),
800                                    GetReplacementLow(input)));
801       break;
802     }
803 
804     default: { DefaultLowering(node); }
805   }
806 }  // NOLINT(readability/fn_size)
807 
LowerComparison(Node * node,const Operator * high_word_op,const Operator * low_word_op)808 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
809                                     const Operator* low_word_op) {
810   DCHECK(node->InputCount() == 2);
811   Node* left = node->InputAt(0);
812   Node* right = node->InputAt(1);
813   Node* replacement = graph()->NewNode(
814       machine()->Word32Or(),
815       graph()->NewNode(high_word_op, GetReplacementHigh(left),
816                        GetReplacementHigh(right)),
817       graph()->NewNode(
818           machine()->Word32And(),
819           graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
820                            GetReplacementHigh(right)),
821           graph()->NewNode(low_word_op, GetReplacementLow(left),
822                            GetReplacementLow(right))));
823 
824   ReplaceNode(node, replacement, nullptr);
825 }
826 
DefaultLowering(Node * node)827 bool Int64Lowering::DefaultLowering(Node* node) {
828   bool something_changed = false;
829   for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
830     Node* input = node->InputAt(i);
831     if (HasReplacementLow(input)) {
832       something_changed = true;
833       node->ReplaceInput(i, GetReplacementLow(input));
834     }
835     if (HasReplacementHigh(input)) {
836       something_changed = true;
837       node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
838     }
839   }
840   return something_changed;
841 }
842 
ReplaceNode(Node * old,Node * new_low,Node * new_high)843 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
844   // if new_low == nullptr, then also new_high == nullptr.
845   DCHECK(new_low != nullptr || new_high == nullptr);
846   replacements_[old->id()].low = new_low;
847   replacements_[old->id()].high = new_high;
848 }
849 
HasReplacementLow(Node * node)850 bool Int64Lowering::HasReplacementLow(Node* node) {
851   return replacements_[node->id()].low != nullptr;
852 }
853 
GetReplacementLow(Node * node)854 Node* Int64Lowering::GetReplacementLow(Node* node) {
855   Node* result = replacements_[node->id()].low;
856   DCHECK(result);
857   return result;
858 }
859 
HasReplacementHigh(Node * node)860 bool Int64Lowering::HasReplacementHigh(Node* node) {
861   return replacements_[node->id()].high != nullptr;
862 }
863 
GetReplacementHigh(Node * node)864 Node* Int64Lowering::GetReplacementHigh(Node* node) {
865   Node* result = replacements_[node->id()].high;
866   DCHECK(result);
867   return result;
868 }
869 
PreparePhiReplacement(Node * phi)870 void Int64Lowering::PreparePhiReplacement(Node* phi) {
871   MachineRepresentation rep = PhiRepresentationOf(phi->op());
872   if (rep == MachineRepresentation::kWord64) {
873     // We have to create the replacements for a phi node before we actually
874     // lower the phi to break potential cycles in the graph. The replacements of
875     // input nodes do not exist yet, so we use a placeholder node to pass the
876     // graph verifier.
877     int value_count = phi->op()->ValueInputCount();
878     Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
879     Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
880     for (int i = 0; i < value_count; i++) {
881       inputs_low[i] = placeholder_;
882       inputs_high[i] = placeholder_;
883     }
884     inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
885     inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
886     ReplaceNode(phi,
887                 graph()->NewNode(
888                     common()->Phi(MachineRepresentation::kWord32, value_count),
889                     value_count + 1, inputs_low, false),
890                 graph()->NewNode(
891                     common()->Phi(MachineRepresentation::kWord32, value_count),
892                     value_count + 1, inputs_high, false));
893   }
894 }
895 }  // namespace compiler
896 }  // namespace internal
897 }  // namespace v8
898