1// Copyright 2017 syzkaller project authors. All rights reserved.
2// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
3
4// See Intel Software Developer’s Manual Volume 2: Instruction Set Reference
5// and AMD64 Architecture Programmer’s Manual Volume 3: General-Purpose and System Instructions
6// for details of instruction encoding.
7
8package ifuzz
9
10import (
11	"math/rand"
12)
13
14// nolint: gocyclo
15func (insn *Insn) Encode(cfg *Config, r *rand.Rand) []byte {
16	if !insn.isCompatible(cfg) {
17		panic("instruction is not suitable for this mode")
18	}
19	if insn.Pseudo {
20		return insn.generator(cfg, r)
21	}
22
23	var operSize, immSize, dispSize, addrSize int
24	switch cfg.Mode {
25	case ModeLong64:
26		operSize, immSize, dispSize, addrSize = 4, 4, 4, 8
27	case ModeProt32:
28		operSize, immSize, dispSize, addrSize = 4, 4, 4, 4
29	case ModeProt16, ModeReal16:
30		operSize, immSize, dispSize, addrSize = 2, 2, 2, 2
31	default:
32		panic("bad mode")
33	}
34
35	var code []byte
36
37	rexR := false
38	var vvvv, vexR, vexX, vexB byte
39
40	// LEGACY PREFIXES
41	if insn.Vex == 0 {
42		for r.Intn(3) == 0 {
43			// LOCK 0xF0 is always added to insn.Prefix
44			prefixes := []byte{
45				0x2E, // CS
46				0x3E, // DS
47				0x26, // ES
48				0x64, // FS
49				0x65, // GS
50				0x36, // SS
51			}
52			if !insn.No66Prefix {
53				prefixes = append(prefixes, 0x66) // operand size
54			}
55			if cfg.Mode == ModeLong64 || !insn.Mem32 {
56				prefixes = append(prefixes, 0x67) // address size
57			}
58			if !insn.NoRepPrefix {
59				prefixes = append(prefixes,
60					0xF3, // REP
61					0xF2, // REPNE
62				)
63			}
64			pref := prefixes[r.Intn(len(prefixes))]
65			code = append(code, pref)
66		}
67
68		code = append(code, insn.Prefix...)
69
70		// REX
71		var rex byte
72		if cfg.Mode == ModeLong64 && r.Intn(2) == 0 {
73			// bit 0 - B
74			// bit 1 - X
75			// bit 2 - R
76			// bit 3 - W
77			rex = byte(0x40 | r.Intn(16))
78			if insn.Rexw == 1 {
79				rex |= 1 << 3
80			} else {
81				rex &^= 1 << 3
82			}
83			rexR = rex&0x4 != 0
84			code = append(code, rex)
85		}
86
87		operSize1, immSize1, dispSize1, addrSize1 := operSize, immSize, dispSize, addrSize
88		for _, pref := range code {
89			switch pref {
90			case 0x66:
91				if immSize == 4 {
92					immSize1 = 2
93					operSize1 = 2
94				} else if immSize == 2 {
95					immSize1 = 4
96					operSize1 = 4
97				}
98			case 0x67:
99				if addrSize == 8 {
100					addrSize1 = 4
101				} else if addrSize == 4 {
102					dispSize1 = 2
103					addrSize1 = 2
104				} else if addrSize == 2 {
105					dispSize1 = 4
106					addrSize1 = 4
107				}
108			}
109			if rex&(1<<3) != 0 {
110				operSize1 = 8
111				immSize1 = 4
112			}
113		}
114		operSize, immSize, dispSize, addrSize = operSize1, immSize1, dispSize1, addrSize1
115	} else {
116		// VEX/VOP
117		code = append(code, insn.Vex)
118		vexR = byte(1)
119		vexX = byte(1)
120		if cfg.Mode == ModeLong64 {
121			vexR = byte(r.Intn(2))
122			vexX = byte(r.Intn(2))
123		}
124		vexB = byte(r.Intn(2))
125		W := byte(r.Intn(2))
126		if insn.Rexw == 1 {
127			W = 1
128		} else if insn.Rexw == -1 {
129			W = 0
130		}
131		L := byte(r.Intn(2))
132		if insn.VexL == 1 {
133			L = 1
134		} else if insn.VexL == -1 {
135			L = 0
136		}
137		pp := byte(r.Intn(4))
138		if insn.VexP != -1 {
139			pp = byte(insn.VexP)
140		}
141		vvvv = 15
142		if !insn.VexNoR {
143			vvvv = byte(r.Intn(16))
144		}
145		code = append(code, vexR<<7|vexX<<6|vexB<<5|insn.VexMap)
146		code = append(code, W<<7|vvvv<<3|L<<2|pp)
147		// TODO: short encoding
148		if cfg.Mode != ModeLong64 {
149			vvvv |= 8
150		}
151	}
152
153	// OPCODE
154	code = append(code, insn.Opcode...)
155
156	if insn.Srm {
157		rm := byte(insn.Rm)
158		if insn.Rm == -1 {
159			rm = byte(r.Intn(8))
160		}
161		code[len(code)-1] |= rm
162	} else if insn.Modrm {
163		// MODRM
164		var mod byte
165		switch insn.Mod {
166		case 0, 1, 2, 3:
167			mod = byte(insn.Mod)
168		case -1:
169			mod = byte(r.Intn(4))
170		case -3:
171			mod = byte(r.Intn(3))
172		}
173
174		reg := byte(insn.Reg)
175		if insn.Reg == -1 {
176			reg = byte(r.Intn(8))
177		} else if insn.Reg == -6 {
178			reg = byte(r.Intn(6)) // segment register
179		} else if insn.Reg == -8 {
180			if rexR {
181				reg = 0 // CR8
182			} else {
183				crs := []byte{0, 2, 3, 4}
184				reg = crs[r.Intn(len(crs))]
185			}
186		}
187		if insn.Avx2Gather {
188			if reg|(1-vexR)<<3 == vvvv^0xf {
189				reg = (reg + 1) & 7
190			}
191		}
192
193		rm := byte(insn.Rm)
194		if insn.Rm == -1 {
195			rm = byte(r.Intn(8))
196		}
197
198		modrm := mod<<6 | reg<<3 | rm
199		code = append(code, modrm)
200
201		if !insn.NoSibDisp {
202			if addrSize == 2 {
203				if mod == 1 {
204					// disp8
205					code = append(code, generateArg(cfg, r, 1)...)
206				} else if mod == 2 || mod == 0 && rm == 6 {
207					// disp16
208					code = append(code, generateArg(cfg, r, 2)...)
209				}
210			} else {
211				var sibbase byte
212				if mod != 3 && rm == 4 {
213					// SIB
214					scale := byte(r.Intn(4))
215					index := byte(r.Intn(8))
216					sibbase = byte(r.Intn(8))
217					if insn.Avx2Gather {
218						rrrr := reg | (1-vexR)<<3
219						for {
220							iiii := index | (1-vexX)<<3
221							if iiii != vvvv^0xf && iiii != rrrr {
222								break
223							}
224							index = (index + 1) & 7
225						}
226					}
227					sib := scale<<6 | index<<3 | sibbase
228					code = append(code, sib)
229				}
230
231				if mod == 1 {
232					// disp8
233					code = append(code, generateArg(cfg, r, 1)...)
234				} else if mod == 2 || mod == 0 && rm == 5 || mod == 0 && sibbase == 5 {
235					// disp16/32
236					code = append(code, generateArg(cfg, r, dispSize)...)
237				}
238			}
239		}
240	}
241
242	addImm := func(imm int) {
243		if imm == -1 {
244			imm = immSize
245		} else if imm == -2 {
246			imm = addrSize
247		} else if imm == -3 {
248			imm = operSize
249		}
250		if imm != 0 {
251			code = append(code, generateArg(cfg, r, imm)...)
252		}
253	}
254	addImm(int(insn.Imm))
255	addImm(int(insn.Imm2))
256
257	code = append(code, insn.Suffix...)
258	return code
259}
260