10 — System Calls¶

System calls (syscalls) are how user-space programs request services from the OS kernel. In assembly, you invoke them directly — no C standard library required.

What is a System Call?¶

When your program needs to do I/O, allocate memory, create processes, or access files, it can't do so directly (Ring 3 has no hardware access). Instead it:

Loads syscall arguments into registers
Executes the syscall instruction
The CPU switches to Ring 0 (kernel mode)
Kernel performs the operation
CPU switches back to Ring 3
Result is in RAX

Linux x86-64 Syscall ABI¶

Role	Register
Syscall number	`RAX`
Argument 1	`RDI`
Argument 2	`RSI`
Argument 3	`RDX`
Argument 4	`R10`
Argument 5	`R8`
Argument 6	`R9`
Return value	`RAX`
Clobbered by kernel	`RCX`, `R11`

Note: The 4th argument is R10, not RCX (unlike the function calling convention). The kernel uses RCX internally.

Essential Linux Syscalls¶

Number	Name	Arguments	Return
0	`read`	`fd`, `buf`, `count`	bytes read
1	`write`	`fd`, `buf`, `count`	bytes written
2	`open`	`path`, `flags`, `mode`	file descriptor
3	`close`	`fd`	0 on success
9	`mmap`	`addr`, `len`, `prot`, `flags`, `fd`, `off`	mapped address
11	`munmap`	`addr`, `len`	0 on success
12	`brk`	`addr`	new break
39	`getpid`	—	PID
57	`fork`	—	0 (child) / PID (parent)
59	`execve`	`path`, `argv`, `envp`	—
60	`exit`	`status`	—
231	`exit_group`	`status`	—

Full list: man 2 syscall, /usr/include/asm/unistd_64.h, or chromium.googlesource.com/chromiumos/docs/+/master/constants/syscalls.md

sys_write — Writing to stdout¶

; write(fd=1, buf=msg, count=len)
mov rax, 1       ; syscall number: sys_write
mov rdi, 1       ; fd: 1 = stdout, 2 = stderr
mov rsi, msg     ; pointer to buffer
mov rdx, len     ; number of bytes
syscall          ; invoke kernel

; rax = number of bytes actually written, or -errno on error

File descriptors: - 0 — stdin - 1 — stdout - 2 — stderr

sys_read — Reading from stdin¶

section .bss
    buf  resb 256    ; 256-byte input buffer

section .text
    ; read(fd=0, buf, count=256)
    mov rax, 0        ; sys_read
    mov rdi, 0        ; stdin
    mov rsi, buf
    mov rdx, 256
    syscall           ; rax = bytes read (including newline)

sys_exit — Terminating the Program¶

mov rax, 60    ; sys_exit
mov rdi, 0     ; exit code (0 = success)
syscall

Or exit_group (231) to exit all threads cleanly:

mov rax, 231
xor rdi, rdi
syscall

sys_open and sys_close¶

section .data
    filename db "/tmp/test.txt", 0    ; null-terminated path

section .text
    ; open(path, O_RDONLY=0)
    mov rax, 2         ; sys_open
    mov rdi, filename
    mov rsi, 0         ; flags: O_RDONLY
    mov rdx, 0         ; mode (ignored for read-only)
    syscall
    ; rax = file descriptor (≥0) or -errno (<0)

    mov r12, rax       ; save fd

    ; read from file...
    mov rax, 0         ; sys_read
    mov rdi, r12       ; our file descriptor
    mov rsi, buf
    mov rdx, 256
    syscall

    ; close(fd)
    mov rax, 3         ; sys_close
    mov rdi, r12
    syscall

Open Flags (from `fcntl.h`)¶

Flag	Value	Meaning
`O_RDONLY`	0	Read only
`O_WRONLY`	1	Write only
`O_RDWR`	2	Read + write
`O_CREAT`	64 (0x40)	Create if not exists
`O_TRUNC`	512 (0x200)	Truncate on open
`O_APPEND`	1024 (0x400)	Append mode

Combine with OR: O_WRONLY | O_CREAT | O_TRUNC = 1 | 64 | 512 = 577

sys_mmap — Memory Mapping¶

; mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
; Returns: pointer to mapped region

mov rax, 9          ; sys_mmap
xor rdi, rdi        ; addr = NULL (OS chooses)
mov rsi, 4096       ; length = 4096 bytes
mov rdx, 3          ; prot = PROT_READ(1) | PROT_WRITE(2)
mov r10, 34         ; flags = MAP_PRIVATE(2) | MAP_ANONYMOUS(32)
mov r8,  -1         ; fd = -1 (anonymous)
xor r9,  r9         ; offset = 0
syscall             ; rax = mapped address or -errno

This is how dynamic memory allocation works at the lowest level.

Error Handling¶

On error, syscalls return a negative errno value in RAX:

syscall
test rax, rax
js   .error         ; if rax < 0, error occurred

; On error: rax = -errno
; neg rax → errno value (e.g., 2 = ENOENT, 13 = EACCES)

Common errno values:

Value	Name	Meaning
1	EPERM	Operation not permitted
2	ENOENT	No such file or directory
9	EBADF	Bad file descriptor
12	ENOMEM	Out of memory
13	EACCES	Permission denied
22	EINVAL	Invalid argument

Complete Example: File Copy¶

; copy.asm — read from stdin, write to stdout (cat behavior)
; Usage: ./copy < input.txt

section .bss
    buf resb 4096

section .text
global _start

_start:
.read_loop:
    ; read(0, buf, 4096)
    mov rax, 0
    xor rdi, rdi       ; stdin
    mov rsi, buf
    mov rdx, 4096
    syscall

    test rax, rax
    jle  .done         ; 0 = EOF, negative = error

    ; write(1, buf, bytes_read)
    mov rdx, rax       ; bytes read = bytes to write
    mov rax, 1
    mov rdi, 1         ; stdout
    mov rsi, buf
    syscall

    jmp .read_loop

.done:
    mov rax, 60
    xor rdi, rdi
    syscall

Build and test:

nasm -f elf64 copy.asm -o copy.o && ld copy.o -o copy
echo "Hello from assembly!" | ./copy

strace — Verify Your Syscalls¶

strace ./copy < /dev/null

Output shows every syscall with arguments and return values. Invaluable for debugging.

Legacy Syscall Interface (int 0x80)¶

The 32-bit syscall interface uses int 0x80 with different registers and numbers. You may see it in old code or shellcode:

; 32-bit exit (do NOT use in 64-bit programs)
mov eax, 1    ; sys_exit (32-bit number)
xor ebx, ebx  ; exit code (arg in EBX, not EDI)
int 0x80

Do not mix int 0x80 with 64-bit code — it uses 32-bit argument truncation.

Key Takeaways¶

Set RAX = syscall number, RDI/RSI/RDX/R10/R8/R9 = arguments, then syscall
Return value is in RAX; negative value = -errno
The kernel clobbers RCX and R11 — save them if needed
Use strace to audit and debug syscall behavior
exit (60) or exit_group (231) to terminate; write (1) for output; read (0) for input