<|start_filename|>include/kernel/console.h<|end_filename|>
#ifndef KERNEL_CONSOLE_H
#define KERNEL_CONSOLE_H

#include <stddef.h>

void console_init(void);
void console_write_char(char ch);
void console_write_str(const char *s);
void console_write(const char *s, size_t count);

#endif


<|start_filename|>include/kernel/cpu.h<|end_filename|>
#ifndef KERNEL_CPU_H
#define KERNEL_CPU_H

/// Halt the current CPU, and wait for an interrupt to wake it up.
void arch_halt_cpu(void);

#endif


<|start_filename|>usr/liblinux/include/arpa/inet.h<|end_filename|>
#ifndef __ARPA_INET_H
#define __ARPA_INET_H

#include <netinet/in.h>
#include <inttypes.h>

static inline uint32_t htonl(uint32_t hostlong)
{
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	return hostlong;
#else
	return __builtin_bswap32(hostlong);
#endif
}

static inline uint16_t htons(uint16_t hostshort)
{
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	return hostshort;
#else
	return __builtin_bswap16(hostshort);
#endif
}

static inline uint32_t ntohl(uint32_t netlong)
{
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	return netlong;
#else
	return __builtin_bswap16(netlong);
#endif
}

static inline uint16_t ntohs(uint16_t netshort)
{
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	return netshort;
#else
	return __builtin_bswap16(netshort);
#endif
}

in_addr_t inet_addr(const char *cp);

#endif


<|start_filename|>usr/liblinux/src/internal/socket.h<|end_filename|>
#ifndef __LIBLINUX_INTERNAL_SOCKET_H
#define __LIBLINUX_INTERNAL_SOCKET_H

#include <stddef.h>
#include <stdint.h>

#include <sys/socket.h>

struct packet_view;
struct socket;

struct socket_operations {
	ssize_t (*recvfrom)(struct socket *sk, void *restrict buf, size_t len, int flags,
			    struct sockaddr *restrict src_addr, socklen_t *restrict addrlen);
	ssize_t (*sendto)(struct socket *sk, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
			  socklen_t addrlen);
};

struct socket {
	const struct socket_operations *ops;
	uint16_t local_port;
	char rx_buffer[1500]; /* FIXME make bigger */
};

int socket_alloc(int domain, int type, int protocol);

struct socket *socket_lookup_by_fd(int sockfd);
struct socket *socket_lookup_by_flow(uint16_t local_port, uint16_t foreign_port);

void socket_input(struct socket *sk, struct packet_view *pk);

int socket_accept(struct socket *sk, struct sockaddr *restrict addr, socklen_t *restrict addrlen);
int socket_bind(struct socket *sk, const struct sockaddr *addr, socklen_t addrlen);
int socket_getsockopt(struct socket *sk, int level, int optname, void *restrict optval, socklen_t *restrict optlen);
int socket_listen(struct socket *sk, int backlog);
ssize_t socket_recvfrom(struct socket *sk, void *restrict buf, size_t len, int flags,
			struct sockaddr *restrict src_addr, socklen_t *restrict addrlen);
ssize_t socket_sendto(struct socket *sk, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
		      socklen_t addrlen);
int socket_setsockopt(struct socket *sk, int level, int optname, const void *optval, socklen_t optlen);
int socket_shutdown(struct socket *sk, int how);

#endif


<|start_filename|>usr/libmanticore/src/syscalls/console_print.c<|end_filename|>
#include <manticore/syscalls.h>

ssize_t console_print(const char *text, size_t count)
{
	return syscall2(SYS_console_print, (unsigned long) text, count);
}


<|start_filename|>usr/liblinux/src/socket.c<|end_filename|>
#include <sys/socket.h>

#include <errno.h>
#include <netinet/in.h>
#include <stddef.h>
#include <sys/types.h>

#include "internal/socket.h"

int accept(int sockfd, struct sockaddr *restrict addr, socklen_t *restrict addrlen)
{
	struct socket *sk = socket_lookup_by_fd(sockfd);
	if (!sk) {
		errno = EBADF;
		return -1;
	}
	return socket_accept(sk, addr, addrlen);
}

int bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen)
{
	struct socket *sk = socket_lookup_by_fd(sockfd);
	if (!sk) {
		errno = EBADF;
		return -1;
	}
	return socket_bind(sk, addr, addrlen);
}

int getsockopt(int sockfd, int level, int optname, void *restrict optval, socklen_t *restrict optlen)
{
	struct socket *sk = socket_lookup_by_fd(sockfd);
	if (!sk) {
		errno = EBADF;
		return -1;
	}
	return socket_getsockopt(sk, level, optname, optval, optlen);
}

int listen(int sockfd, int backlog)
{
	struct socket *sk = socket_lookup_by_fd(sockfd);
	if (!sk) {
		errno = EBADF;
		return -1;
	}
	return socket_listen(sk, backlog);
}

ssize_t recv(int sockfd, void *buf, size_t len, int flags)
{
	return recvfrom(sockfd, buf, len, flags, NULL, NULL);
}

ssize_t recvfrom(int sockfd, void *restrict buf, size_t len, int flags, struct sockaddr *restrict src_addr, socklen_t *restrict addrlen)
{
	struct socket *sk = socket_lookup_by_fd(sockfd);
	if (!sk) {
		errno = EBADF;
		return -1;
	}
	return socket_recvfrom(sk, buf, len, flags, src_addr, addrlen);
}

ssize_t send(int sockfd, const void *buf, size_t len, int flags)
{
	return sendto(sockfd, buf, len, flags, NULL, 0);
}

ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen)
{
	struct socket *sk = socket_lookup_by_fd(sockfd);
	if (!sk) {
		errno = EBADF;
		return -1;
	}
	return socket_sendto(sk, buf, len, flags, dest_addr, addrlen);
}

int setsockopt(int sockfd, int level, int optname, const void *optval, socklen_t optlen)
{
	struct socket *sk = socket_lookup_by_fd(sockfd);
	if (!sk) {
		errno = EBADF;
		return -1;
	}
	return socket_setsockopt(sk, level, optname, optval, optlen);
}

int shutdown(int sockfd, int how)
{
	struct socket *sk = socket_lookup_by_fd(sockfd);
	if (!sk) {
		errno = EBADF;
		return -1;
	}
	return socket_shutdown(sk, how);
}

int socket(int domain, int type, int protocol)
{
	return socket_alloc(domain, type, protocol);
}


<|start_filename|>usr/liblinux/include/string.h<|end_filename|>
#ifndef __STRING_H
#define __STRING_H

#include <stddef.h>

void *memset(void *s, int c, size_t len);
void *memcpy(void *dst, const void *src, size_t n);
char *strerror(int errnum);
size_t strlen(const char *s);

#endif


<|start_filename|>include/uapi/manticore/io_queue_abi.h<|end_filename|>
#ifndef __MANTICORE_UAPI_IO_QUEUE_H
#define __MANTICORE_UAPI_IO_QUEUE_H

#include <stddef.h>
#include <stdint.h>

typedef void *io_queue_t;

enum io_opcode {
	IO_OPCODE_SUBMIT = 0x1,
	IO_OPCODE_COMPLETE = 0x2,
};

struct io_cmd {
	uint32_t opcode;
	void *addr;
	size_t len;
};

#endif


<|start_filename|>include/kernel/mmu.h<|end_filename|>
#ifndef KERNEL_MMU_H
#define KERNEL_MMU_H 1

#include <arch/mmu.h>
#include <arch/vmem.h>

#include <stddef.h>
#include <stdint.h>

typedef enum {
	MMU_PROT_READ	= 1UL << 0,
	MMU_PROT_WRITE	= 1UL << 1,
	MMU_PROT_EXEC	= 1UL << 2,
} mmu_prot_t;

typedef enum {
	MMU_USER_PAGE = 1UL << 0,
	MMU_NOCACHE = 1UL << 1,
} mmu_flags_t;

void mmu_invalidate_tlb(void);
mmu_map_t mmu_current_map(void);
void mmu_load_map(mmu_map_t map);
int mmu_map_range(mmu_map_t map, virt_t vaddr, phys_t paddr, size_t size, mmu_prot_t prot, mmu_flags_t flags);
int mmu_map_small_page(mmu_map_t map, virt_t vaddr, phys_t paddr, mmu_prot_t prot, mmu_flags_t flags);
int mmu_map_large_page(mmu_map_t map, virt_t vaddr, phys_t paddr, mmu_prot_t prot, mmu_flags_t flags);
void mmu_map_dump(mmu_map_t map);

#endif


<|start_filename|>include/kernel/page-fault.h<|end_filename|>
#ifndef KERNEL_PAGE_FAULT_H
#define KERNEL_PAGE_FAULTH

void page_fault_set_fixup(void *);
void *page_fault_get_fixup(void);

#endif


<|start_filename|>include/kernel/panic.h<|end_filename|>
#ifndef KERNEL_PANIC_H
#define KERNEL_PANIC_H

void panic(const char *fmt, ...) __attribute__((__format__(__printf__, 1, 2)));

#endif


<|start_filename|>usr/libmanticore/src/syscall.c<|end_filename|>
#include <manticore/syscalls.h>

long syscall0(long number)
{
        unsigned long ret;
        asm volatile(
		"syscall"
		: "=a"(ret)
		: "a"(number)
		: "rcx", "r11", "memory");
        return ret;
}

long syscall1(long number, long arg0)
{
        unsigned long ret;
        asm volatile(
		"syscall"
		: "=a"(ret)
		: "a"(number), "D"(arg0)
		: "rcx", "r11", "memory");
        return ret;
}

long syscall2(long number, long arg0, long arg1)
{
        unsigned long ret;
        asm volatile(
		"syscall"
		: "=a"(ret)
		: "a"(number), "D"(arg0), "S"(arg1)
		: "rcx", "r11", "memory");
        return ret;
}

long syscall4(long number, long arg0, long arg1, long arg2, long arg3)
{
        unsigned long ret;
        asm volatile(
		"syscall"
		: "=a"(ret)
		: "a"(number), "D"(arg0), "S"(arg1), "d"(arg2), "c"(arg3)
		: "r11", "memory");
        return ret;
}


<|start_filename|>usr/liblinux/include/netinet/ip.h<|end_filename|>
#ifndef __LIBLINUX_NETINET_IP_H
#define __LIBLINUX_NETINET_IP_H

#include <stdint.h>

#define IP_RF 0x8000U	   /* Reserved Flag (RF) */
#define IP_DF 0x4000U	   /* Don't Fragment (DF) */
#define IP_MF 0x2000U	   /* More Fragments (MF) */
#define IP_OFFMASK 0x1fffU /* Fragment offset mask */

struct iphdr {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
	unsigned ihl : 4;
	unsigned version : 4;
#else
	unsigned version : 4;
	unsigned ihl : 4;
#endif
	uint8_t tos;
	uint16_t tot_len;
	uint16_t id;
	uint16_t frag_off;
	uint8_t ttl;
	uint8_t protocol;
	uint16_t check;
	uint32_t saddr;
	uint32_t daddr;
};

#endif


<|start_filename|>usr/liblinux/include/stdlib.h<|end_filename|>
#ifndef __STDLIB_H
#define __STDLIB_H

#include <stddef.h>

void *malloc(size_t size);
void free(void *ptr);

void abort(void);

void exit(int status);

#endif


<|start_filename|>include/uapi/manticore/atomic_ring_buffer_abi.h<|end_filename|>
#ifndef __MANTICORE_UAPI_ATOMIC_RING_BUFFER_ABI_H
#define __MANTICORE_UAPI_ATOMIC_RING_BUFFER_ABI_H

#include <stdatomic.h>
#include <stddef.h>
#include <stdint.h>

// Alignment for atomic ring buffer metadata elements (e.g. head and tail
// pointers). Basically, aligning to L1 cache line size to avoid false sharing.
#define ATOMIC_RING_BUFFER_ALIGN 64

// An atomic ring buffer.
struct atomic_ring_buffer {
	size_t element_size;
	size_t capacity;
	char header_padding[ATOMIC_RING_BUFFER_ALIGN - sizeof(size_t) - sizeof(size_t)];
	atomic_ullong tail;
	char tail_pad[ATOMIC_RING_BUFFER_ALIGN - sizeof(atomic_ullong)];
	atomic_ullong head;
	char head_pad[ATOMIC_RING_BUFFER_ALIGN - sizeof(atomic_ullong)];
	char data[0];
};

#endif


<|start_filename|>include/kernel/memory.h<|end_filename|>
#ifndef KERNEL_MEMORY_H
#define KERNEL_MEMORY_H

#include <arch/vmem.h>

#include <stddef.h>
#include <stdint.h>

#define MAX_MEM_REGIONS 8

struct memory_region {
	uint64_t base;
	uint64_t len;
};

extern struct memory_region mem_regions[MAX_MEM_REGIONS];
extern size_t nr_mem_regions;
extern virt_t kernel_vm_end;

void memory_add_span(uint64_t addr, uint64_t size);

#endif


<|start_filename|>include/kernel/align.h<|end_filename|>
#ifndef KERNEL_ALIGN_H
#define KERNEL_ALIGN_H

#include <stdbool.h>
#include <stdint.h>

/// Check if \size is aligned to \align.
///
/// \param size  The size to check for alignment.
/// \param align The alignment to check for.
/// \returns true if \size is aligned to \align; otherwise returns false.
static inline bool is_aligned(uint64_t size, uint64_t align)
{
	return (size % align) == 0;
}

static inline uint64_t align_down(uint64_t size, uint64_t align)
{
       return size & ~(align - 1);
}

static inline uint64_t align_up(uint64_t size, uint64_t align)
{
       return align_down(size + align - 1, align);
}

#endif


<|start_filename|>arch/aarch64/include/arch/vmem-defs.h<|end_filename|>
#ifndef __MANTICORE_AARCH64_VMEM_DEFS_H
#define __MANTICORE_AARCH64_VMEM_DEFS_H

/// Kernel virtual memory base address that maps to physical address zero.
#define KERNEL_VMA 0xffff000000000000ULL

#endif


<|start_filename|>Makefile<|end_filename|>
ARCH ?= $(shell uname -m)

# Make the build silent by default
V =
ifeq ($(strip $(V)),)
	E = @echo
	Q = @
else
	E = @\#
	Q =
endif
export E Q

include arch/$(ARCH)/Makefile
include lib/libc/Makefile

LD := ld.bfd

#
# The filename of the kernel image. This is the main output of the build
# system, which contains an ELF image of the operating system kernel.
#
KERNEL_IMAGE = kernel.elf

includes += -include include/kernel/kernel.h -Iinclude

objs += kernel/atomic-ring-buffer.o
objs += kernel/init.o
objs += kernel/initrd.o
objs += kernel/panic.o
objs += kernel/printf.o
objs += kernel/syscall.o
objs += kernel/thread.o
objs += kernel/user-copy.o
objs += mm/kmem.o
objs += mm/mmu.o

#
# The filename of the kernel static library. This static library is the output
# of the `cargo` build of kernel Rust source code. The library is linked to
# the built C source code to produce the final kernel image.
#
KERNEL_LIB = target/$(ARCH)-unknown-none/release/libmanticore.a

#
# The source files for the kernel static library. Although `cargo` manages the
# build of the static library, we need to define Rust source files for `make`
# to calculate dependencies correctly to execute `cargo` when sources change.
#
KERNEL_LIB_SRC += drivers/pci/lib.rs
KERNEL_LIB_SRC += drivers/virtio/lib.rs
KERNEL_LIB_SRC += drivers/virtio/net.rs
KERNEL_LIB_SRC += drivers/virtio/virtqueue.rs
KERNEL_LIB_SRC += kernel/atomic_ring_buffer.rs
KERNEL_LIB_SRC += kernel/device.rs
KERNEL_LIB_SRC += kernel/errno.rs
KERNEL_LIB_SRC += kernel/event.rs
KERNEL_LIB_SRC += kernel/ioport.rs
KERNEL_LIB_SRC += kernel/ioqueue.rs
KERNEL_LIB_SRC += kernel/lib.rs
KERNEL_LIB_SRC += kernel/memory.rs
KERNEL_LIB_SRC += kernel/mmu.rs
KERNEL_LIB_SRC += kernel/print.rs
KERNEL_LIB_SRC += kernel/process.rs
KERNEL_LIB_SRC += kernel/sched.rs
KERNEL_LIB_SRC += kernel/vm.rs
KERNEL_LIB_SRC += manticore.rs

#
# The source code to manual pages.
#
MAN_PAGES += man/exit.txt
MAN_PAGES += man/get_config.txt
MAN_PAGES += man/vmspace_alloc.txt
MAN_PAGES += man/wait.txt

ifdef TEST
CFLAGS += -DHAVE_TEST
tests += tests/tst-kmem.o
tests += tests/tst-page-alloc.o
tests += tests/tst-printf.o
endif

WARNINGS = -Wall -Wextra -Wno-unused-parameter
CFLAGS += -std=gnu11 -O3 -g $(WARNINGS) -ffreestanding $(includes) -fno-pie -fno-stack-protector
ASFLAGS += -D__ASSEMBLY__ $(includes)
LDFLAGS += --gc-sections

DEPS=.deps
$(objs): | $(DEPS)
$(DEPS):
	$(Q) mkdir -p $(DEPS)

all: $(KERNEL_IMAGE) usr/echod/echod.iso

$(KERNEL_IMAGE): arch/$(ARCH)/kernel.ld $(objs) $(KERNEL_LIB) $(tests)
	$(E) "  LD      " $@
	$(Q) $(CROSS_PREFIX)$(LD) $(LDFLAGS) -Tarch/$(ARCH)/kernel.ld $(objs) $(KERNEL_LIB) $(tests) -o $@ -Ltarget/$(ARCH)-unknown-none/release -lmanticore

$(KERNEL_LIB): $(KERNEL_LIB_SRC)
	$(E) "  XARGO"
	$(Q) CC=$(CROSS_PREFIX)gcc RUST_TARGET_PATH=$(PWD) xargo build --release --target $(ARCH)-unknown-none

%.o: %.c
	$(E) "  CC      " $@
	$(Q) $(CROSS_PREFIX)gcc $(CFLAGS) -MD -c -o $@ $< -MF $(DEPS)/$(notdir $*).d

%.o: %.S
	$(E) "  AS      " $@
	$(Q) $(CROSS_PREFIX)gcc $(ASFLAGS) -MD -c $< -o $@ -MF $(DEPS)/$(notdir $*).d

%.ld: %.ld.S
	$(E) "  CPP     " $@
	$(Q) $(CROSS_PREFIX)cpp $(CFLAGS) -P $< $@

usr/echod/echod.iso: $(KERNEL_IMAGE)
	$(E) "  MAKE    " $@
	$(Q) make -s -C usr/echod
.PHONY: usr/echod/echod.iso

man: $(MAN_PAGES)
	$(E) "  ASCIIDOCTOR"
	$(Q) asciidoctor -b manpage $?
.PHONY: man

clean: archclean
	$(E) "  CLEAN"
	$(Q) rm -f $(KERNEL_IMAGE) $(objs) $(tests)
	$(Q) rm -f arch/$(ARCH)/kernel.ld
	$(Q) rm -rf target
	$(Q) rm -rf $(DEPS)
	$(Q) make --silent -C usr/echod clean

.PHONY: all clean

-include $(DEPS)/*.d


<|start_filename|>usr/libmanticore/include/manticore/types.h<|end_filename|>
#ifndef MANTICORE_TYPES_H
#define MANTICORE_TYPES_H

typedef long ssize_t;

#endif


<|start_filename|>arch/x86_64/mmu.c<|end_filename|>
#include <kernel/mmu.h>

#include <kernel/align.h>
#include <kernel/errno.h>
#include <kernel/page-alloc.h>
#include <kernel/printf.h>

#include <arch/processor.h>
#include <arch/vmem.h>

#include <string.h>

phys_t virt_to_phys(virt_t addr)
{
	return addr - KERNEL_VMA;
}

virt_t phys_to_virt(phys_t addr)
{
	return addr + KERNEL_VMA;
}

// Entries in paging structure:
#define NR_PG_ENTRIES 512
// Bits used for translation per level:
#define PG_TABLE_BITS 9
// Mask for translation per level:
#define PG_INDEX_MASK ((1ULL << PG_TABLE_BITS) - 1)

// Page Map Level 4 (PML4)
#define PML4_INDEX_SHIFT 39
#define PML4_INDEX_MASK PG_INDEX_MASK
// Page Directory Pointer Table (PDPT)
#define PDPT_INDEX_SHIFT 30
#define PDPT_INDEX_MASK PG_INDEX_MASK
// Page Directory (PD)
#define PD_INDEX_SHIFT 21
#define PD_INDEX_MASK PG_INDEX_MASK
// Page Table (PT)
#define PT_INDEX_SHIFT 12
#define PT_INDEX_MASK PG_INDEX_MASK

// Page Map Level 4 Entry (PML4E)
#define PML4E_FLAGS_MASK 0x80000000000007ffULL
#define PML4E_PADDR_MASK (~PML4E_FLAGS_MASK)
// Page Directory Pointer Table Entry (PDPTE)
#define PDPTE_FLAGS_MASK 0x80000000000007ffULL
#define PDPTE_PADDR_MASK (~PDPTE_FLAGS_MASK)
// Page Directory Entry (PDE)
#define PDE_FLAGS_MASK 0x80000000000007ffULL
#define PDE_PADDR_MASK (~PDE_FLAGS_MASK)
// Page Table Entry (PTE)
#define PTE_FLAGS_MASK 0x80000000000007ffULL
#define PTE_PADDR_MASK (~PTE_FLAGS_MASK)

typedef struct {
	uint64_t pml4e;
} pml4e_t;

typedef struct {
	uint64_t pdpte;
} pdpte_t;

typedef struct {
	uint64_t pde;
} pde_t;

typedef struct {
	uint64_t pte;
} pte_t;

static pml4e_t make_pml4e(phys_t paddr, uint64_t flags)
{
	return (pml4e_t){pml4e: paddr | flags};
}

static pdpte_t make_pdpte(phys_t paddr, uint64_t flags)
{
	return (pdpte_t){pdpte: paddr | flags};
}

static pde_t make_pde(phys_t paddr, uint64_t flags)
{
	return (pde_t){pde: paddr | flags};
}

static pte_t make_pte(phys_t paddr, uint64_t flags)
{
	return (pte_t){pte: paddr | flags};
}

static bool pml4e_is_none(pml4e_t pml4e)
{
	return !pml4e.pml4e;
}

static bool pdpte_is_none(pdpte_t pdpte)
{
	return !pdpte.pdpte;
}

static bool pde_is_none(pde_t pde)
{
	return !pde.pde;
}

static bool pte_is_none(pte_t pte)
{
	return !pte.pte;
}

static bool pde_is_large(pde_t pde)
{
	return pde.pde & X86_PE_PS;
}

static uint64_t pml4e_paddr(pml4e_t pml4e)
{
	return pml4e.pml4e & PML4E_PADDR_MASK;
}

static uint64_t pdpte_paddr(pdpte_t pdpte)
{
	return pdpte.pdpte & PDPTE_PADDR_MASK;
}

static uint64_t pde_paddr(pde_t pde)
{
	return pde.pde & PDE_PADDR_MASK;
}

static uint64_t pte_paddr(pte_t pte)
{
	return pte.pte & PTE_PADDR_MASK;
}

static uint64_t pml4e_flags(pml4e_t pml4e)
{
	return pml4e.pml4e & PML4E_FLAGS_MASK;
}

static uint64_t pdpte_flags(pdpte_t pdpte)
{
	return pdpte.pdpte & PDPTE_FLAGS_MASK;
}

static uint64_t pde_flags(pde_t pde)
{
	return pde.pde & PDE_FLAGS_MASK;
}

static uint64_t pte_flags(pte_t pte)
{
	return pte.pte & PTE_FLAGS_MASK;
}

static uint64_t x86_read_cr3(void)
{
	uint64_t ret;
	asm volatile(
		"mov %%cr3, %0"
		: "=r"(ret));
	return ret;
}

static void x86_write_cr3(uint64_t value)
{
	asm volatile(
		"mov %0, %%cr3"
		:
		: "r"(value));
}

void mmu_invalidate_tlb(void)
{
	x86_write_cr3(x86_read_cr3());
}

mmu_map_t mmu_current_map(void)
{
	return (mmu_map_t){cr3: x86_read_cr3()};
}

void mmu_load_map(mmu_map_t map)
{
	x86_write_cr3(map.cr3);
}

/// Converts paging structure indices to a virtual address.
static virt_t pg_index_to_vaddr(uint64_t pml4_idx, uint64_t pdpt_idx, uint64_t pd_idx,
				uint64_t pt_idx)
{
	return (pml4_idx << PML4_INDEX_SHIFT) | (pdpt_idx << PDPT_INDEX_SHIFT) |
	       (pd_idx << PD_INDEX_SHIFT) | (pt_idx << PT_INDEX_SHIFT);
}

static uint64_t mmu_prot_to_hw(mmu_prot_t prot)
{
	uint64_t hw_flags = X86_PE_P;
	if (prot & MMU_PROT_WRITE) {
		hw_flags |= X86_PE_RW;
	}
	if (!(prot & MMU_PROT_EXEC)) {
		hw_flags |= X86_PE_XD;
	}
	return hw_flags;
}

static uint64_t mmu_flags_to_hw(mmu_flags_t flags)
{
	uint64_t hw_flags = 0;
	if (flags & MMU_USER_PAGE) {
		hw_flags |= X86_PE_US;
	}
	if (flags & MMU_NOCACHE) {
		hw_flags |= X86_PE_PCD;
	}
	return hw_flags;
}

int mmu_map_small_page(mmu_map_t map, virt_t vaddr, phys_t paddr, mmu_prot_t prot, mmu_flags_t flags)
{
	uint64_t hw_flags = mmu_flags_to_hw(flags);
	pml4e_t *pml4_table = paddr_to_ptr(map.cr3);
	uint64_t pml4_idx = (vaddr >> PML4_INDEX_SHIFT) & PML4_INDEX_MASK;
	pml4e_t pml4e = pml4_table[pml4_idx];
	if (pml4e_is_none(pml4e)) {
		void *pdp_page = page_alloc_small();
		if (!pdp_page) {
			return -ENOMEM;
		}
		memset(pdp_page, 0, PAGE_SIZE_SMALL);
		pml4e = make_pml4e(ptr_to_paddr(pdp_page), X86_PE_P | X86_PE_RW | hw_flags);
		pml4_table[pml4_idx] = pml4e;
	}
	pdpte_t *pdp_table = paddr_to_ptr(pml4e_paddr(pml4e));
	uint64_t pdpt_idx = (vaddr >> PDPT_INDEX_SHIFT) & PDPT_INDEX_MASK;
	pdpte_t pdpte = pdp_table[pdpt_idx];
	if (pdpte_is_none(pdpte)) {
		void *pd_page = page_alloc_small();
		if (!pd_page) {
			return -ENOMEM;
		}
		memset(pd_page, 0, PAGE_SIZE_SMALL);
		pdpte = make_pdpte(ptr_to_paddr(pd_page), X86_PE_P | X86_PE_RW | hw_flags);
		pdp_table[pdpt_idx] = pdpte;
	}
	pde_t *pd = paddr_to_ptr(pdpte_paddr(pdpte));
	uint64_t pd_idx = (vaddr >> PD_INDEX_SHIFT) & PD_INDEX_MASK;
	pde_t pde = pd[pd_idx];
	if (pde_is_none(pde)) {
		void *pt_page = page_alloc_small();
		if (!pt_page) {
			return -ENOMEM;
		}
		memset(pt_page, 0, PAGE_SIZE_SMALL);
		pde = make_pde(ptr_to_paddr(pt_page), X86_PE_P | X86_PE_RW | hw_flags);
		pd[pd_idx] = pde;
	}
	if (pde_is_large(pde)) {
		/* PDE is already mapped as a large page. */
		return -EINVAL;
	}
	uint64_t hw_prot = mmu_prot_to_hw(prot);
	pte_t *pt = paddr_to_ptr(pde_paddr(pde));
	uint64_t pt_idx = (vaddr >> PT_INDEX_SHIFT) & PT_INDEX_MASK;
	pt[pt_idx] = make_pte(paddr, hw_prot | hw_flags);
	return 0;
}

int mmu_map_large_page(mmu_map_t map, virt_t vaddr, phys_t paddr, mmu_prot_t prot, mmu_flags_t flags)
{
	uint64_t hw_flags = mmu_flags_to_hw(flags);
	pml4e_t *pml4_table = paddr_to_ptr(map.cr3);
	uint64_t pml4_idx = (vaddr >> PML4_INDEX_SHIFT) & PML4_INDEX_MASK;
	pml4e_t pml4e = pml4_table[pml4_idx];
	if (pml4e_is_none(pml4e)) {
		void *pdp_page = page_alloc_small();
		if (!pdp_page) {
			return -ENOMEM;
		}
		memset(pdp_page, 0, PAGE_SIZE_SMALL);
		pml4e = make_pml4e(ptr_to_paddr(pdp_page), X86_PE_P | X86_PE_RW | hw_flags);
		pml4_table[pml4_idx] = pml4e;
	}
	pdpte_t *pdp_table = paddr_to_ptr(pml4e_paddr(pml4e));
	uint64_t pdpt_idx = (vaddr >> PDPT_INDEX_SHIFT) & PDPT_INDEX_MASK;
	pdpte_t pdpte = pdp_table[pdpt_idx];
	if (pdpte_is_none(pdpte)) {
		void *pd_page = page_alloc_small();
		if (!pd_page) {
			return -ENOMEM;
		}
		memset(pd_page, 0, PAGE_SIZE_SMALL);
		pdpte = make_pdpte(ptr_to_paddr(pd_page), X86_PE_P | X86_PE_RW | hw_flags);
		pdp_table[pdpt_idx] = pdpte;
	}
	pde_t *pd = paddr_to_ptr(pdpte_paddr(pdpte));
	uint64_t pd_idx = (vaddr >> PD_INDEX_SHIFT) & PD_INDEX_MASK;
	pde_t pde = pd[pd_idx];
	if (!pde_is_none(pde) && !pde_is_large(pde)) {
		/* PDE is already mapped as a page table. */
		return -EINVAL;
	}
	uint64_t hw_prot = mmu_prot_to_hw(prot);
	pd[pd_idx] = make_pde(paddr, hw_prot | hw_flags | X86_PE_PS);
	return 0;
}

static void mmu_dump_pde(unsigned pml4_idx, unsigned pdpt_idx, unsigned pd_idx, pde_t pde)
{
	if (pde_is_large(pde)) {
		printf("      PDE: %4d %016lx -> %016lx [%lx]\n", pd_idx,
		       pg_index_to_vaddr(pml4_idx, pdpt_idx, pd_idx, 0), pde_paddr(pde),
		       pde_flags(pde));
	} else {
		printf("      PDE: %d @ %016lx [%lx]\n", pd_idx, pde_paddr(pde), pde_flags(pde));

		pte_t *pt = paddr_to_ptr(pde_paddr(pde));
		for (unsigned pt_idx = 0; pt_idx < NR_PG_ENTRIES; pt_idx++) {
			pte_t pte = pt[pt_idx];
			if (pte_is_none(pte)) {
				continue;
			}
			printf("      PTE: %4d %016lx -> %016lx [%lx]\n", pd_idx,
			       pg_index_to_vaddr(pml4_idx, pdpt_idx, pd_idx, pt_idx),
			       pte_paddr(pte), pte_flags(pte));
		}
	}
}

static void mmu_dump_pdpte(unsigned pml4_idx, unsigned pdpt_idx, pdpte_t pdpte)
{
	printf("    PDPTE: %d @ %016lx [%lx]\n", pdpt_idx, pdpte_paddr(pdpte), pdpte_flags(pdpte));

	pde_t *pd = paddr_to_ptr(pdpte_paddr(pdpte));
	for (unsigned pd_idx = 0; pd_idx < NR_PG_ENTRIES; pd_idx++) {
		pde_t pde = pd[pd_idx];
		if (pde_is_none(pde)) {
			continue;
		}
		mmu_dump_pde(pml4_idx, pdpt_idx, pd_idx, pde);
	}
}

static void mmu_dump_pml4e(unsigned pml4_idx, pml4e_t pml4e)
{
	printf("  PML4E: %d @ %016lx [%lx]\n", pml4_idx, pml4e_paddr(pml4e), pml4e_flags(pml4e));

	pdpte_t *pdp_table = paddr_to_ptr(pml4e_paddr(pml4e));
	for (unsigned pdpt_idx = 0; pdpt_idx < NR_PG_ENTRIES; pdpt_idx++) {
		pdpte_t pdpte = pdp_table[pdpt_idx];
		if (pdpte_is_none(pdpte)) {
			continue;
		}
		mmu_dump_pdpte(pml4_idx, pdpt_idx, pdpte);
	}
}

void mmu_map_dump(mmu_map_t map)
{
	pml4e_t *pml4_table = paddr_to_ptr(map.cr3);
	printf("PML4 table: %p\n", pml4_table);

	for (unsigned pml4_idx = 0; pml4_idx < NR_PG_ENTRIES; pml4_idx++) {
		pml4e_t pml4e = pml4_table[pml4_idx];
		if (pml4e_is_none(pml4e)) {
			continue;
		}
		mmu_dump_pml4e(pml4_idx, pml4e);
	}
}


<|start_filename|>usr/libmanticore/src/atomic-ring-buffer.c<|end_filename|>
#include <manticore/atomic-ring-buffer.h>

#include <stdatomic.h>
#include <stddef.h>
#include <string.h>

bool atomic_ring_buffer_is_empty(struct atomic_ring_buffer *queue)
{
	uint64_t head = atomic_load_explicit(&queue->head, memory_order_acquire);
	uint64_t tail = atomic_load_explicit(&queue->tail, memory_order_acquire);
	return head == tail;
}

void *atomic_ring_buffer_front(struct atomic_ring_buffer *queue)
{
	uint64_t head = atomic_load_explicit(&queue->head, memory_order_acquire);
	uint64_t tail = atomic_load_explicit(&queue->tail, memory_order_relaxed);
	if (head == tail) {
		return NULL;
	}
	return (void *)queue->data + tail;
}

void atomic_ring_buffer_pop(struct atomic_ring_buffer *queue)
{
	unsigned long tail = atomic_load_explicit(&queue->tail, memory_order_relaxed);
	unsigned long next_tail = tail + queue->element_size;
	if (next_tail == queue->capacity) {
		next_tail = 0;
	}
	atomic_store_explicit(&queue->tail, next_tail, memory_order_release);
}

/* The libmanticore library has no dependency to libc, so we need to implement
   memcpy() ourselves.  Please consider this as an official apology to whoever
   discovers this to be a performance bottleneck for kernel and user space
   communication.  */
static void *memcpy_internal(void *dest, const void *src, size_t n)
{
       char *d = dest;
       const char *s = src;
       for (int i = 0; i < n; i++) {
               *d++ = *s++;
       }
}

bool atomic_ring_buffer_emplace(struct atomic_ring_buffer *queue, void *element)
{
	uint64_t head = atomic_load_explicit(&queue->head, memory_order_relaxed);
	uint64_t next_head = head + queue->element_size;
	if (next_head == queue->capacity) {
		next_head = 0;
	}
	if (next_head == atomic_load_explicit(&queue->tail, memory_order_acquire)) {
		return false;
	}
	memcpy_internal(queue->data + head, element, queue->element_size);
	atomic_store_explicit(&queue->head, next_head, memory_order_release);
	return true;
}


<|start_filename|>usr/liblinux/src/internal/arp.c<|end_filename|>
#include "internal/net.h"

#include "internal/arp_cache.h"
#include "internal/setup.h"
#include "internal/trace.h"

#include <arpa/inet.h>
#include <linux/if_ether.h>
#include <linux/if_arp.h>
#include <manticore/config_abi.h>
#include <manticore/io_queue.h>
#include <manticore/syscalls.h>

#include <string.h>

struct arp_ip4 {
	/// MAC address of the sender of this ARP packet.
	uint8_t ar_smac[ETH_ALEN];
	/// IP address of the sender of this ARP packet.
	uint32_t ar_sip;
	/// MAC address of the sender of this ARP packet.
	unsigned char ar_dmac[ETH_ALEN];
	uint32_t ar_dip;
} __attribute__((packed));

static void arp_reply(struct arphdr *request_arph)
{
	LIBLINUX_TRACE(arp_reply);

	// FIXME: Use some generic TX buffer region
	static char tx_buf[128];

	struct arp_ip4 *request_arp = (void *)request_arph + sizeof(*request_arph);
	struct ethhdr *reply_ethh = (void *) tx_buf;
	struct arphdr *reply_arph = (void *) tx_buf + sizeof(*reply_ethh);
	struct arp_ip4 *reply_arp = (void *) tx_buf + sizeof(*reply_ethh) + sizeof(*reply_arph);

	arp_cache_insert(&__liblinux_arp_cache, request_arp->ar_sip, request_arp->ar_smac);

	memcpy(reply_ethh->h_dest, request_arp->ar_smac, ETH_ALEN);
	memcpy(reply_ethh->h_source, __liblinux_mac_addr, ETH_ALEN);
	reply_ethh->h_proto = htons(ETH_P_ARP);
	reply_arph->ar_hrd = htons(ARPHRD_ETHER);
	reply_arph->ar_pro = htons(ETH_P_IP);
	reply_arph->ar_hln = ETH_ALEN;
	reply_arph->ar_pln = 4;
	reply_arph->ar_op = htons(ARPOP_REPLY);

	memcpy(reply_arp->ar_smac, __liblinux_mac_addr, ETH_ALEN);
	reply_arp->ar_sip = __liblinux_host_ip;

	memcpy(reply_arp->ar_dmac, request_arp->ar_smac, ETH_ALEN);
	reply_arp->ar_dip = request_arp->ar_sip;

	// FIXME: io_submit() error handling
	io_submit(__liblinux_eth_ioqueue, reply_ethh, sizeof(*reply_ethh) + sizeof(*reply_arph) + sizeof(*reply_arp));
}

void arp_input(struct packet_view *pk)
{
	LIBLINUX_TRACE(arp_input);

	struct arphdr *arph = pk->start;

	switch (ntohs(arph->ar_op)) {
	case ARPOP_REQUEST:
		arp_reply(arph);
		break;
	default:
		break;
	}

	// FIXME: Notify OS that packet is consumed.
}


<|start_filename|>kernel/panic.c<|end_filename|>
#include <kernel/panic.h>

#include <kernel/console.h>
#include <kernel/printf.h>
#include <kernel/cpu.h>

#include <arch/interrupts.h>

static void do_panic(char *msg)
{
	console_write_str("Kernel panic: ");
	console_write_str(msg);
	console_write_str("\n");
	arch_local_interrupt_disable();
	for (;;) {
		arch_halt_cpu();
	}
}

void panic(const char *fmt, ...)
{
	static char msg[64];
	va_list ap;
	va_start(ap, fmt);
	vsnprintf(msg, ARRAY_SIZE(msg), fmt, ap);
	va_end(ap);
	do_panic(msg);
}

void __assert_fail(const char *assertion, const char *file, unsigned int line, const char *function)
{
	static char msg[64];
	sprintf(msg, "%s:%d: %s: Assertion %s failed", file, line, function, assertion);
	do_panic(msg);
}


<|start_filename|>include/uapi/manticore/config_abi.h<|end_filename|>
#ifndef __MANTICORE_UAPI_CONFIG_ABI_H
#define __MANTICORE_UAPI_CONFIG_ABI_H

/*
 * Ethernet device configuration options:
 */
enum {
	/* The MAC address of an ethernet interface.  */
	CONFIG_ETHERNET_MAC_ADDRESS = 0,
	/* The I/O queue of the ethernet interface.  */
	CONFIG_IO_QUEUE = 1,
};

#endif


<|start_filename|>arch/x86_64/platform.c<|end_filename|>
#include <kernel/platform.h>

#include <kernel/page-alloc.h>
#include <kernel/printf.h>
#include <kernel/initrd.h>
#include <kernel/align.h>
#include <kernel/memory.h>

#include <arch/vmem-defs.h>
#include <arch/vmem.h>

#include <stddef.h>
#include <stdint.h>

enum {
	COMMAND_LINE_TAG	= 1,
	BOOT_LOADER_NAME_TAG	= 2,
	MODULES_TAG		= 3,
	MEMORY_INFO_TAG		= 4,
	BIOS_BOOT_DEVICE_TAG	= 5,
	MEMORY_MAP_TAG		= 6,
	ELF_SYMBOLS_TAG		= 9,
	ACPI_V1_RSDP_TAG	= 14,
};

struct header {
	uint32_t total_size;
	uint32_t reserved;
};

struct tag {
	uint32_t tag;
	uint32_t size;
};

struct module {
	uint32_t mod_start;
	uint32_t mod_end;
	const char string[0];
};

struct memory_map {
	uint32_t entry_size;
	uint32_t entry_version;
};

struct memory_map_entry {
	uint64_t base_addr;
	uint64_t length;
	uint32_t type;
	uint32_t reserved;
};

enum {
	MEMORY_TYPE_AVAILABLE = 1,
	MEMORY_TYPE_RESERVED = 2,
	MEMORY_TYPE_ACPI_RECLAIMABLE = 3,
	MEMORY_TYPE_NVS = 4,
	MEMORY_TYPE_BADRAM = 5,
};

static const char *memory_types[] = {
	[MEMORY_TYPE_AVAILABLE]		= "available",
	[MEMORY_TYPE_RESERVED]		= "reserved",
	[MEMORY_TYPE_ACPI_RECLAIMABLE]	= "ACPI reclaimable",
	[MEMORY_TYPE_NVS]		= "NVS",
	[MEMORY_TYPE_BADRAM]		= "bad RAM",
};

struct memory_region mem_regions[MAX_MEM_REGIONS];
size_t nr_mem_regions;

static void parse_boot_loader_name(struct tag *tag, void *data)
{
	const char *boot_loader_name = data + sizeof(*tag);

	printf("Boot loader: '%s'\n", boot_loader_name);
}

static void parse_module(struct tag *tag, void *data)
{
	struct module *mod = data + sizeof(*tag);

	printf("Module: %08x-%08x %s\n", mod->mod_start, mod->mod_end, mod->string);

	initrd_start = paddr_to_ptr(mod->mod_start);
	initrd_end = paddr_to_ptr(mod->mod_end);
}

static void parse_memory_map(struct tag *tag, void *data)
{
	size_t offset = sizeof(*tag);

	struct memory_map *memory_map = data + offset;
	offset += sizeof(*memory_map);

	printf("Memory map:\n");

	for (; offset < tag->size; offset += sizeof(struct memory_map_entry)) {
		struct memory_map_entry *entry = data + offset;
		virt_t vaddr = align_up(entry->base_addr + KERNEL_VMA, PAGE_SIZE_SMALL);
		if (vaddr > kernel_vm_end) {
			kernel_vm_end = vaddr;
		}
		const char *memory_type = "unknown";
		if (entry->type < ARRAY_SIZE(memory_types)) {
			memory_type = memory_types[entry->type];
		}
		if (entry->length < 1024*1024) {
			printf("  %016lx-%016lx %4ld KiB [%s]\n", entry->base_addr, entry->base_addr + entry->length, entry->length / 1024, memory_type);
		} else {
			printf("  %016lx-%016lx %4ld MiB [%s]\n", entry->base_addr, entry->base_addr + entry->length, entry->length / 1024 / 1024, memory_type);
		}
		if (entry->type != MEMORY_TYPE_AVAILABLE) {
			continue;
		}
		if (nr_mem_regions >= MAX_MEM_REGIONS) {
			printf("WARNING: not registering region, increase MAX_MEM_REGIONS!");
			continue;
		}
		struct memory_region *mem_region = &mem_regions[nr_mem_regions++];
		mem_region->base = entry->base_addr;
		mem_region->len = align_down(entry->length, PAGE_SIZE_SMALL);
		extern char _kernel_end;
		uint64_t kernel_end = (uint64_t) &_kernel_end;
		uint64_t base_addr = entry->base_addr + KERNEL_VMA;
		uint64_t length = entry->length;
		if (base_addr + length < kernel_end) {
			continue;
		}
		if (base_addr + length < (uint64_t) initrd_end) {
			continue;
		}
		if (base_addr < kernel_end) {
			length -= kernel_end - base_addr;
			base_addr = kernel_end;
		}
		if (base_addr < (uint64_t) initrd_end) {
			length -= (uint64_t) initrd_end - base_addr;
			base_addr = (uint64_t) initrd_end;
		}
		base_addr = align_up(base_addr, PAGE_SIZE_SMALL);
		length = align_down(length, PAGE_SIZE_SMALL);
		memory_add_span(base_addr, length);
	}
}

extern void acpi_parse_config(void *);

static void parse_acpi_v1_rsdp(struct tag *tag, void *data)
{
	acpi_parse_config(data + sizeof(*tag));
}

/*
 * Boot data section provided by the Multiboot 2 compatible boot loader:
 */
extern void *boot_data;

void parse_platform_config(void)
{
	void *data = boot_data;
	size_t offset = 0;

	struct header *header = data + offset;
	offset += sizeof(*header);

	while (offset < header->total_size) {
		struct tag *tag = data + offset;
		if (!tag || !tag->size) {
			break;
		}
		switch (tag->tag) {
		case BOOT_LOADER_NAME_TAG:
			parse_boot_loader_name(tag, data + offset);
			break;
		case MODULES_TAG:
			parse_module(tag, data + offset);
			break;
		case MEMORY_MAP_TAG:
			parse_memory_map(tag, data + offset);
			break;
		case ACPI_V1_RSDP_TAG:
			parse_acpi_v1_rsdp(tag, data + offset);
			break;
		default:
			break;
		}
		offset += tag->size;
		offset = align_up(offset, 8);
	}
}


<|start_filename|>usr/liblinux/include/errno.h<|end_filename|>
#ifndef _ERRNO_H
#define _ERRNO_H

#define EBADF 9
#define EFAULT 14
#define EINVAL 22
#define EMFILE 24
#define ENOSYS 38
#define ENOPROTOOPT 92
#define EOPNOTSUPP 95

extern int errno;

#endif


<|start_filename|>kernel/user-copy.c<|end_filename|>
#include <kernel/user-access.h>

#include <kernel/errno.h>

#include <arch/vmem.h>

extern int __memcpy_user_safe(void __user *dest, const void *src, size_t len);

int memcpy_to_user(void __user *dest, const void *src, size_t len)
{
	/* FIXME: Make this check more strict by looking at process virtual
		  memory limits.  */
	if ((unsigned long)dest >= (unsigned long)KERNEL_VMA) {
		return -EFAULT;
	}
	return __memcpy_user_safe(dest, src, len);
}

int memcpy_from_user(void *dest, const void __user *src, size_t len)
{
	/* FIXME: Make this check more strict by looking at process virtual
		  memory limits.  */
	if ((unsigned long)src >= (unsigned long)KERNEL_VMA) {
		return -EFAULT;
	}
	return __memcpy_user_safe(dest, src, len);
}

int raw_strncpy_from_user(void *dest, const void __user *src, size_t len);

int strncpy_from_user(void *dest, const void __user *src, size_t len)
{
	/* FIXME: Make this check more strict by looking at process virtual
		  memory limits.  */
	if ((unsigned long)src >= (unsigned long)KERNEL_VMA) {
		return -EFAULT;
	}
	return raw_strncpy_from_user(dest, src, len);
}


<|start_filename|>usr/tests/tst-console_print.c<|end_filename|>
#include <assert.h>
#include <errno.h>

#include <manticore/syscalls.h>

int main(int argc, char *argv[])
{
	assert(console_print("hello\n", 6) == 6);
	assert(console_print((void *)0xdeadbeef, 1) == -EFAULT);
	assert((int)console_print((void *)0xffffffff80000000, 1) == (int)-EFAULT);

	exit(0);
}


<|start_filename|>usr/echod/Makefile<|end_filename|>
PROGRAMS = echod.linux echod.manti

CFLAGS = -Wall -O3 -g -static -fno-stack-protector

all: libs echod.linux echod.manti echod.iso

clean:
	rm -f $(PROGRAMS) echod.iso
	rm -rf build

libs:
	mkdir -p build && cd build && cmake ../.. && make
.PHONY: libs
	
echod.linux: echod.c
	gcc $(CFLAGS) echod.c -o echod.linux

echod.manti: libs echod.c
	gcc $(CFLAGS) -I../liblinux/include -nostdlib echod.c build/liblinux/liblinux.a build/libmanticore/libmanticore.a -o echod.manti

echod.iso: echod.manti ../../kernel.elf
	../../scripts/mkiso --kernel ../../kernel.elf --initrd echod.manti echod.iso


<|start_filename|>include/kernel/syscall.h<|end_filename|>
#ifndef KERNEL_SYSCALL_H
#define KERNEL_SYSCALL_H

#include <uapi/manticore/syscall_abi.h>

long syscall(int nr, ...);

#endif


<|start_filename|>usr/liblinux/include/stdio.h<|end_filename|>
#ifndef __STDIO_H
#define __STDIO_H

#include <stddef.h>
#include <stdarg.h>

struct __file {
};

typedef struct __file *FILE;

extern FILE *stdin;
extern FILE *stdout;
extern FILE *stderr;

int printf(const char *fmt, ...);
int fprintf(FILE *stream, const char *fmt, ...);

int vprintf(const char *fmt, va_list ap);
int vfprintf(FILE *stream, const char *fmt, va_list ap);

size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream);

#endif


<|start_filename|>include/kernel/sched.h<|end_filename|>
#ifndef KERNEL_SCHED_H
#define KERNEL_SCHED_H

#include <stdint.h>

void schedule();

int process_subscribe(const char *name);
void *process_getevents(void);
int process_get_config(int desc, int opt, void *buf, size_t len);
void *process_get_io_queue(void);
int process_acquire(const char *name, int flags);
void process_wait(void);
void wake_up_processes(void);
int process_vmspace_alloc(uint64_t size, uint64_t align, uint64_t *start);

#endif


<|start_filename|>usr/liblinux/src/malloc.c<|end_filename|>
/*
 * malloc() is a general-purpose dynamic memory allocator.
 *
 * The design of this allocator is inspired by mimalloc.
 *
 * Layout in memory looks as follows.
 *
 *                               Heap
 *               Segment 1                   Segment 2
 *     +--------------------------+ +--------------------------+
 *     | Page 1 | Page 2 | Page 3 | | Page 4 | Page 5 | Page 6 |
 *     +--------------------------+ +--------------------------+
 *     | B B B  | B B B  | B B B  | | B B B  | B B B  | B B B  |
 *     +--------------------------+ +--------------------------+
 *
 * The allocator maintains one or more heaps, which consists of segments that
 * are divided into pages. Each segment is 2 MiB (a large page) of contiguous
 * virtual memory, which is divided into 64 KiB pages. Each page is divided
 * into equal-sized blocks, which hold memory for objects.
 *
 * The malloc() interface allocates from the default heap. Users can create as
 * many heaps as they want and allocate from them with the
 * malloc_heap_alloc() function.
 */

#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>

#ifdef __MANTICORE_OS__ 
#include <manticore/syscalls.h>
#include <manticore/vmspace_abi.h>
#else
#include <sys/mman.h>
#endif

/* Enable malloc benchmark.  */
//#define MALLOC_BENCHMARK 1

/* Enable malloc tracing.  */
//#define MALLOC_TRACING 1

/* Enable malloc test.  */
//#define MALLOC_TEST 1

#ifdef MALLOC_TRACING
#define MALLOC_TRACE(fmt, ...)                                                                                         \
	do {                                                                                                           \
		printf("%s:%d: %s(" fmt ")\n", __FILE__, __LINE__, __func__, __VA_ARGS__);                             \
	} while (0)
#else
#define MALLOC_TRACE(fmt, ...)                                                                                         \
	do {                                                                                                           \
	} while (0)
#endif

#define malloc_likely(x) __builtin_expect((x), 1)
#define malloc_unlikely(x) __builtin_expect((x), 0)

#define malloc_align(value, align) ((value + align - 1) & ~(align - 1))

/* The minimum object size.  */
#define MIN_OBJ_SIZE_SHIFT 3
#define MIN_OBJ_SIZE (1ULL << MIN_OBJ_SIZE_SHIFT)
#define MIN_OBJ_SIZE_MASK (MIN_OBJ_SIZE - 1)

/* The size of a malloc segment.  */
#define MALLOC_SEGMENT_SIZE (1UL << 21) /* 2 MiB */

/* The size of a malloc page.  */
#define MALLOC_PAGE_SIZE (1UL << 16) /* 64 KiB */

/* The maximum size of a small object.  */
#define SMALL_OBJ_SIZE_MAX (1UL << 10) /* 1 KiB */

/* Number of size classes in heap.  */
#define NUM_SIZE_CLASSES ((SMALL_OBJ_SIZE_MAX + 7) >> MIN_OBJ_SIZE_SHIFT)

/* A block of memory that is allocated with malloc() and de-allocated
   with free(). Each block belongs to a single page.  */
struct malloc_block {
	struct malloc_block *next;
};

/* A page is a region in memory that holds one or more equal-sized
   blocks of memory. Each page belongs to a single segment.  */
struct malloc_page {
	struct malloc_block *free;
	size_t used;
	size_t capacity;
	size_t block_size;
	uint8_t idx;
	/* FIXME: hole in struct! */
};

/* A segment is a region in memory that holds one or more pages.  */
struct malloc_segment {
	struct malloc_segment *prev;
	struct malloc_segment *next;
	size_t nr_pages;
	struct malloc_page pages[]; // pages within this segment
};

/* A segment queue holds one more more segments that have free pages.  */
struct malloc_segment_queue {
	struct malloc_segment *head;
	struct malloc_segment *tail;
};

/* A heap is a region in memory that holds one more more segments.  */
struct malloc_heap {
	struct malloc_page *pages[NUM_SIZE_CLASSES];
	struct malloc_segment_queue segment_queue;
};

static struct malloc_heap malloc_default_heap;

/* The heap has pages for each size-class for quick allocation. The pages are
   initially set to this empty page so that we allocate actual pages lazily.  */
static struct malloc_page empty_page = {
	.block_size	= 0,
	.capacity	= 0,
	.used		= 0,
	.free		= NULL,
};

/* Convert an object size to a block size.  */
static inline size_t to_block_size(size_t obj_size)
{
	return (obj_size + MIN_OBJ_SIZE_MASK) & ~(MIN_OBJ_SIZE_MASK); // FIXME: use malloc_align()
}

/* Convert an object size to a size class.  */
static inline size_t to_size_class(size_t obj_size)
{
	return to_block_size(obj_size) / MIN_OBJ_SIZE;
}

/* Convert a malloc block to an object.  */
static void *malloc_block_to_obj(struct malloc_block *block)
{
	return (void *) block;
}

/* Convert a malloc object to a block.  */
static struct malloc_block *malloc_obj_to_block(void *obj)
{
	return obj;
}

/* Number of remaining free objects in a page.  */
static size_t malloc_page_remaining(struct malloc_page *page)
{
	return page->capacity - page->used;
}

/* Look up the page of an object.  */
static struct malloc_page *malloc_obj_to_page(struct malloc_segment *segment, void *obj)
{
	size_t idx = ((void *) obj - (void *) segment) / MALLOC_PAGE_SIZE;

	return &segment->pages[idx];
}

/* Append object to the page freelist.  */
static void malloc_page_append_obj(struct malloc_page *page, void *obj)
{
	struct malloc_block *block = malloc_obj_to_block(obj);
	block->next = page->free;
	page->free = block;
}

/* Allocate an object from `page`.  */
static void *malloc_page_alloc_obj(struct malloc_page *page)
{
	struct malloc_block *block = page->free;
	if (block) {
		page->free = block->next;
		page->used++;
		return malloc_block_to_obj(block);
	}
	return NULL;
}

/* Check if `page` is full and has no more available blocks.  */
static bool malloc_page_full(struct malloc_page *page)
{
	return page->free == NULL;
}

/* Look up the segment of an object.  */
static struct malloc_segment *malloc_obj_to_segment(void *obj)
{
	return obj - ((unsigned long) obj % MALLOC_SEGMENT_SIZE);
}

/* Return the size of `segment` metadata.  */
static size_t malloc_segment_metadata_size(struct malloc_segment *segment)
{
	// TODO: Align the end of segment metadata?
	return sizeof(struct malloc_segment) + (sizeof(struct malloc_page) * segment->nr_pages);
}

/* Return the object buffer for `page` in `segment`.

   Please note that the first page in segment is smaller than rest of the pages
   because segment begins with a metadata section, which steals space from the
   first page.  */
static void *malloc_segment_page_obj(struct malloc_segment *segment, struct malloc_page *page)
{
	void *segment_start = segment;
	if (page->idx == 0) {
		return segment_start + malloc_segment_metadata_size(segment);
	}
	return segment_start + (page->idx * MALLOC_PAGE_SIZE);
}

/* Return the size of `page` in `segment`.

   Please note that the first page in segment is smaller than rest of the pages
   because segment begins with a metadata section, which steals space from the
   first page.  */
static size_t malloc_segment_page_size(struct malloc_segment *segment, struct malloc_page *page)
{
	if (page->idx == 0) {
		return MALLOC_PAGE_SIZE - malloc_segment_metadata_size(segment);
	}
	return MALLOC_PAGE_SIZE;
}

/* Initialize `segment` data structure.  */
static void malloc_segment_init(struct malloc_segment *segment, size_t segment_size)
{
	MALLOC_TRACE("segment=%p, segment_size=%ld", segment, segment_size);

	segment->prev = NULL;
	segment->next = NULL;
	segment->nr_pages = segment_size / MALLOC_PAGE_SIZE;

	for (size_t page_idx = 0; page_idx < segment->nr_pages; page_idx++) {
		struct malloc_page *page = &segment->pages[page_idx];

		page->idx = page_idx;
	}
}

static void *malloc_os_alloc(size_t size, size_t align)
{
	MALLOC_TRACE("size=%lu, align=%lu", size, align);

#ifdef __MANTICORE_OS__
	struct vmspace_region vmr = {
		.size = MALLOC_SEGMENT_SIZE,
		.align = MALLOC_SEGMENT_SIZE,
	};
	if (vmspace_alloc(&vmr, sizeof(vmr)) != 0) {
		return NULL;
	}
	assert((vmr.start % MALLOC_SEGMENT_SIZE) == 0);
	return (void *) vmr.start;
#else
	void *p = mmap(NULL, size + align, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
	if (p == MAP_FAILED) {
		return NULL;
	}
	// TODO: Virtual memory up to "align" is wasted here, but it's expensive to munmap it so...
	void *q = (void*) malloc_align((size_t) p, align);
	return q;
#endif
}

/* Allocate a new segment from the OS.  */
static struct malloc_segment *malloc_heap_alloc_segment(struct malloc_heap *heap)
{
	MALLOC_TRACE("heap=%p", heap);

	void *p = malloc_os_alloc(MALLOC_SEGMENT_SIZE, MALLOC_SEGMENT_SIZE);
	if (!p) {
		return NULL;
	}
	struct malloc_segment *segment = p;
	malloc_segment_init(segment, MALLOC_SEGMENT_SIZE);
	return segment;
}

/* Return `segment queue` head.  */
static struct malloc_segment *malloc_segment_queue_head(struct malloc_segment_queue *queue)
{
	return queue->head;
}

static void malloc_segment_queue_remove(struct malloc_segment_queue *queue, struct malloc_segment *segment)
{
	if (segment == queue->head) {
		queue->head = segment->next;
	}
	if (segment == queue->tail) {
		queue->tail = segment->prev;
	}
	if (segment->prev) {
		segment->prev->next = segment->next;
		segment->prev = NULL;
	}
	if (segment->next) {
		segment->next->prev = segment->prev;
		segment->next = NULL;
	}
}

/* Remove `segment queue` head.  */
static void malloc_segment_queue_pop(struct malloc_segment_queue *queue)
{
	if (queue->head)
		malloc_segment_queue_remove(queue, queue->head);
}

/* Enqueue `segment` to `queue`.  */
static void malloc_segment_queue_enqueue(struct malloc_segment_queue *queue, struct malloc_segment *segment)
{
	segment->prev = queue->tail;
	segment->next = NULL;
	if (queue->tail) {
		queue->tail->next = segment;
		queue->tail = segment;
	} else {
		queue->head = queue->tail = segment;
	}
}

/* Claim a `page` from `segment` for given `block_size`.

   When this function completes, `page` can only be used to satisfy allocations
   for `block_size` bytes.  */
static void malloc_segment_claim_page(struct malloc_segment *segment, struct malloc_page *page, size_t block_size)
{
	MALLOC_TRACE("segment=%p, page=%p, block_size=%lu", segment, page, block_size);

	assert(page->block_size == 0); // unclaimed page

	page->block_size = block_size;
	page->used = 0;
	page->capacity = malloc_segment_page_size(segment, page) / block_size;

	/* The page freelist is a singly linked list. Add blocks in reverse
	   order to ensure that the block with the smallest address (beginning
	   of page) is in the front of the list.  */
	void *obj = malloc_segment_page_obj(segment, page);
	for (size_t idx = 0; idx < page->capacity; idx++) {
		malloc_page_append_obj(page, obj + ((page->capacity - idx - 1) * block_size));
	}
}

/* Allocate a new page from `heap` that can satisfy an allocation for `size`
   bytes.  */
static struct malloc_page *malloc_heap_alloc_page(struct malloc_heap *heap, size_t size)
{
	MALLOC_TRACE("heap=%p, size=%lu", heap, size);

	struct malloc_segment *segment;
retry:
	segment = malloc_segment_queue_head(&heap->segment_queue);
	if (!segment) {
		segment = malloc_heap_alloc_segment(heap);
		if (!segment) {
			return NULL; /* out of memory */
		}
		malloc_segment_queue_enqueue(&heap->segment_queue, segment);
	}
	size_t block_size = to_block_size(size);
	/* FIXME: replace segment->nr_pages traversal with page queue to speed it up.  */
	for (int i = 0; i < segment->nr_pages; i++) {
		struct malloc_page *page = &segment->pages[i];

		if (!page->block_size) {
			malloc_segment_claim_page(segment, page, block_size);
		} else if (page->block_size != block_size) {
			continue;
		}
		if (malloc_page_remaining(page)) {
			// TODO: remove segment from free queue when there are no pages with free blocks.
			return page;
		}
	}
	malloc_segment_queue_pop(&heap->segment_queue);
	goto retry;
}

/* Find or allocate a page from `heap` which can satisfy allocation for `size` bytes.  */
static struct malloc_page *malloc_heap_get_page(struct malloc_heap *heap, size_t size)
{
	MALLOC_TRACE("heap=%p, size=%lu", heap, size);

	size_t size_class = to_size_class(size);
	struct malloc_page *page = heap->pages[size_class];
	if (!malloc_page_full(page)) {
		return page;
	}
	page = malloc_heap_alloc_page(heap, size);
	if (page) {
		heap->pages[size_class] = page;
	}
	return page;
}

/* Allocate `size` bytes of memory from the small object pool of `heap`.  */
static void *malloc_heap_alloc_small(struct malloc_heap *heap, size_t size)
{
	MALLOC_TRACE("heap=%p, size=%lu", heap, size);

	if (!size) {
		size = sizeof(void *);
	}
	struct malloc_page *page = malloc_heap_get_page(heap, size);
	if (!page) {
		return NULL;
	}
	return malloc_page_alloc_obj(page);
}

/* Allocate `size` bytes of memory from `heap`.  */
static void *malloc_heap_alloc(struct malloc_heap *heap, size_t size)
{
	MALLOC_TRACE("heap=%p, size=%lu", heap, size);

	if (malloc_likely(size < SMALL_OBJ_SIZE_MAX)) {
		return malloc_heap_alloc_small(heap, size);
	}
	/* TODO: support for medium and large size allocations */
	return NULL;
}

/* Allocate `size` bytes of memory. */
void *malloc(size_t size)
{
	MALLOC_TRACE("size=%lu", size);

	return malloc_heap_alloc(&malloc_default_heap, size);
}

/* Free memory pointed to by `ptr` in `heap`.  */
static void malloc_heap_free(struct malloc_heap *heap, void *ptr)
{
	MALLOC_TRACE("heap=%p, ptr=%p", heap, ptr);

	struct malloc_segment *segment = malloc_obj_to_segment(ptr);
	struct malloc_page *page = malloc_obj_to_page(segment, ptr);
	// FIXME: append to page local freelist like in mimalloc
	malloc_page_append_obj(page, ptr);
	assert(page->used > 0);
	page->used--;
	if (!page->used) {
		/* TODO: free page */
	}
}

/* Free memory pointed to by `ptr`.  */
void free(void *ptr)
{
	MALLOC_TRACE("ptr=%p", ptr);

	if (!ptr) {
		return;
	}
	malloc_heap_free(&malloc_default_heap, ptr);
}

/* Initialize `heap` data structure.  */
static void malloc_heap_init(struct malloc_heap *heap)
{
	MALLOC_TRACE("heap=%p", heap);

	for (size_t idx = 0; idx < NUM_SIZE_CLASSES; idx++) {
		heap->pages[idx] = &empty_page;
	}
}

static void malloc_test(void)
{
#ifdef MALLOC_TEST
	/* malloc() recycles the same object on malloc/free */
	void *q = malloc(5);
	free(q);
	for (int i = 0; i < 100; i++) {
		void *p = malloc(5);
		assert(p == q);
		free(p);
	}

	/* malloc across different sizes */
	for (int i = 0; i < 100; i++) {
		void *p = malloc(i * 5);
		assert(p != NULL);
		free(p);
	}

	/* allocate lots of objects */
	for (int i = 0; i < 10000000; i++) {
		void *p = malloc(64);
		assert(p != NULL);
		printf("%p\n", p);
	}
	assert(0);
#endif
}

/* Initialize the dynamic memory allocator.

   Programs are expected to call this function before calling `malloc`, `free`,
   and other dynamic memory allocator APIs.  */
void __liblinux_malloc_init(void)
{
	malloc_heap_init(&malloc_default_heap);

	malloc_test();
}

#ifdef MALLOC_BENCHMARK
#include <time.h>

static uint64_t timespec_to_ns(struct timespec *ts)
{
	return ts->tv_sec * 1e9 + ts->tv_nsec;
}

static uint64_t time_diff(struct timespec *start, struct timespec *end)
{
	uint64_t start_ns = timespec_to_ns(start);
	uint64_t end_ns = timespec_to_ns(end);
	assert(start_ns < end_ns);
	return end_ns - start_ns;
}


int main(int argc, char *argv[])
{
	__liblinux_malloc_init();

	struct timespec start, end;

	if (clock_gettime(CLOCK_MONOTONIC, &start) < 0) {
		assert(0);
	}
	int nr_iter = 10000000;
	for (int i = 0; i < nr_iter; i++) {
		void *p = malloc(64);
		assert(p != NULL);
	}
	if (clock_gettime(CLOCK_MONOTONIC, &end) < 0) {
		assert(0);
	}
	printf("%.2f ns/call\n", (double) time_diff(&start, &end) / (double) nr_iter);
}
#endif


<|start_filename|>kernel/atomic-ring-buffer.c<|end_filename|>
//
// An atomic ring buffer.
//
// This is a concurrent ring buffer that the kernel and a user space process
// use to communicate with each other over shared memory. The implementation is
// a single-producer (SPSC) queue that is bounded, lock-free, and wait-free.
//

#include <kernel/atomic-ring-buffer.h>

#include <kernel/align.h>

#include <stdatomic.h>
#include <stddef.h>
#include <string.h>

struct atomic_ring_buffer *atomic_ring_buffer_new(void *buf, size_t buf_size, size_t element_size)
{
	struct atomic_ring_buffer *ret = buf;
	memset(ret, 0, sizeof(struct atomic_ring_buffer));
	ret->element_size = element_size;

	size_t capacity = buf_size - sizeof(struct atomic_ring_buffer);
	ret->capacity = capacity - (capacity % element_size);

	return ret;
}

bool atomic_ring_buffer_is_empty(struct atomic_ring_buffer *queue)
{
	uint64_t head = atomic_load_explicit(&queue->head, memory_order_acquire);
	uint64_t tail = atomic_load_explicit(&queue->tail, memory_order_acquire);
	return head == tail;
}

void *atomic_ring_buffer_front(struct atomic_ring_buffer *queue)
{
	uint64_t head = atomic_load_explicit(&queue->head, memory_order_acquire);
	uint64_t tail = atomic_load_explicit(&queue->tail, memory_order_relaxed);
	if (head == tail) {
		return NULL;
	}
	return (void *)queue->data + tail;
}

void atomic_ring_buffer_pop(struct atomic_ring_buffer *queue)
{
	unsigned long tail = atomic_load_explicit(&queue->tail, memory_order_relaxed);
	unsigned long next_tail = tail + queue->element_size;
	if (next_tail == queue->capacity) {
		next_tail = 0;
	}
	atomic_store_explicit(&queue->tail, next_tail, memory_order_release);
}

bool atomic_ring_buffer_emplace(struct atomic_ring_buffer *queue, void *element)
{
	uint64_t head = atomic_load_explicit(&queue->head, memory_order_relaxed);
	uint64_t next_head = head + queue->element_size;
	if (next_head == queue->capacity) {
		next_head = 0;
	}
	if (next_head == atomic_load_explicit(&queue->tail, memory_order_acquire)) {
		return false;
	}
	memcpy(queue->data + head, element, queue->element_size);
	atomic_store_explicit(&queue->head, next_head, memory_order_release);
	return true;
}


<|start_filename|>usr/libmanticore/src/syscalls/getevents.c<|end_filename|>
#include <manticore/syscalls.h>

int getevents(void **events)
{
	/* FIXME: Execute system call in crt0, and cache the address for this function.  */
	return syscall1(SYS_getevents, (long) events);
}


<|start_filename|>arch/x86_64/include/arch/vmem.h<|end_filename|>
#ifndef ARCH_VMEM_H
#define ARCH_VMEM_H

#include <arch/vmem-defs.h>

#include <stdint.h>

/// Physical address.
typedef uint64_t phys_t;

/// Virtual address.
typedef uint64_t virt_t;

/// Converts a virtual address to a physical address.
///
/// \param addr A virtual address to translate to its physical address.
/// \returns the physical address of \addr.
phys_t virt_to_phys(virt_t addr);

/// Converts a physical address to a virtual address.
virt_t phys_to_virt(phys_t addr);

/// Converts a virtual address pointer to physical address.
static inline phys_t ptr_to_paddr(void *page)
{
	return virt_to_phys((virt_t)page);
}

/// Converts a physical address to a virtual address pointer.
static inline void *paddr_to_ptr(phys_t paddr)
{
	return (void *)(paddr + KERNEL_VMA);
}

#endif


<|start_filename|>usr/liblinux/src/internal/arp_cache.h<|end_filename|>
#ifndef __LIBLINUX_INTERNAL_ARP_CACHE_H
#define __LIBLINUX_INTERNAL_ARP_CACHE_H

#include <linux/if_ether.h> /* for ETH_ALEN */
#include <netinet/in.h>     /* for in_addr_t */

struct arp_cache_entry {
	in_addr_t ip_addr;
	uint8_t mac_addr[ETH_ALEN];
};

#define ARP_CACHE_SHIFT 7
#define ARP_CACHE_SIZE (1 << ARP_CACHE_SHIFT)

struct arp_cache {
	struct arp_cache_entry entries[ARP_CACHE_SIZE];
};

void arp_cache_init(struct arp_cache *cache);

void arp_cache_insert(struct arp_cache *cache, in_addr_t ip_addr, uint8_t *mac_addr);

uint8_t *arp_cache_lookup(struct arp_cache *cache, in_addr_t ip_addr);

#endif


<|start_filename|>usr/liblinux/include/assert.h<|end_filename|>
#ifndef _ASSERT_H
#define _ASSERT_H

#include <stdlib.h>
#include <stdio.h>

void __assert_fail(const char *expr, const char *file, unsigned int line, const char *function);

#define assert(expr)                                                        \
	do {                                                                \
		if (!(expr)) {                                              \
			__assert_fail(#expr, __FILE__, __LINE__, __func__); \
		}                                                           \
	} while (0)

#endif


<|start_filename|>include/kernel/user-access.h<|end_filename|>
//
// Userspace access functions (i.e. data copies between kernel and userspace).
//
#ifndef KERNEL_USER_ACCESS_H
#define KERNEL_USER_ACCESS_H

#include <stddef.h>

// TODO: userspace pointer annotation
#define __user

/// Copy memory region from userspace to a kernel buffer.
int memcpy_from_user(void *dest, const void __user *src, size_t len);

/// Copy a NULL-terminated string from userspace to a kernel buffer.
int strncpy_from_user(void *dest, const void __user *src, size_t len);

/// Copy memory region from kernel space to an userspace buffer.
int memcpy_to_user(void __user *dest, const void *src, size_t len);

#endif


<|start_filename|>arch/x86_64/ioremap.c<|end_filename|>
#include <kernel/mmu.h>

#include <kernel/page-alloc.h>
#include <kernel/align.h>

#include <stddef.h>

///
/// Kernel virtual memory end address.
///
/// The ioremap() function uses this address to allocate virtual memory for I/O memory maps.
///
virt_t kernel_vm_end;

///
/// Map an I/O memory region to kernel virtual address space.
///
/// This function maps an I/O memory region to virtual address space so that the I/O memory region can be accessed by
/// the kernel.
///
/// \param paddr I/O memory region start address.
/// \param size I/O memory region size.
/// \return I/O memory region start address in virtual address space.
///
void *ioremap(phys_t io_mem_start, size_t io_mem_size)
{
	virt_t ret = kernel_vm_end;
	kernel_vm_end += align_up(io_mem_size, PAGE_SIZE_SMALL);
	mmu_map_t map = mmu_current_map();
	int err = mmu_map_range(map, ret, io_mem_start, io_mem_size, MMU_PROT_READ | MMU_PROT_WRITE, MMU_NOCACHE);
	if (err) {
		return NULL;
	}
	return (void *)ret;
}


<|start_filename|>arch/aarch64/mmu.c<|end_filename|>
#include <kernel/mmu.h>

#include <kernel/errno.h>

phys_t virt_to_phys(virt_t addr)
{
	return addr;
}

mmu_map_t mmu_current_map(void)
{
	/* FIXME: not implemented.  */
	return (mmu_map_t){ttbr0: 0, ttbr1: 0};
}

void mmu_load_map(mmu_map_t map)
{
	/* FIXME: not implemented.  */
}

int mmu_map_small_page(mmu_map_t map, virt_t vaddr, phys_t paddr, mmu_prot_t prot, mmu_flags_t flags)
{
	/* FIXME: not implemented.  */
	return -EINVAL;
}

int mmu_map_large_page(mmu_map_t map, virt_t vaddr, phys_t paddr, mmu_prot_t prot, mmu_flags_t flags)
{
	/* FIXME: not implemented.  */
	return -EINVAL;
}


<|start_filename|>usr/liblinux/src/internal/net.c<|end_filename|>
#include "internal/net.h"

#include "internal/setup.h"
#include "internal/trace.h"

#include <arpa/inet.h>
#include <linux/if_ether.h>
#include <netinet/in.h>

#include <string.h>
#include <stdio.h>

struct net_statistics {
	uint64_t	packets_dropped;
};

static struct net_statistics stats;

static bool net_input_one(struct packet_view *pk)
{
	LIBLINUX_TRACE(net_input);

	struct ethhdr *ethh = pk->start;

	if (packet_view_len(pk) < sizeof(*ethh)) {
		stats.packets_dropped++;
		packet_view_trim(pk, packet_view_len(pk));
		return false;
	}
	packet_view_trim(pk, sizeof(*ethh));

	if (ethh->h_proto == ntohs(ETH_P_IP)) {
		return ip_input(pk);
	} else if (ethh->h_proto == ntohs(ETH_P_ARP)) {
		arp_input(pk);
		return false;
	} else {
		stats.packets_dropped++;
		packet_view_trim(pk, packet_view_len(pk));
		return false;
	}
}

bool net_input(struct packet_view *pk)
{
	bool ret = false;

	void *addr = pk->start;

	size_t len = pk->end - pk->start;

	while (packet_view_len(pk) > 0) {
		ret |= net_input_one(pk);
	}
	io_complete(__liblinux_eth_ioqueue, addr, len);

	return ret;
}


<|start_filename|>usr/liblinux/src/internal/ip.c<|end_filename|>
#include "internal/net.h"

#include "internal/setup.h"
#include "internal/socket.h"
#include "internal/trace.h"

#include <arpa/inet.h>
#include <assert.h> // FIXME
#include <errno.h>
#include <netinet/ip.h>
#include <netinet/udp.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>

#define WARN(str) fprintf(stderr, "warning: %s\n", str)

struct ip_statistics {
	uint64_t	ip_datagrams_dropped;
};

static struct ip_statistics stats;

ssize_t udp_recvfrom(struct socket *sk, void *restrict buf, size_t len, int flags, struct sockaddr *restrict src_addr,
		     socklen_t *restrict addrlen)
{
	LIBLINUX_TRACE(udp_recvfrom);

	struct iphdr *iph = (void *) sk->rx_buffer;

	// FIXME: check pbuf length

	struct udphdr *udph = (void *) sk->rx_buffer + sizeof(struct iphdr);

	uint16_t udp_len = ntohs(udph->len);

	size_t data_off = sizeof(struct iphdr) + sizeof(struct udphdr);
	size_t data_len = udp_len - sizeof(struct udphdr);

	// FIXME: check pbuf length

	if (*addrlen < sizeof(struct sockaddr_in)) {
		errno = EINVAL;
		return -1;
	}
	struct sockaddr_in *saddr_in = (void *)src_addr;
	saddr_in->sin_addr.s_addr = iph->saddr;
	saddr_in->sin_port = udph->source;
	*addrlen = sizeof(struct sockaddr_in);

	// FIXME: How can we retrieve rest of the data?
	size_t nr = data_len;
	if (nr > len) {
		nr = len;
	}
	memcpy(buf, sk->rx_buffer + data_off, nr);

	return nr;
}

static inline uint64_t checksum_add(uint64_t sum, const void *data, size_t count)
{
	const uint32_t *p = data;
	while (count >= 4) {
		sum += *p++;
		count -= 4;
	}
	const uint16_t *q = (void *)p;
	if (count >= 2) {
		sum += *q++;
		count -= 2;
	}
	const uint8_t *r = (void *)q;
	if (count) {
		sum += *r++;
		count -= 1;
	}
	return sum;
}

static inline uint16_t checksum_finalize(uint64_t sum)
{
	while (sum >> 16) {
		sum = (sum & 0xffff) + (sum >> 16);
	}
	return ~sum;
}

uint16_t ipv4_checksum(const void *buf, size_t len)
{
	uint64_t sum = checksum_add(0, buf, len);

	return checksum_finalize(sum);
}

uint16_t udp_checksum(const void *buf, size_t len, in_addr_t dest_ip, in_addr_t src_ip)
{
	uint64_t sum = checksum_add(0, buf, len);

	// Add IPv4 "pseudo-header" to the sum:
	sum += src_ip & 0xffffU;
	sum += src_ip >> 16;

	sum += dest_ip & 0xffffU;
	sum += dest_ip >> 16;

	sum += htons(IPPROTO_UDP);
	sum += htons(len);

	return checksum_finalize(sum);
}

struct packet_buf {
	void *buf;
	size_t len;
	size_t capacity;
};

void packet_buf_init(struct packet_buf *pk, void *buf, size_t capacity)
{
	pk->buf = buf;
	pk->len = 0;
	pk->capacity = capacity;
}

void *packet_buf_start(struct packet_buf *pk)
{
	return pk->buf;
}

size_t packet_buf_len(struct packet_buf *pk)
{
	return pk->len;
}

void *packet_buf_reserve(struct packet_buf *pk, size_t size)
{
	assert(pk->len + size < pk->capacity);
	size_t off = pk->len;
	pk->len += size;
	return pk->buf + off;
}

void packet_buf_append(struct packet_buf *pk, const void *buf, size_t len)
{
	void *dst = packet_buf_reserve(pk, len);

	memcpy(dst, buf, len);
}

struct ethhdr *ethhdr_append(struct packet_buf *pk, const char *dest, const char *source, uint16_t proto)
{
	struct ethhdr *ethh = packet_buf_reserve(pk, sizeof(*ethh));
	if (ethh) {
		memcpy(ethh->h_dest, dest, ETH_ALEN);
		memcpy(ethh->h_source, source, ETH_ALEN);
		ethh->h_proto = htons(proto);
	}
	return ethh;
}

struct iphdr *iphdr_append(struct packet_buf *pk, uint16_t tot_len, uint8_t proto, in_addr_t dest_ip, in_addr_t src_ip)
{
	struct iphdr *iph = packet_buf_reserve(pk, sizeof(*iph));
	if (iph) {
		iph->version = 4;
		iph->ihl = 5;
		iph->tos = 0;
		iph->tot_len = htons(tot_len);
		iph->id = 0;
		iph->frag_off = htons(IP_DF);
		iph->ttl = 64;
		iph->protocol = proto;
		iph->check = 0;
		iph->saddr = src_ip;
		iph->daddr = dest_ip;
	}
	return iph;
}

struct udphdr *udphdr_append(struct packet_buf *pk, uint16_t len, in_port_t dest, in_port_t source)
{
	struct udphdr *udph = packet_buf_reserve(pk, sizeof(*udph));
	if (udph) {
		udph->source = source;
		udph->dest = dest;
		udph->len = htons(len);
		udph->check = 0;
	}
	return udph;
}

ssize_t udp_sendto(struct socket *sk, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
		   socklen_t addrlen)
{
	LIBLINUX_TRACE(udp_sendto);

	if (addrlen != sizeof(struct sockaddr_in)) {
		errno = EINVAL;
		return -1;
	}

	struct sockaddr_in *saddr_in = (void *)dest_addr;

	in_addr_t src_ip = __liblinux_host_ip;
	in_addr_t dest_ip = saddr_in->sin_addr.s_addr;

	char *dest_arp = arp_cache_lookup(&__liblinux_arp_cache, dest_ip);
	assert(dest_arp != NULL);
	char *src_arp = __liblinux_mac_addr;

	uint16_t udp_len = sizeof(struct udphdr) + len;
	uint16_t ip_len = sizeof(struct iphdr) + udp_len;

	static char _tx_buf[4096]; /* FIXME */

	struct packet_buf pk;
	packet_buf_init(&pk, _tx_buf, 4096);

	ethhdr_append(&pk, dest_arp, src_arp, ETH_P_IP);

	struct iphdr *iph = iphdr_append(&pk, ip_len, IPPROTO_UDP, dest_ip, src_ip);

	struct udphdr *udph = udphdr_append(&pk, udp_len, saddr_in->sin_port, sk->local_port);

	packet_buf_append(&pk, buf, len);

	iph->check = ipv4_checksum(iph, sizeof(*iph));

	udph->check = udp_checksum(udph, udp_len, dest_ip, src_ip);

	io_submit(__liblinux_eth_ioqueue, packet_buf_start(&pk), packet_buf_len(&pk)); /* FIXME: error handling */

	return len;
}

static void udp_input(struct packet_view *pk)
{
	LIBLINUX_TRACE(udp_input);

	const struct udphdr *udph = pk->start + sizeof(struct iphdr);

	uint16_t udp_len = ntohs(udph->len);

	struct socket *sk = socket_lookup_by_flow(udph->dest, udph->source);

	assert(sk != NULL);

	socket_input(sk, pk);

	packet_view_trim(pk, sizeof(struct iphdr) + udp_len);
}

/* Returns true if datagram is fragmented.  */
static inline bool ip_is_fragment(uint16_t frag_off)
{
	return frag_off & IP_MF || frag_off & IP_OFFMASK;
}

bool ip_input(struct packet_view *pk)
{
	LIBLINUX_TRACE(ip_input);

	const struct iphdr *iph = pk->start;

	if (packet_view_len(pk) < sizeof(*iph)) {
		goto drop_datagram;
	}
	uint16_t ip_len = ntohs(iph->tot_len);
	if (ip_len < sizeof(*iph)) {
		goto drop_datagram;
	}
	if (iph->version != 4) {
		goto drop_datagram;
	}
	/* FIXME: datagram reassembly */
	uint16_t ip_frag_off = ntohs(iph->frag_off);
	if (ip_is_fragment(ip_frag_off)) {
		goto drop_datagram;
	}

	/* FIXME: verify checksum */

	switch (iph->protocol) {
	case IPPROTO_UDP:
		udp_input(pk);
		break;
	default:
		goto drop_datagram;
	}
	return true;

drop_datagram:
	stats.ip_datagrams_dropped++;
	return false;
}


<|start_filename|>include/kernel/const.h<|end_filename|>
#ifndef KERNEL_CONST_H
#define KERNEL_CONST_H

#ifdef __ASSEMBLY__
#define _AC(x,y) x
#else
#define _AC(x,y) (x##y)
#endif

#define _UL(x) _AC(x, UL)
#define _ULL(x) _AC(x, ULL)

#define _UL_BIT(x) (_UL(1) << (x))
#define _ULL_BIT(x) (_ULL(1) << (x))

#endif


<|start_filename|>include/uapi/manticore/vmspace_abi.h<|end_filename|>
#ifndef __MANTICORE_UAPI_VMSPACE_ABI_H
#define __MANTICORE_UAPI_VMSPACE_ABI_H

#include <stdint.h>

// Virtual memory space region.
struct vmspace_region {
	// Size of the virtual memory space region in bytes (input).
	uint64_t	size;
	// Alignment of the virtual memory space region start address (input/output).
	uint64_t	align;
	// Start address of the virtual memory space region (output).
	uint64_t	start;
};

#endif


<|start_filename|>include/uapi/manticore/syscall_abi.h<|end_filename|>
#ifndef __MANTICORE_UAPI_SYSCALL_ABI_H
#define __MANTICORE_UAPI_SYSCALL_ABI_H

enum {
	SYS_exit		= 1,
	SYS_wait		= 2,
	SYS_console_print	= 3,
	SYS_subscribe		= 4,
	SYS_getevents		= 5,
	SYS_get_config		= 7,
	SYS_acquire		= 8,
	SYS_vmspace_alloc	= 9,
};

#endif


<|start_filename|>arch/x86_64/include/arch/processor.h<|end_filename|>
#ifndef X86_PROCESSOR_H
#define X86_PROCESSOR_H

#include <kernel/const.h>

/*
 * RFLAGS:
 */
#define X86_RFLAGS_IF		_UL_BIT(9)

/*
 * CR0:
 */
#define X86_CR0_PE		_UL_BIT(0)
#define X86_CR0_PG		_UL_BIT(31)

/*
 * CR4:
 */
#define X86_CR4_PAE		_UL_BIT(5)
#define X86_CR4_OSFXSR		_UL_BIT(9)
#define X86_CR4_OSXMMEXCPT	_UL_BIT(10)

/*
 * MMU:
 */
#define X86_PE_P	_UL_BIT(0)	/* Present */
#define X86_PE_RW	_UL_BIT(1)	/* Read/write */
#define X86_PE_US	_UL_BIT(2)	/* User/supervisor */
#define X86_PE_PWT	_UL_BIT(3)	/* Page-level write-through */
#define X86_PE_PCD	_UL_BIT(4)	/* Page-level cache disable */
#define X86_PE_A	_UL_BIT(5)	/* Accessed */
#define X86_PE_D	_UL_BIT(6)	/* Dirty */
#define X86_PE_PS	_UL_BIT(7)	/* Page size */
#define X86_PE_G	_UL_BIT(8)	/* Global */
#define X86_PE_XD	_UL_BIT(63)	/* Execute-disable */

#endif


<|start_filename|>arch/x86_64/Makefile<|end_filename|>
CFLAGS += -mno-sse -mno-red-zone -mcmodel=kernel
LDFLAGS += -n

includes += -Iarch/x86_64/include

objs += arch/x86_64/acpi.o
objs += arch/x86_64/apic.o
objs += arch/x86_64/cpu.o
objs += arch/x86_64/boot.o
objs += arch/x86_64/entry.o
objs += arch/x86_64/exceptions.o
objs += arch/x86_64/i8259.o
objs += arch/x86_64/ioport.o
objs += arch/x86_64/ioremap.o
objs += arch/x86_64/irq.o
objs += arch/x86_64/mmio.o
objs += arch/x86_64/mmu.o
objs += arch/x86_64/multiboot_header.o
objs += arch/x86_64/pci.o
objs += arch/x86_64/platform.o
objs += arch/x86_64/setup.o
objs += arch/x86_64/syscall.o
objs += arch/x86_64/task.o
objs += arch/x86_64/thread.o
objs += arch/x86_64/user-copy.o
objs += drivers/uart/8250.o

all: kernel.iso

kernel.iso: kernel.elf
	$(E) "  MKISO   " $@
	$(Q) ./scripts/mkiso $@

archclean:
	$(Q) rm -f kernel.iso

.PHONY: archclean


<|start_filename|>usr/liblinux/src/assert.c<|end_filename|>
#include <assert.h>

void __assert_fail(const char *expr, const char *file, unsigned int line, const char *function)
{
	fprintf(stderr, "%s:%d: %s: Assertion `%s' failed.\n", file, line, expr, function);
	exit(1);
}


<|start_filename|>usr/liblinux/src/internal/trace.h<|end_filename|>
#ifndef __LIBLINUX_INTERNAL_TRACE_H
#define __LBILINUX_INTERNAL_TRACE_H

#include <stdio.h>

#ifdef __LIBLINUX_TRACING
#  define LIBLINUX_TRACE(func) do {				\
	fprintf(stderr, "liblinux -> %s\n", #func);		\
} while (0)
#else
#  define LIBLINUX_TRACE(func)
#endif

#endif


<|start_filename|>include/kernel/platform.h<|end_filename|>
#ifndef KERNEL_PLATFORM_H
#define KERNEL_PLATFORM_H

/// Parse platform-specific configuration to, for example, discover processors, memory, and IRQs.
void parse_platform_config(void);

#endif


<|start_filename|>usr/libmanticore/include/manticore/atomic-ring-buffer.h<|end_filename|>
#ifndef KERNEL_ATOMIC_RING_BUFFER_QUEUE_H
#define KERNEL_ATOMIC_RING_BUFFER_QUEUE_H

#include <manticore/atomic_ring_buffer_abi.h>

#include <stdbool.h>

bool atomic_ring_buffer_is_empty(struct atomic_ring_buffer *queue);
void *atomic_ring_buffer_front(struct atomic_ring_buffer *queue);
void atomic_ring_buffer_pop(struct atomic_ring_buffer *queue);
bool atomic_ring_buffer_emplace(struct atomic_ring_buffer *queue, void *element);

#endif


<|start_filename|>usr/liblinux/src/arpa/inet.c<|end_filename|>
#include <arpa/inet.h>

#include <limits.h>

in_addr_t inet_addr(const char *cp)
{
	uint32_t ret = 0, acc = 0;

	/* FIXME: Fail if "cp" is not a valid IPv4 address. */

	for (;;) {
		int c = *cp++;

		switch (c) {
		case '0' ... '9': {
			int digit = c - '0';

			acc = acc * 10 + digit;

			if (acc > UCHAR_MAX) {
				return INADDR_NONE;
			}
			break;
		}
		case '.': {
			ret = (ret << 8) | acc;
			acc = 0;
			break;
		}
		case '\0': {
			ret = (ret << 8) | acc;
			return htonl(ret);
		}
		default:
			return INADDR_NONE;
		}
	}
}


<|start_filename|>usr/liblinux/src/epoll.c<|end_filename|>
#include <sys/epoll.h>

#include <errno.h>

#include <manticore/atomic-ring-buffer.h>
#include <manticore/events.h>
#include <manticore/syscalls.h>

#include "internal/net.h"

#define EPOLL_FD	200

static int nr_epoll_fds = 0;

static int do_epoll_create(int flags)
{
	if (flags != 0) {
		errno = EINVAL;
		return -1;
	}
	if (nr_epoll_fds++ > 0) {
		errno = EMFILE;
		return -1;
	}
	return EPOLL_FD;
}

int epoll_create(int size)
{
	if (size <= 0) {
		errno = EINVAL;
		return -1;
	}
	return do_epoll_create(0);
}

int epoll_create1(int flags)
{
	return do_epoll_create(flags);
}

static struct epoll_event interest_set[1]; // FIXME: make bigger

int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)
{
	if (epfd != EPOLL_FD) {
		errno = EBADF;
		return -1;
	}
	interest_set[0] = *event;
	return 0;
}

int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout)
{
	int err;

	if (epfd != EPOLL_FD) {
		errno = EBADF;
		return -1;
	}
	if (maxevents <= 0) {
		errno = EINVAL;
		return -1;
	}

	err = wait();
	if (err) {
		errno = -err;
		return -1;
	}

	struct atomic_ring_buffer *queue;
	err = getevents((void **)&queue);
	if (err) {
		errno = -err;
		return -1;
	}

	int nr_events = 0;
	while (!atomic_ring_buffer_is_empty(queue) && nr_events < maxevents) {
		struct event *kern_event = atomic_ring_buffer_front(queue);
		uint32_t ep_events = 0;

		switch (kern_event->type) {
		case EVENT_PACKET_RX: {
			struct packet_view pk = {
				.start = kern_event->addr,
				.end = kern_event->addr + kern_event->len,
			};
			if (net_input(&pk)) {
				struct epoll_event *ep_event = &events[nr_events++];
				*ep_event = interest_set[0]; // FIXME
				ep_event->events = EPOLLIN;
			}
			break;
		}
		default:
			break;
		}
		atomic_ring_buffer_pop(queue);
	}
	return nr_events;
}


<|start_filename|>arch/aarch64/setup.c<|end_filename|>
#include <arch/setup.h>

void arch_early_setup(void)
{
	parse_platform_config();
}

void arch_late_setup(void)
{
}


<|start_filename|>usr/liblinux/include/linux/if_ether.h<|end_filename|>
#ifndef __LINUX_IF_ETHER_H
#define __LINUX_IF_ETHER_H

#include <stdint.h>

#define ETH_ALEN 6

#define ETH_P_IP 0x0800
#define ETH_P_ARP 0x0806

struct ethhdr {
	uint8_t h_dest[ETH_ALEN];
	uint8_t h_source[ETH_ALEN];
	uint16_t h_proto;
} __attribute__((packed));

#endif


<|start_filename|>include/uapi/manticore/events.h<|end_filename|>
#ifndef __MANTICORE_UAPI_EVENTS_H
#define __MANTICORE_UAPI_EVENTS_H

#include <stddef.h>

enum {
	EVENT_PACKET_RX = 0x01,
};

struct event {
	size_t type;
	void *addr;
	size_t len;
};

#endif


<|start_filename|>arch/aarch64/include/arch/mmu.h<|end_filename|>
#ifndef __MANTICORE_AARCH64_MMU_H
#define __MANTICORE_AARCH64_MMU_H

#include <stdint.h>

/// MMU translation map
typedef struct {
	uint64_t ttbr0;
	uint64_t ttbr1;
} mmu_map_t;

#endif


<|start_filename|>usr/libmanticore/src/syscalls/acquire.c<|end_filename|>
#include <manticore/syscalls.h>

int acquire(const char *event, int flags)
{
	return syscall2(SYS_acquire, (long) event, (long) flags);
}


<|start_filename|>usr/liblinux/src/internal/net.h<|end_filename|>
#ifndef __LIBLINUX_INTERNAL_NET_H
#define __LIBLINUX_INTERNAL_NET_H

#include <assert.h>
#include <stdbool.h>
#include <stddef.h>
#include <sys/socket.h>

struct socket;

/// A packet descriptor.
///
/// A packet descriptor specifies an (start, end) tuple that points to a
/// contiguous memory area that contains one packet.
struct packet_view {
	void *start;
	void *end;
};

/// Returns the length of the packet pointed to by \pk
static inline size_t packet_view_len(struct packet_view *pk)
{
	return pk->end - pk->start;
}

/// Trims \size bytes from the packet descriptor \pk.
static inline void packet_view_trim(struct packet_view *pk, size_t size)
{
	assert(packet_view_len(pk) >= size);

	pk->start += size;
}

/// Forward a packet descritpro to the network stack.
///
/// \return @true if packet caused an epoll event; otherwise returns @false.
bool net_input(struct packet_view *pk);

bool net_input(struct packet_view *pk);
bool ip_input(struct packet_view *pk);
void arp_input(struct packet_view *pk);

ssize_t udp_recvfrom(struct socket *sk, void *restrict buf, size_t len, int flags, struct sockaddr *restrict src_addr, socklen_t *restrict addrlen);
ssize_t udp_sendto(struct socket *sk, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);

#endif


<|start_filename|>usr/libmanticore/src/io_queue.c<|end_filename|>
#include <manticore/io_queue.h>

#include <manticore/atomic-ring-buffer.h>
#include <manticore/io_queue_abi.h>
#include <manticore/syscalls.h>

static int __io_queue_append(io_queue_t queue, enum io_opcode opcode, void *addr, size_t len)
{
	struct atomic_ring_buffer *buf = queue;
	struct io_cmd io_cmd = {
	    .opcode = opcode,
	    .addr = addr,
	    .len = len,
	};
	atomic_ring_buffer_emplace(buf, &io_cmd);

	return 0;
}

int io_submit(io_queue_t queue, void *addr, size_t len)
{
	return __io_queue_append(queue, IO_OPCODE_SUBMIT, addr, len);
}

int io_complete(io_queue_t queue, void *addr, size_t len)
{
	return __io_queue_append(queue, IO_OPCODE_COMPLETE, addr, len);
}


<|start_filename|>usr/tests/tst-vmspace_alloc.c<|end_filename|>
#include <assert.h>
#include <errno.h>

#include <manticore/syscalls.h>
#include <manticore/vmspace_abi.h>

int main(int argc, char *argv[])
{
	struct vmspace_region vmr = {
		.size = 4096,
		.align = 4096,
	};
	assert(vmspace_alloc(&vmr, sizeof(vmr)) == 0);
	assert(vmr.start != 0);
	assert(vmr.size == 4096);
	unsigned char *p = (void *) vmr.start;
	for (int i = 0; i < 4096; i++) {
		p[i] = i % 0xff;
	}
	for (int i = 0; i < 4096; i++) {
		assert(p[i] == (i % 0xff));
	}
	exit(0);
}


<|start_filename|>arch/x86_64/apic.c<|end_filename|>
/*
 * Advanced programmable interrupt controller (APIC) driver for x86
 *
 * The APIC is local to a CPU, and is responsible for interrupt handling,
 * timers, and inter-process interrupts (IPIs).
 */
#include <arch/apic.h>

#include <arch/cpu.h>
#include <arch/cpuid.h>
#include <arch/msr.h>

#include <kernel/printf.h>
#include <kernel/panic.h>
#include <kernel/irq.h>

#include <stdbool.h>

/* Local APIC registers in MSR offsets. Specified in Table 10-6 ("Local APIC
   Register Address Map Supported by x2APIC") of Intel SDM.  */
enum {
	APIC_EOI = 0x80b,
	APIC_SPIV = 0x80f,
	APIC_LVT_TIMER  = 0x832,
	ACPI_LVT_LINT0  = 0x835,
	APIC_TIMER_IC = 0x838,		// Timer initial count (IC) register
	APIC_TIMER_CC = 0x839,		// Timer current count (DC) register
	APIC_TIMER_DC  = 0x83e,		// Timer divide configuration (DD) register
};

enum {
	APIC_LVT_TIMER_PERIODIC = 0x01 << 17,
};

/* MSI message data register flags. Specified in Section 10.11.2
   ("Message Data Register Format)") of Intel SDM. */
enum {
	/* Delivery mode: */
	__MSI_DELIVERY_SHIFT = 8,
	MSI_DELIVERY_FIXED = 0b000UL << __MSI_DELIVERY_SHIFT,

	/* Trigger mode: */
	__MSI_TRIGGER_SHIFT = 15,
	MSI_TRIGGER_EDGE = 1UL << __MSI_TRIGGER_SHIFT,
};

static uint64_t _apic_base;

void apic_compose_msi_msg(struct msi_message *msg, uint8_t vector, uint8_t dest_id)
{
	msg->msg_addr = (_apic_base & 0xfff00000ULL) | (dest_id << 12);
	msg->msg_data = MSI_TRIGGER_EDGE | MSI_DELIVERY_FIXED | vector;
}

static void apic_write(uint32_t addr, uint64_t val)
{
	wrmsr(addr, val);
}

static void apic_timer_intr(void *arg)
{
	/* An interrupt wakes up the kernel from its idle loop that blocks on
	   an arch_halt_cpu() call.  */
}

static void apic_eoi(void)
{
	apic_write(APIC_EOI, 0);
}

void end_of_interrupt(void)
{
	apic_eoi();
}

static void apic_timer_init(void)
{
	irq_vector_t vector = request_irq(apic_timer_intr, NULL);
	if (vector < 0) {
		panic("Unable to initialize APIC timer.");
	}
	apic_write(APIC_LVT_TIMER, vector | APIC_LVT_TIMER_PERIODIC);
	apic_write(APIC_TIMER_DC, 0x3);
	/* TODO: Configure timer counter using unit of time. */
	apic_write(APIC_TIMER_IC, 100000000);
}

static inline bool probe_x2apic(void)
{
	uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
	cpuid(0x01, &eax, &ebx, &ecx, &edx);
	return ecx & X86_CPUID_FEATURE_ECX_X2APIC;
}

bool apic_is_bsp(void)
{
	return (rdmsr(X86_IA32_APIC_BASE) & X86_IA32_APIC_BASE_BSP) == X86_IA32_APIC_BASE_BSP;
}

void init_apic(void)
{
	if (!probe_x2apic()) {
		printf("No x2APIC found\n");
		return;
	}
	_apic_base = rdmsr(X86_IA32_APIC_BASE);
	wrmsr(X86_IA32_APIC_BASE, _apic_base | X86_IA32_APIC_BASE_EXTD | X86_IA32_APIC_BASE_EN);

	_apic_base &= ~X86_IA32_APIC_BASE_BSP;
	printf("Found x2APIC at %lx\n", _apic_base);

	apic_write(ACPI_LVT_LINT0, 0);
	apic_write(APIC_SPIV, 0x1ff);

	apic_timer_init();
}


<|start_filename|>usr/libmanticore/include/manticore/syscalls.h<|end_filename|>
#ifndef MANTICORE_SYSCALLS_H
#define MANTICORE_SYSCALLS_H

#include <manticore/syscall_abi.h>
#include <manticore/types.h>

#include <stddef.h>

struct vmspace_region;

void exit(int status) __attribute__ ((noreturn));
int wait(void);
ssize_t console_print(const char *text, size_t count);
int acquire(const char *name, int flags);
int subscribe(const char *event);
int getevents(void **events);
int get_config(int desc, int opt, void *buf, size_t len);
int vmspace_alloc(struct vmspace_region *, size_t size);

long syscall0(long number);
long syscall1(long number, long arg0);
long syscall2(long number, long arg0, long arg1);
long syscall4(long number, long arg0, long arg1, long arg2, long arg3);

#endif


<|start_filename|>usr/liblinux/src/internal/setup.c<|end_filename|>
#include "internal/setup.h"

#include "internal/arp_cache.h"

#include <manticore/config_abi.h>
#include <manticore/syscalls.h>

#include <assert.h>
#include <arpa/inet.h>
#include <linux/if_ether.h>
#include <stdio.h>

char __liblinux_mac_addr[ETH_ALEN];

io_queue_t __liblinux_eth_ioqueue;

// FIXME: This is the default QEMU SLIPR guest IP address. Make it configurable.
#define HOST_IP_ADDR "10.0.2.15"

uint32_t __liblinux_host_ip;

struct arp_cache __liblinux_arp_cache;

extern void __liblinux_malloc_init(void);

void __liblinux_setup(void)
{
	int eth_desc = acquire("/dev/eth", 0);
	if (eth_desc < 0) {
		assert(0);
	}

	get_config(eth_desc, CONFIG_IO_QUEUE, &__liblinux_eth_ioqueue, sizeof(io_queue_t));

	get_config(eth_desc, CONFIG_ETHERNET_MAC_ADDRESS, __liblinux_mac_addr, ETH_ALEN);

	fprintf(stderr, "MAC address = %02x:%02x:%02x:%02x:%02x:%02x\n", __liblinux_mac_addr[0], __liblinux_mac_addr[1],
		__liblinux_mac_addr[2], __liblinux_mac_addr[3], __liblinux_mac_addr[4], __liblinux_mac_addr[5]);

	__liblinux_host_ip = inet_addr(HOST_IP_ADDR);

	fprintf(stderr, "IP address = %s\n", HOST_IP_ADDR);

	arp_cache_init(&__liblinux_arp_cache);

	__liblinux_malloc_init();
}


<|start_filename|>arch/x86_64/include/arch/mmu.h<|end_filename|>
#ifndef __MANTICORE_X86_MMU_H
#define __MANTICORE_X86_MMU_H

#include <stdint.h>

/// MMU translation map
typedef struct {
	uint64_t cr3;
} mmu_map_t;

#endif


<|start_filename|>usr/liblinux/src/string/strerror.c<|end_filename|>
#include <string.h>

#include <errno.h>

char *strerror(int errnum)
{
	switch (errnum) {
	case EBADF:
		return "Bad file descriptor";
	case EINVAL:
		return "Invalid argument";
	case EMFILE:
		return "Too many open files";
	case ENOSYS:
		return "Invalid system call number";
	default:
		return "Unknown error";
	}
}


<|start_filename|>usr/libmanticore/src/syscalls/subscribe.c<|end_filename|>
#include <manticore/syscalls.h>

int subscribe(const char *event)
{
	return syscall1(SYS_subscribe, (long) event);
}


<|start_filename|>usr/libmanticore/src/syscalls/vmspace_alloc.c<|end_filename|>
#include <manticore/syscalls.h>

int vmspace_alloc(struct vmspace_region *vmr, size_t size)
{
	return syscall2(SYS_vmspace_alloc, (long) vmr, (long) size);
}


<|start_filename|>usr/libmanticore/include/manticore/io_queue.h<|end_filename|>
#ifndef __LIBMANTICORE_IO_QUEUE_H
#define __LIBMANTICORE_IO_QUEUE_H

#include <stddef.h>

#include <manticore/io_queue_abi.h>

int io_submit(io_queue_t queue, void *addr, size_t len);

int io_complete(io_queue_t queue, void *addr, size_t len);

#endif


<|start_filename|>usr/liblinux/src/internal/arp_cache.c<|end_filename|>
/*
 * ARP cache.
 */

#include "internal/arp_cache.h"

#include <string.h>

/* Initialize ARP cache.  */
void arp_cache_init(struct arp_cache *cache)
{
	memset(cache, 0, sizeof(*cache));
}

/* Hash an IP address.  */
static inline uint32_t arp_hash(in_addr_t ip_addr)
{
	/* Multiplicative hashing.  */
	return (ip_addr * (uint32_t)2654435761U) >> (32 - ARP_CACHE_SHIFT);
}

/* Find an entry for IP address.

   This function searches the ARP cache for an entry for IP address using
   linear probe. The probe starts from the index of the provided hash modulo
   cache size, and proceeds to scan the whole cache until a match for provided
   IP address is found.

   See find_occupied_entry() and find_empty_entry() for users of this
   function.

   If succesful, returns a pointer to the entry; otherwise returns NULL.  */
static struct arp_cache_entry *find_entry(struct arp_cache *cache, in_addr_t ip_addr, uint32_t hash)
{
	for (uint32_t count = 0; count < ARP_CACHE_SIZE; count++) {
		size_t idx = (hash + count) & (ARP_CACHE_SIZE - 1);
		struct arp_cache_entry *entry = &cache->entries[idx];

		if (__builtin_expect(entry->ip_addr == ip_addr, 1)) {
			return entry;
		}
	}
	return NULL;
}

/* Find an occupied entry for IP address.

   If succesful, returns a pointer to the entry; otherwise returns NULL.  */
static struct arp_cache_entry *find_occupied_entry(struct arp_cache *cache, in_addr_t ip_addr)
{
	return find_entry(cache, ip_addr, arp_hash(ip_addr));
}

/* Find an empty entry for IP address.

   If succesful, returns a pointer to the entry; otherwise returns NULL.  */
static struct arp_cache_entry *find_empty_entry(struct arp_cache *cache, in_addr_t ip_addr)
{
	/* IP address 0.0.0.0 represents any IP address and, therefore,
	   can have no MAC address.  Let's use it to represent an empty
	   slot.  */
	return find_entry(cache, 0, arp_hash(ip_addr));
}

/* Insert IP and MAC address tuple to the ARP cache.  */
void arp_cache_insert(struct arp_cache *cache, in_addr_t ip_addr, uint8_t *mac_addr)
{
	struct arp_cache_entry *entry = find_occupied_entry(cache, ip_addr);
	if (!entry) {
		entry = find_empty_entry(cache, ip_addr);
	}
	entry->ip_addr = ip_addr;
	memcpy(entry->mac_addr, mac_addr, ETH_ALEN);
}

/* Find the MAC address for an IP address.

   If successful, returns a pointer to the MAC address; otherise returns NULL.  */
uint8_t *arp_cache_lookup(struct arp_cache *cache, in_addr_t ip_addr)
{
	struct arp_cache_entry *entry = find_occupied_entry(cache, ip_addr);
	if (entry && entry->ip_addr == ip_addr) {
		return entry->mac_addr;
	}
	return NULL;
}


<|start_filename|>usr/liblinux/include/sys/epoll.h<|end_filename|>
#ifndef __SYS_EPOLL_H
#define __SYS_EPOLL_H

#include <stdint.h>

#define EPOLL_CTL_ADD 1
#define EPOLL_CT_DEL 2
#define EPOLL_CTL_MOD 3

enum epoll_events {
	EPOLLIN = 0x0001,
	EPOLLHUP = 0x0010,
};

typedef union epoll_data {
	void *ptr;
	int fd;
	uint32_t u32;
	uint64_t u64;
} epoll_data_t;

struct epoll_event {
	uint32_t events;
	epoll_data_t data;
} __attribute__((__packed__));

int epoll_create(int size);
int epoll_create1(int flags);
int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout);

#endif


<|start_filename|>arch/x86_64/include/arch/segment.h<|end_filename|>
#ifndef ARCH_SEGMENT_H
#define ARCH_SEGMENT_H

#include <kernel/const.h>

#define X86_KERNEL_CS		0x08
#define X86_KERNEL_DS		0x10
#define X86_USER_CS		0x23
#define X86_USER_DS		0x1b
#define X86_GDT_TSS_IDX		5

#define X86_GDT_TYPE_DATA	(_ULL(2) << (32+8))
#define X86_GDT_TYPE_CODE	(_ULL(8) << (32+8))
#define X86_GDT_S		(_ULL(1) << (32+12))
#define X86_GDT_DPL(level)	(_ULL(level) << (32+13))
#define X86_GDT_P		(_ULL(1) << (32+15))
#define X86_GDT_L		(_ULL(1) << (32+21))
#define X86_GDT_DB		(_ULL(1) << (32+22))
#define X86_GDT_G		(_ULL(1) << (32+23))

#define X86_GDT_ENTRY(flags, base, limit)	\
	((((base) & _ULL(0xff000000)) << 32)	\
         | (flags)				\
         | (((limit) & _ULL(0xf0000)) << 32)	\
         | (((base) & _ULL(0x00ffffff)) << 16) 	\
         | (((limit) & _ULL(0x0ffff))))

#endif


<|start_filename|>arch/aarch64/include/arch/thread.h<|end_filename|>
#ifndef ARM64_THREAD_H
#define ARM64_THREAD_H

#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

//! The TIF_NEW flag represents a task that has never been scheduled.
//! The switch_to() function takes this into account and switches the task
//! to userspace context.
#define __TIF_NEW	0
#define TIF_NEW		(1ULL << __TIF_NEW)

struct task_state {
	void *sp;
	void *pc;
	uint32_t flags;
};

struct task_state *task_state_new(void *pc, void *sp);
void task_state_delete(struct task_state *task_state);
void task_state_init(struct task_state *task_state, void *pc, void *sp);
void *task_state_stack_top(struct task_state *task_state);
void *task_state_entry_point(struct task_state *task_state);

void switch_to(struct task_state *old, struct task_state *new);
void switch_to_first(struct task_state *task_state);
void ret_to_usespace(void *pc, void *sp);


#endif


<|start_filename|>usr/liblinux/src/internal/socket.c<|end_filename|>
/*
 * User-level socket interface.
 */

#include "internal/socket.h"

#include "internal/net.h"

#include <errno.h>
#include <netinet/in.h>
#include <string.h>

static const struct socket_operations udp_socket_ops = {
    .recvfrom = udp_recvfrom,
    .sendto = udp_sendto,
};

/* Maximum number of supported sockets.  */
#define MAX_SOCKETS 8
static struct socket sockets[MAX_SOCKETS];
static size_t nr_sockets;

/* The starting offset of socket file descriptors.  */
#define SOCKET_FD_OFFSET 100

static const struct socket_operations *find_socket_ops(int domain, int type, int protocol)
{
	if (domain != AF_INET || type != SOCK_DGRAM || protocol != 0) {
		return NULL;
	}
	return &udp_socket_ops;
}

int socket_alloc(int domain, int type, int protocol)
{
	if (nr_sockets >= MAX_SOCKETS) {
		errno = EMFILE;
		return -1;
	}
	const struct socket_operations *ops = find_socket_ops(domain, type, protocol);
	if (!ops) {
		errno = EINVAL;
		return -1;
	}
	int sockidx = nr_sockets++;
	int sockfd = SOCKET_FD_OFFSET + sockidx;

	struct socket *sk = &sockets[sockidx];
	sk->ops = ops;
	sk->local_port = 0;

	return sockfd;
}

struct socket *socket_lookup_by_fd(int sockfd)
{
	if (sockfd < SOCKET_FD_OFFSET) {
		return NULL;
	}
	int idx = sockfd - SOCKET_FD_OFFSET;
	if (idx > MAX_SOCKETS) {
		return NULL;
	}
	return &sockets[idx];
}

struct socket *socket_lookup_by_flow(uint16_t local_port, uint16_t foreign_port)
{
	for (int i = 0; i < MAX_SOCKETS; i++) {
		// FIXME: Optimize lookup.
		struct socket *sk = &sockets[i];
		if (sk->local_port == local_port) {
			return sk;
		}
	}
	return NULL;
}

void socket_input(struct socket *sk, struct packet_view *pk)
{
	// FIXME: This overwrites existing data.
	memcpy(sk->rx_buffer, pk->start, packet_view_len(pk));
}

int socket_accept(struct socket *sk, struct sockaddr *restrict addr, socklen_t *restrict addrlen)
{
	errno = EINVAL;
	return -1;
}

int socket_bind(struct socket *sk, const struct sockaddr *addr, socklen_t addrlen)
{
	if (addrlen != sizeof(struct sockaddr_in)) {
		errno = EINVAL;
		return -1;
	}

	struct sockaddr_in *sa_in = (void *)addr;

	sk->local_port = sa_in->sin_port;

	/* FIXME: Tell the kernel we are interested in this flow. */
	return 0;
}

int socket_getsockopt(struct socket *sk, int level, int optname, void *restrict optval, socklen_t *restrict optlen)
{
	errno = ENOPROTOOPT;
	return -1;
}

int socket_listen(struct socket *sk, int backlog)
{
	errno = EOPNOTSUPP;
	return -1;
}

ssize_t socket_recvfrom(struct socket *sk, void *restrict buf, size_t len, int flags,
			struct sockaddr *restrict src_addr, socklen_t *restrict addrlen)
{
	return sk->ops->recvfrom(sk, buf, len, flags, src_addr, addrlen);
}

ssize_t socket_sendto(struct socket *sk, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
		      socklen_t addrlen)
{
	return sk->ops->sendto(sk, buf, len, flags, dest_addr, addrlen);
}

int socket_setsockopt(struct socket *sk, int level, int optname, const void *optval, socklen_t optlen)
{
	errno = ENOPROTOOPT;
	return -1;
}

int socket_shutdown(struct socket *sk, int how)
{
	return 0;
}


<|start_filename|>usr/liblinux/src/internal/setup.h<|end_filename|>
#ifndef __LIBLINUX_INTERNAL_SETUP_H
#define __LIBLINUX_INTERNAL_SETUP_H

#include <manticore/io_queue.h>

#include <linux/if_ether.h>

#include "internal/arp_cache.h"

void __liblinux_setup(void);

extern io_queue_t __liblinux_eth_ioqueue;

extern char __liblinux_mac_addr[ETH_ALEN];

extern uint32_t __liblinux_host_ip;

extern struct arp_cache __liblinux_arp_cache;

#endif


<|start_filename|>kernel/initrd.c<|end_filename|>
#include <kernel/initrd.h>

#include <kernel/elf.h>
#include <kernel/page-alloc.h>
#include <kernel/printf.h>

void *initrd_start;
void *initrd_end;

extern int process_spawn(const char *image_start, unsigned long image_size);

void initrd_load(void)
{
	if (!initrd_start) {
		printf("No initrd found.\n");
		return;
	}
	unsigned long initrd_size = initrd_end - initrd_start;

	printf("Found initrd at %p (%lu bytes)\n", initrd_start, initrd_size);

	int err = process_spawn(initrd_start, initrd_size);
	if (err) {
		printf("Failed to spawn initrd process: %d\n", err);
		return;
	}
}


<|start_filename|>arch/aarch64/platform.c<|end_filename|>
#include <kernel/platform.h>

#include <kernel/page-alloc.h>
#include <kernel/memory.h>
#include <kernel/printf.h>
#include <kernel/align.h>
#include <kernel/panic.h>

#include <stdbool.h>
#include "libfdt.h"

void *dtb;

void parse_platform_config(void)
{
	if (!dtb) {
		panic("No FDT found");
	}
	struct fdt_header *fdt = dtb;
	if (fdt_check_header(fdt) != 0) {
		panic("Bad FDT found");
	}
	printf("Found FDT at address %x\n", (unsigned long)dtb);
	int memory_node = fdt_path_offset(dtb, "/memory");
	if (memory_node < 0) {
		panic("No memory node found in FDT");
	}
	int addr_cells = fdt_address_cells(fdt, memory_node);
	if (addr_cells < 0) {
		panic("Bad address cells property");
	}
	int size_cells = fdt_size_cells(fdt, memory_node);
	if (size_cells < 0) {
		panic("Bad size cells property");
	}
	int size;
	const uint32_t *reg_prop = fdt_getprop(fdt, memory_node, "reg", &size);
	if (!reg_prop) {
		panic("No reg property found");
	}
	uint64_t memory_addr = 0;
	for (uint32_t i = 0; i < addr_cells; i++) {
		memory_addr = (memory_addr << 32) | fdt32_to_cpu(*reg_prop++);
	}
	uint64_t memory_size = 0;
	for (uint32_t i = 0; i < size_cells; i++) {
		memory_size = (memory_size << 32) | fdt32_to_cpu(*reg_prop++);
	}
	printf("Memory map:\n");
	printf("  %016lx-%016lx %4d MiB [available]\n", memory_addr, memory_addr + memory_size, memory_size / 1024 / 1024);

	extern char _kernel_end;
	uint64_t kernel_end = (uint64_t) &_kernel_end;

	if (memory_addr < kernel_end) {
		memory_size -= kernel_end - memory_addr;
		memory_addr = kernel_end;
	}
	memory_addr = align_up(memory_addr, PAGE_SIZE_SMALL);
	memory_size = align_down(memory_size, PAGE_SIZE_SMALL);
	memory_add_span(memory_addr, memory_size);
}


<|start_filename|>mm/mmu.c<|end_filename|>
#include <kernel/mmu.h>

#include <kernel/align.h>
#include <kernel/errno.h>
#include <kernel/page-alloc.h>

/// Maps virtual address range to a physical address range.
///
/// This function does not invalidate TLB. Callers are expected to do that by calling
/// mmu_invalidate_tlb().
//
/// \param map MMU translation map.
/// \param vaddr Start of virtual address range to map from.
/// \param paddr Start of physical address range to map to.
/// \param size Size of the address range to map.
/// \param flags MMU flags for the mapped address range.
/// \param prot MMU protection for the mapped address range.
///
/// \return 0 if successful
///         -EINVAL if passed parameters are invalid
///         -ENOMEM if out of memory
int mmu_map_range(mmu_map_t map, virt_t vaddr, phys_t paddr, size_t size, mmu_prot_t prot, mmu_flags_t flags)
{
	if (!is_aligned(vaddr, PAGE_SIZE_SMALL)) {
		return -EINVAL;
	}
	virt_t start = vaddr;
	virt_t end = vaddr + size;
	virt_t large_start = align_up(vaddr, PAGE_SIZE_LARGE);
	virt_t large_end = align_down(vaddr + size, PAGE_SIZE_LARGE);
	if (large_end < large_start) {
		/* No large pages: */
		for (virt_t offset = start; offset < end; offset += PAGE_SIZE_SMALL) {
			int err = mmu_map_small_page(map, offset, paddr, prot, flags);
			if (err) {
				return err;
			}
			paddr += PAGE_SIZE_SMALL;
		}
	} else {
		/* First map any small pages before the large pages: */
		if (start != large_start) {
			for (virt_t offset = start; offset < large_start; offset += PAGE_SIZE_SMALL) {
				int err = mmu_map_small_page(map, offset, paddr, prot, flags);
				if (err) {
					return err;
				}
				paddr += PAGE_SIZE_SMALL;
			}
		}
		/* Then map the large pages: */
		if (large_start != large_end) {
			for (virt_t offset = large_start; offset < large_end; offset += PAGE_SIZE_LARGE) {
				int err = mmu_map_large_page(map, offset, paddr, prot, flags);
				if (err) {
					return err;
				}
				paddr += PAGE_SIZE_LARGE;
			}
		}
		/* Finally, map left-over small pages: */
		if (large_end != end) {
			for (virt_t offset = large_end; offset < end; offset += PAGE_SIZE_SMALL) {
				int err = mmu_map_small_page(map, offset, paddr, prot, flags);
				if (err) {
					return err;
				}
				paddr += PAGE_SIZE_SMALL;
			}
		}
	}
	return 0;
}


<|start_filename|>arch/x86_64/include/arch/msr.h<|end_filename|>
#ifndef ARCH_MSR_H
#define ARCH_MSR_H

#include <kernel/const.h>

#define X86_IA32_APIC_BASE	0x0000001b
#define X86_IA32_APIC_BASE_BSP	_UL_BIT(8)
#define X86_IA32_APIC_BASE_EXTD	_UL_BIT(10)
#define X86_IA32_APIC_BASE_EN	_UL_BIT(11)

#define X86_IA32_EFER		0xc0000080
#define X86_IA32_EFER_SCE	_UL_BIT(0)
#define X86_IA32_EFER_LME	_UL_BIT(8)
#define X86_IA32_EFER_NXE	_UL_BIT(11)
#define X86_IA32_STAR		0xc0000081
#define X86_IA32_LSTAR		0xc0000082
#define X86_IA32_FMASK		0xc0000084

#endif


<|start_filename|>kernel/syscall.c<|end_filename|>
#include <kernel/syscall.h>

#include <kernel/types.h>
#include <kernel/console.h>
#include <kernel/errno.h>
#include <kernel/page-alloc.h>
#include <kernel/panic.h>
#include <kernel/sched.h>
#include <kernel/user-access.h>

#include <uapi/manticore/vmspace_abi.h>

#include <stdarg.h>
#include <stddef.h>

static int sys_exit(int status)
{
	panic("Process terminated with exit status %d.", status);
	return 0;
}

static int sys_wait(void)
{
	process_wait();

	return 0;
}

static int sys_subscribe(const char *uevent)
{
#define EVENT_SIZE 32
	char event[EVENT_SIZE];
	int err;

	err = strncpy_from_user(event, uevent, EVENT_SIZE);
	if (err < 0) {
		return err;
	}

	return process_subscribe(event);
}

static int sys_getevents(void **events)
{
	*events = process_getevents();

	return 0;
}

static int sys_get_config(int desc, int opt, void *buf, size_t len)
{
	return process_get_config(desc, opt, buf, len);
}

static int sys_acquire(const char /* __user */ *uname, int flags)
{
#define MAX_NAME_LEN 32
	char name[MAX_NAME_LEN];
	int err;

	err = strncpy_from_user(name, uname, MAX_NAME_LEN);
	if (err < 0) {
		return err;
	}
	return process_acquire(name, flags);
}

static ssize_t sys_console_print(const char /* __user */ *ubuf, size_t count)
{
	ssize_t off = 0;

	while (count) {
#define BUF_SIZE 64
		char buf[BUF_SIZE];
		size_t nr;
		int err;

		nr = count;
		if (nr > BUF_SIZE) {
			nr = BUF_SIZE;
		}
		err = memcpy_from_user(buf, ubuf + off, nr);
		if (err < 0) {
			return err;
		}
		console_write(buf, nr);
		count -= nr;
		off += nr;
	}
	return off;
}

static int sys_vmspace_alloc(struct vmspace_region /* __user */ *uvmr, size_t size)
{
	struct vmspace_region vmr;
	int err;

	if (size != sizeof(vmr)) {
		return -EINVAL;
	}
	err = memcpy_from_user(&vmr, uvmr, sizeof(vmr));
	if (err) {
		return err;
	}
	if (!vmr.align) {
		vmr.align = PAGE_SIZE_SMALL;
	}
	uint64_t start = 0;
	err = process_vmspace_alloc(vmr.size, vmr.align, &start);
	if (err) {
		return err;
	}
	vmr.start = start;
	return memcpy_to_user(uvmr, &vmr, sizeof(vmr));
}

#define SYSCALL0(fn)                                                                                                   \
	case (SYS_##fn):                                                                                               \
		do {                                                                                                   \
			return sys_##fn();                                                                             \
		} while (0)

#define SYSCALL1(fn, arg0_type)                                                                                        \
	case (SYS_##fn):                                                                                               \
		do {                                                                                                   \
			va_list args;                                                                                  \
			arg0_type arg0;                                                                                \
			va_start(args, nr);                                                                            \
			arg0 = va_arg(args, arg0_type);                                                                \
			va_end(args);                                                                                  \
			return sys_##fn(arg0);                                                                         \
		} while (0)

#define SYSCALL2(fn, arg0_type, arg1_type)                                                                             \
	case (SYS_##fn):                                                                                               \
		do {                                                                                                   \
			va_list args;                                                                                  \
			arg0_type arg0;                                                                                \
			arg1_type arg1;                                                                                \
			va_start(args, nr);                                                                            \
			arg0 = va_arg(args, arg0_type);                                                                \
			arg1 = va_arg(args, arg1_type);                                                                \
			va_end(args);                                                                                  \
			return sys_##fn(arg0, arg1);                                                                   \
		} while (0)

#define SYSCALL4(fn, arg0_type, arg1_type, arg2_type, arg3_type)                                                       \
	case (SYS_##fn):                                                                                               \
		do {                                                                                                   \
			va_list args;                                                                                  \
			arg0_type arg0;                                                                                \
			arg1_type arg1;                                                                                \
			arg2_type arg2;                                                                                \
			arg3_type arg3;                                                                                \
			va_start(args, nr);                                                                            \
			arg0 = va_arg(args, arg0_type);                                                                \
			arg1 = va_arg(args, arg1_type);                                                                \
			arg2 = va_arg(args, arg2_type);                                                                \
			arg3 = va_arg(args, arg3_type);                                                                \
			va_end(args);                                                                                  \
			return sys_##fn(arg0, arg1, arg2, arg3);                                                       \
		} while (0)

long syscall(int nr, ...)
{
	switch (nr) {
	SYSCALL1(exit, int);
	SYSCALL0(wait);
	SYSCALL2(console_print, const char *, size_t);
	SYSCALL1(subscribe, const char *);
	SYSCALL1(getevents, void **);
	SYSCALL4(get_config, int, int, void *, size_t);
	SYSCALL2(acquire, const char *, int);
	SYSCALL2(vmspace_alloc, struct vmspace_region *, size_t);
	}
	return -ENOSYS;
}


<|start_filename|>arch/x86_64/setup.c<|end_filename|>
#include <arch/setup.h>

#include <kernel/page-alloc.h>
#include <kernel/platform.h>
#include <kernel/memory.h>
#include <kernel/virtio.h>
#include <kernel/panic.h>
#include <kernel/mmu.h>
#include <kernel/pci.h>

#include <arch/exceptions.h>
#include <arch/syscall.h>
#include <arch/segment.h>
#include <arch/i8259.h>
#include <arch/apic.h>
#include <arch/task.h>
#include <arch/cpu.h>
#include <arch/gdt.h>
#include <arch/msr.h>

#include <string.h>
#include <stdint.h>

uint64_t gdt[] __attribute__ ((aligned (8))) = {
	0,
	/* Kernel code segment: */
	X86_GDT_ENTRY(X86_GDT_TYPE_CODE | X86_GDT_P | X86_GDT_S | X86_GDT_DPL(0) | X86_GDT_L, 0, 0xfffff),
	/* Kernel data segment: */
	X86_GDT_ENTRY(X86_GDT_TYPE_DATA | X86_GDT_P | X86_GDT_S | X86_GDT_DPL(0) | X86_GDT_DB, 0, 0xfffff),

	/* Userspace data segment: */
	X86_GDT_ENTRY(X86_GDT_TYPE_DATA | X86_GDT_P | X86_GDT_S | X86_GDT_DPL(3) | X86_GDT_DB, 0, 0xfffff),
	/* Userspace code segment: */
	X86_GDT_ENTRY(X86_GDT_TYPE_CODE | X86_GDT_P | X86_GDT_S | X86_GDT_DPL(3) | X86_GDT_L, 0, 0xfffff),

	/* TSS: */
	0,
	0,
};

struct gdt_desc64 gdt_desc = {
	.limit	= ARRAY_SIZE(gdt) * sizeof(uint64_t) - 1,
	.base	= (uint64_t) gdt,
};

static void init_gdt(void)
{
	asm volatile(
		"lgdt %0\n"
		:
		: "m"(gdt_desc)
		: "memory");
}

void init_mmu_map(void)
{
	void *page = page_alloc_small();
	if (!page) {
		panic("Unable to allocate kernel MMU map");
	}
	memset(page, 0, PAGE_SIZE_SMALL);
	mmu_map_t mmu_map = (mmu_map_t){ cr3: ptr_to_paddr(page) };
	for (unsigned i = 0; i < nr_mem_regions; i++) {
		struct memory_region *mem_region = &mem_regions[i];
		int err = mmu_map_range(mmu_map, mem_region->base + KERNEL_VMA, mem_region->base, mem_region->len,
					MMU_PROT_READ | MMU_PROT_WRITE | MMU_PROT_EXEC, 0);
		if (err) {
			panic("Unable to setup kernel MMU map");
		}
	}
	mmu_load_map(mmu_map);
	mmu_invalidate_tlb();
}

static void setup_nxe(void)
{
	wrmsr(X86_IA32_EFER, rdmsr(X86_IA32_EFER) | X86_IA32_EFER_NXE);
}

void arch_early_setup(void)
{
	i8259_remap();
	init_gdt();
	init_idt();
	init_task();
	init_syscall();
	parse_platform_config();
	init_mmu_map();
	init_apic();
	setup_nxe();
}

void arch_late_setup(void)
{
	virtio_register_drivers();
	pci_probe();
}


<|start_filename|>arch/aarch64/thread.c<|end_filename|>
#include <arch/thread.h>

#include <kernel/kmem.h>

void task_state_init(struct task_state *task_state, void *pc, void *sp)
{
	task_state->sp = sp;
	task_state->pc = pc;
}

struct task_state *task_state_new(void *pc, void *sp)
{
	struct task_state *ret = kmem_alloc(sizeof(struct task_state));
	if (ret) {
		ret->flags = TIF_NEW;
		ret->pc = pc;
		ret->sp = sp;
	}
	return ret;
}

void task_state_delete(struct task_state *task_state)
{
	kmem_free(task_state, sizeof(struct task_state));
}

void *task_state_stack_top(struct task_state *task_state)
{
	return task_state->sp;
}

void *task_state_entry_point(struct task_state *task_state)
{
	return task_state->pc;
}

void switch_to_first(struct task_state *new)
{
	asm volatile(
		"ldp	x9, x10, %0\n"
		"mov	sp, x9\n"
		"br	x10\n"
		"0:\n"
		:
		: "m"(new->sp)
		: "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "x30",
		  "memory");
}

void switch_to(struct task_state *old, struct task_state *new)
{
	asm volatile(
		"mov	x9, sp\n"
		"adr	x10, 0f\n"
		"stp	x9, x10, %0\n"

		"ldp	x9, x10, %1\n"
		"mov	sp, x9\n"
		"br	x10\n"
		"0:\n"
		:
		: "m"(old->sp), "m"(new->sp)
		: "x9", "x10", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "x30",
		  "memory");
}


<|start_filename|>usr/libmanticore/src/syscalls/wait.c<|end_filename|>
#include <manticore/syscalls.h>

int wait(void)
{
	return syscall0(SYS_wait);
}


<|start_filename|>arch/aarch64/Makefile<|end_filename|>
CROSS_PREFIX ?= aarch64-linux-gnu-

CFLAGS += -mgeneral-regs-only

includes += -Iarch/aarch64/include

objs += arch/aarch64/cpu.o
objs += arch/aarch64/boot.o
objs += arch/aarch64/mmu.o
objs += arch/aarch64/platform.o
objs += arch/aarch64/thread.o
objs += arch/aarch64/setup.o
objs += arch/aarch64/stubs.o
objs += drivers/uart/pl011.o

include lib/libfdt/Makefile

all: kernel.bin

kernel.bin: kernel.elf
	$(E) "  OBJCOPY " $@
	$(Q) $(CROSS_PREFIX)objcopy -O binary kernel.elf kernel.bin

archclean:
	$(Q) rm -f kernel.bin

.PHONY: archclean


<|start_filename|>usr/libmanticore/src/syscalls/get_config.c<|end_filename|>
#include <manticore/syscalls.h>

int get_config(int desc, int opt, void *buf, size_t len)
{
	return syscall4(SYS_get_config, (long) desc, (long) opt, (long) buf, (long) len);
}


<|start_filename|>include/kernel/types.h<|end_filename|>
#ifndef KERNEL_TYPES_H
#define KERNEL_TYPES_H

typedef long ssize_t;

#endif
